🪇 perf: Coalesce Ready Background Completions (#16650)

* 🪇 perf: Coalesce Ready Background Completions

* fix: Recover Receipt Batches and Gate Rolling Deploys

* fix: Fence Background Batch Handoffs and Cleanup Epochs

* fix: Bind Resumed Cleanup to Its Observed Batch

* fix: Preserve Background Results Through Recovery Handoffs

* fix: Recover Unconfirmed Manual Background Claims

* fix: Fence Automatic Recovery of Manual Handoffs

* fix: Preserve Background Reconciliation and Recovery Retention

* fix: Preserve Replayed Claims and Finish Follower Proofs

* fix: Recover Stale Background Projection Epochs

---------

Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
lia-by-librechat[bot] 2026-10-02 15:03:22 -04:00 • committed by GitHub
parent 35a2bee409
commit f88f3fb2fe
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
35 changed files with 4610 additions and 196 deletions

View file

@ -709,6 +709,8 @@ if (cluster.isMaster) {
address: server.address(),
completionResultBatchSize:
baseAppConfig?.endpoints?.agents?.backgroundTasks?.completionResultBatchSize,
completionReceiptBatching:
baseAppConfig?.endpoints?.agents?.backgroundTasks?.completionReceiptBatching,
idlePolling: baseAppConfig?.endpoints?.agents?.eventDriven?.idlePolling,
});
} catch (initErr) {

View file

@ -508,6 +508,8 @@ const startServer = async () => {
address: server.address(),
completionResultBatchSize:
appConfig?.endpoints?.agents?.backgroundTasks?.completionResultBatchSize,
completionReceiptBatching:
appConfig?.endpoints?.agents?.backgroundTasks?.completionReceiptBatching,
idlePolling: appConfig?.endpoints?.agents?.eventDriven?.idlePolling,
});
const scheduleEngineArmed = (await initializeScheduleEngine()) != null;

View file

@ -6,6 +6,7 @@ const {
createSubagentCompletionWakeupResolver,
SUBAGENT_COMPLETION_SOURCE,
createBackgroundToolCompletionWakeupResolver,
createBackgroundToolDeadClaimRecovery,
BACKGROUND_TOOL_COMPLETION_SOURCE,
createAgentQueuedTurnLifecycle,
AGENT_QUEUED_TURN_SOURCE,
@ -30,6 +31,22 @@ const backgroundToolCompletionAdapter = createBackgroundToolCompletionWakeupReso
methods,
getGenerationJob: (conversationId) => GenerationJobManager.getJob(conversationId),
getResultBatchSize: () => service.getBackgroundCompletionResultBatchSize(),
getGenerationAdmissionEvidence,
recoverDeadClaim: createBackgroundToolDeadClaimRecovery(
(...args) => service.retire(...args),
methods.releaseBackgroundToolResultClaims,
(conversationId) => GenerationJobManager.getJob(conversationId),
({ userId, conversationId, claimId }) =>
GenerationJobManager.fenceGenerationClaimForRecovery(
userId,
claimId,
conversationId,
conversationId,
),
methods.releaseAgentBackgroundToolResultClaims,
methods,
getGenerationAdmissionEvidence,
),
getWaitMaxIntervalMs: () => service.getCompletionWaitMaxIntervalMs(),
});
const eventActorAdapter = createAgentEventContinueResolver({
@ -95,6 +112,7 @@ module.exports = {
persistAgentBackgroundToolResult: service.persistBackgroundToolResult,
expediteCompletionWakeups: service.expediteCompletionWakeups,
getAgentBackgroundToolResultClaim: service.getBackgroundToolResultClaim,
getBackgroundCompletionReceiptBatching: service.getBackgroundCompletionReceiptBatching,
releaseAgentBackgroundToolResultClaims: service.releaseBackgroundToolResultClaims,
drainAgentTriggerDeliveriesForUser: service.drainUser,
prepareAgentTriggerUserPurge: service.prepareUserPurge,

View file

@ -19,6 +19,7 @@ const mockQueuedTurnLifecycle = {
jest.mock('@librechat/api', () => ({
createCheckpointDeletionReclaimer: jest.fn((getJobs) => () => getJobs('owner', 'tenant')),
createBackgroundToolDeadClaimRecovery: jest.fn(() => jest.fn()),
createAgentTriggerService: (...args) => mockCreateAgentTriggerService(...args),
createAgentContinuationResolver: jest.fn(() => jest.fn()),
createAgentEventContinueResolver: jest.fn(() => jest.fn()),

View file

@ -1,5 +1,6 @@
const {
createBackgroundToolCompletionWakeupHandler,
GenerationJobManager,
createBackgroundToolDeadClaimRecovery,
createPendingBackgroundCompletions,
createBackgroundToolResultHandler,
@ -7,12 +8,15 @@ const {
} = require('@librechat/api');
const {
listPendingAgentBackgroundToolCompletions,
getAgentBackgroundToolResultBatch,
confirmAgentBackgroundToolResultBatch,
listUndeliveredAgentTriggerTaskIds,
} = require('~/models');
const {
enqueueAgentTrigger,
persistAgentBackgroundToolResult,
getAgentBackgroundToolResultClaim,
getBackgroundCompletionReceiptBatching,
releaseAgentBackgroundToolResultClaims,
renewAgentTriggerProducerLease,
retireAgentTrigger,
@ -26,6 +30,7 @@ const preregisterBackgroundToolCompletion = createBackgroundToolCompletionWakeup
(deliveryKey, sourceId, result) =>
persistAgentBackgroundToolResult({ deliveryKey, sourceId, result }),
(deliveryKey) => expediteCompletionWakeups({ deliveryKeys: [deliveryKey] }),
getBackgroundCompletionReceiptBatching,
);
const pendingBackgroundToolCompletions = createPendingBackgroundCompletions({
@ -52,6 +57,8 @@ function createDeadBackgroundToolClaimRecovery(
getGenerationJob,
fenceGenerationClaim,
releaseAgentBackgroundToolResultClaims,
{ getAgentBackgroundToolResultBatch, confirmAgentBackgroundToolResultBatch },
(...args) => GenerationJobManager.getGenerationAdmissionEvidence(...args),
);
}

View file

@ -53,6 +53,42 @@ async function waitForSubagentTaskToSettle(
throw new Error('Timed out waiting for the detached subagent task.');
}
describe('manual reconciliation restoration', () => {
it('upgrades only the confirmed local owner and preserves the marker on replay', () => {
const registry = new BackgroundTaskRegistryClass();
const created = registry.create({
userId: 'marker-user',
conversationId: 'marker-convo',
toolCallId: 'marker-call',
toolName: 'tool',
});
if ('atCapacity' in created) throw new Error('Unexpected capacity');
registry.complete('marker-user', 'marker-convo', created.task.id, { content: 'done' });
const claim = { kind: 'manual' as const, claimId: 'poll' };
expect(registry.claimResult('marker-user', 'marker-convo', created.task.id, claim)).toBe(
'acquired',
);
expect(
registry.claimResult('marker-user', 'marker-convo', created.task.id, {
...claim,
claimId: 'foreign',
receiptReconciled: true,
}),
).toBe('claimed');
expect(created.task.resultClaim?.receiptReconciled).toBeUndefined();
expect(
registry.claimResult('marker-user', 'marker-convo', created.task.id, {
...claim,
receiptReconciled: true,
}),
).toBe('replay');
expect(registry.claimResult('marker-user', 'marker-convo', created.task.id, claim)).toBe(
'replay',
);
expect(created.task.resultClaim?.receiptReconciled).toBe(true);
});
});
describe('isBackgroundEligibleToolName', () => {
it('excludes direct-path, host-special, and machinery tools', () => {
for (const name of [

View file

@ -668,6 +668,7 @@ export interface BackgroundTask {
claimId: string;
claimedAt: number;
generationId?: string;
receiptReconciled?: true;
};
/** The declared tool may return a process-local live artifact. A terminal
* same-generation poll may therefore deliver from the local claim after it
@ -1823,7 +1824,12 @@ export class BackgroundTaskRegistryClass {
userId: string,
conversationId: string,
taskId: string,
claim: { kind: 'manual' | 'wakeup'; claimId: string; generationId?: string },
claim: {
kind: 'manual' | 'wakeup';
claimId: string;
generationId?: string;
receiptReconciled?: true;
},
): 'acquired' | 'replay' | 'claimed' | 'not_ready' {
const task = this.get(userId, conversationId, taskId);
if (task == null || task.status === 'running') {
@ -1834,9 +1840,14 @@ export class BackgroundTaskRegistryClass {
task.updatedAt = Date.now();
return 'acquired';
}
return task.resultClaim.kind === claim.kind && task.resultClaim.claimId === claim.claimId
? 'replay'
: 'claimed';
if (task.resultClaim.kind !== claim.kind || task.resultClaim.claimId !== claim.claimId) {
return 'claimed';
}
if (claim.receiptReconciled === true && task.resultClaim.receiptReconciled !== true) {
task.resultClaim.receiptReconciled = true;
task.updatedAt = Date.now();
}
return 'replay';
}
releaseResultClaim(
@ -2554,6 +2565,7 @@ export async function runCheckBackgroundTask(params: {
conversationId,
messageId: task.messageId,
claimId: existingClaim.claimId,
...(existingClaim.batchId != null && { batchId: existingClaim.batchId }),
...(existingClaim.kind === 'manual'
? {
kind: 'manual' as const,
@ -2674,6 +2686,7 @@ export async function runCheckBackgroundTask(params: {
taskId,
{
kind: 'manual',
receiptReconciled: true,
claimId: invocationId,
...(params.generationId == null ? {} : { generationId: params.generationId }),
},
@ -2744,6 +2757,7 @@ export async function runCheckBackgroundTask(params: {
taskId,
{
kind: 'manual',
receiptReconciled: true,
claimId: invocationId,
...(params.generationId == null ? {} : { generationId: params.generationId }),
},
@ -2854,6 +2868,7 @@ export async function runCheckBackgroundTask(params: {
conversationId,
messageId: claimedResult.messageId,
claimId: existingClaim.claimId,
...(existingClaim.batchId != null && { batchId: existingClaim.batchId }),
...(existingClaim.kind === 'manual'
? {
kind: 'manual' as const,
@ -3242,6 +3257,7 @@ export function getBackgroundCodeDelivery(params: {
kind: task.resultClaim.kind,
claimId: task.resultClaim.claimId,
claimedAt: new Date(task.resultClaim.claimedAt),
...(task.resultClaim.receiptReconciled === true && { receiptReconciled: true }),
...(task.resultClaim.generationId == null
? {}
: { generationId: task.resultClaim.generationId }),

View file

@ -4,7 +4,11 @@ import { BACKGROUND_TOOL_COMPLETION_SOURCE } from './backgroundCompletionWakeup'
/** Reconciles manual polling with the independent automatic-delivery receipt,
* including polls reconstructed after the process-local registry was lost. */
export async function claimBackgroundToolResult(
methods: Pick<MessageMethods, 'claimBackgroundToolResults' | 'releaseBackgroundToolResultClaims'>,
methods: Pick<
MessageMethods,
'claimBackgroundToolResults' | 'releaseBackgroundToolResultClaims'
> &
Partial<Pick<MessageMethods, 'confirmBackgroundToolResultClaim'>>,
getReceiptClaim: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'],
input: Parameters<MessageMethods['claimBackgroundToolResults']>[0],
): ReturnType<MessageMethods['claimBackgroundToolResults']> {
@ -30,7 +34,7 @@ export async function claimBackgroundToolResult(
if (messageId == null) {
throw new Error('The background result claim has no parent message identity');
}
const release = async () => {
const release = async (allowReconciled = false) => {
const released = await methods.releaseBackgroundToolResultClaims({
userId: input.userId,
conversationId: input.conversationId,
@ -38,8 +42,10 @@ export async function claimBackgroundToolResult(
taskIds: messageClaim.results.map((result) => result.taskId),
kind: input.kind,
claimId: input.claimId,
...(input.batchId != null && { batchId: input.batchId }),
...(input.kind === 'manual' && { onlyIfUnreconciled: true }),
});
if (!released) {
if (!released && !allowReconciled) {
throw new Error('The background result claim could not be released');
}
};
@ -51,6 +57,26 @@ export async function claimBackgroundToolResult(
throw error;
}
if (receiptClaim == null || receiptClaim.claimId === input.claimId) {
if (input.kind === 'manual' && methods.confirmBackgroundToolResultClaim != null) {
try {
if (
!(await methods.confirmBackgroundToolResultClaim({
userId: input.userId,
conversationId: input.conversationId,
messageId,
taskId: input.taskId,
claimId: input.claimId,
}))
) {
throw new Error('The reconciled background result claim was superseded');
}
} catch (error) {
// This CAS removes only speculative ownership. A committed or unknown
// handoff remains owned for generation-fenced recovery.
await release(true);
throw error;
}
}
return messageClaim;
}
await release();

View file

@ -65,6 +65,9 @@ export interface BackgroundToolDeadClaimRecoveryInput {
conversationId: string;
messageId: string;
claimId: string;
batchId?: string;
/** Automatic repair cannot reopen a committed manual handoff. */
onlyIfUnreconciled?: true;
/** Omitted for the legacy automatic-wakeup recovery path. */
kind?: 'manual' | 'wakeup';
/** Required to prove that a manual claim's owning generation is no longer active. */

File diff suppressed because it is too large Load diff

View file

@ -1,4 +1,7 @@
import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 } from '@librechat/data-schemas';
import {
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
} from '@librechat/data-schemas';
import type { AgentTriggerProducerLeaseStatus } from '@librechat/data-schemas';
import type { CodeApprovalMode } from 'librechat-data-provider';
import type { EnqueueBackgroundToolCompletion } from './backgroundCompletionWakeup';
@ -113,6 +116,37 @@ describe('background tool completion wakeups', () => {
jest.useRealTimers();
});
it('admits v3 receipts only when the deployment rollout gate is enabled', async () => {
const enqueue = jest.fn<
ReturnType<EnqueueBackgroundToolCompletion>,
Parameters<EnqueueBackgroundToolCompletion>
>(async () => ({ deliveryKey: 'key' }));
let enabled = false;
const notify = createBackgroundToolCompletionWakeupHandler(
enqueue,
async () => true,
async () => true,
undefined,
undefined,
() => enabled,
);
await notify(registration());
expect(enqueue).toHaveBeenLastCalledWith(
expect.anything(),
expect.objectContaining({
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
}),
);
enabled = true;
await notify(registration({ taskId: 'task-2' }));
expect(enqueue).toHaveBeenLastCalledWith(
expect.anything(),
expect.objectContaining({
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
}),
);
});
it('expedites its own delivery when a result it can consume appears', async () => {
const expedite = jest.fn();
const notify = createBackgroundToolCompletionWakeupHandler(
@ -265,6 +299,7 @@ describe('background tool completion wakeups', () => {
userId: 'user-1',
conversationId: 'conversation-1',
messageId: 'response-1',
allowMissingMessage: true,
kind: 'wakeup',
claimId: 'batch-root-delivery',
});
@ -274,6 +309,7 @@ describe('background tool completion wakeups', () => {
conversationId: 'conversation-1',
parentMessageId: 'response-1',
claimId: 'batch-root-delivery',
recoveryFenced: true,
});
});
@ -1034,3 +1070,204 @@ describe('pending background completions', () => {
}
});
});
describe('capability-gated receipt batch resolution', () => {
function batchMethods() {
const { methods } = resolverMethods();
return {
...methods,
claimAgentBackgroundToolResultBatch: jest.fn(async () => ({
status: 'acquired' as const,
batchId: 'batch-1',
results: [
{
taskId: 'task-1',
toolCallId: 'call-1',
toolName: 'tool',
status: 'completed' as const,
output: 'one',
},
{
taskId: 'task-2',
toolCallId: 'call-2',
toolName: 'tool',
status: 'completed' as const,
output: 'two',
},
],
})),
beginAgentBackgroundToolResultBatchDispatch: jest.fn(async () => true),
confirmAgentBackgroundToolResultBatch: jest.fn(async () => true),
};
}
it('does not probe batch storage for rollout-default v2 deliveries', async () => {
const methods = batchMethods();
const resolve = createBackgroundToolCompletionWakeupResolver({
methods: methods as never,
getGenerationJob: async () => null,
});
await resolve(await envelope(), {
idempotencyKey: 'delivery-1',
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
});
expect(methods.claimAgentBackgroundToolResultBatch).not.toHaveBeenCalled();
});
it('reconciles every member before dispatch and confirms only after admission', async () => {
const methods = batchMethods();
const resolve = createBackgroundToolCompletionWakeupResolver({
methods: methods as never,
getGenerationJob: async () => null,
});
const prepared = await resolve(await envelope(), { idempotencyKey: 'delivery-1' });
expect(methods.claimBackgroundToolResults).toHaveBeenNthCalledWith(
1,
expect.objectContaining({ taskId: 'task-1' }),
);
expect(methods.claimBackgroundToolResults).toHaveBeenNthCalledWith(
2,
expect.objectContaining({ taskId: 'task-2' }),
);
expect(methods.confirmAgentBackgroundToolResultBatch).not.toHaveBeenCalled();
expect(methods.claimAgentBackgroundToolResultBatch).toHaveBeenCalledWith(
expect.objectContaining({
tenantId: 'tenant-1',
limit: 8,
maxMetadataChars: BACKGROUND_TOOL_WAKEUP_INPUT_MAX_CHARS - 256,
}),
);
if (prepared?.status !== 'ready') throw new Error('Expected ready');
expect(prepared.input).toContain('task-2');
await prepared.settleOnAdmission?.({
mode: 'continue',
status: 'started',
conversationId: 'conversation-1',
});
expect(methods.confirmAgentBackgroundToolResultBatch).toHaveBeenCalledTimes(1);
});
it('defers followers until the root has confirmed admission', async () => {
const methods = batchMethods();
methods.claimAgentBackgroundToolResultBatch
.mockResolvedValueOnce({
status: 'claimed',
claimId: 'root',
ownerStatus: 'pending',
} as never)
.mockResolvedValueOnce({
status: 'claimed',
claimId: 'root',
ownerStatus: 'applied',
} as never);
const resolve = createBackgroundToolCompletionWakeupResolver({
methods: methods as never,
getGenerationJob: async () => null,
});
await expect(resolve(await envelope(), { idempotencyKey: 'follower' })).rejects.toMatchObject({
code: 'BACKGROUND_TOOL_BATCH_PENDING',
deferWithoutAttempt: true,
});
await expect(resolve(await envelope(), { idempotencyKey: 'follower' })).resolves.toEqual({
status: 'settled',
});
expect(methods.claimBackgroundToolResults).not.toHaveBeenCalled();
});
it('uses fenced dead-owner recovery before retrying a follower as its own root', async () => {
const methods = batchMethods();
const recoverDeadClaim = jest.fn(async () => true);
methods.claimAgentBackgroundToolResultBatch.mockResolvedValueOnce({
status: 'claimed',
claimId: 'root',
ownerStatus: 'recoverable',
} as never);
const resolve = createBackgroundToolCompletionWakeupResolver({
methods: methods as never,
getGenerationJob: async () => null,
recoverDeadClaim,
});
await expect(resolve(await envelope(), { idempotencyKey: 'follower' })).rejects.toMatchObject({
code: 'BACKGROUND_TOOL_BATCH_PENDING',
});
expect(recoverDeadClaim).toHaveBeenCalledWith(
expect.objectContaining({ claimId: 'root', messageId: 'response-1' }),
);
expect((await resolve(await envelope(), { idempotencyKey: 'follower' }))?.status).toBe('ready');
});
it('releases the entire frozen batch when a sibling manual poll wins', async () => {
const methods = batchMethods();
methods.claimBackgroundToolResults
.mockResolvedValueOnce({ status: 'acquired', results: [] })
.mockResolvedValueOnce({
status: 'claimed',
claim: { kind: 'manual', claimId: 'poll' },
} as never);
const resolve = createBackgroundToolCompletionWakeupResolver({
methods: methods as never,
getGenerationJob: async () => null,
});
await expect(resolve(await envelope(), { idempotencyKey: 'root' })).rejects.toMatchObject({
code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
});
expect(methods.releaseAgentBackgroundToolResultClaims).toHaveBeenCalledWith(
expect.objectContaining({ claimId: 'root' }),
);
expect(methods.confirmAgentBackgroundToolResultBatch).not.toHaveBeenCalled();
});
it('releases receipt ownership even when no message projection has landed', async () => {
const methods = batchMethods();
methods.claimBackgroundToolResults.mockResolvedValue({ status: 'not_found' } as never);
const resolve = createBackgroundToolCompletionWakeupResolver({
methods: methods as never,
getGenerationJob: async () => null,
});
const prepared = await resolve(await envelope(), { idempotencyKey: 'root' });
if (prepared?.status !== 'ready') throw new Error('Expected ready');
await prepared.releaseOnDefiniteFailure?.();
expect(methods.releaseBackgroundToolResultClaims).not.toHaveBeenCalled();
expect(methods.releaseAgentBackgroundToolResultClaims).toHaveBeenCalled();
});
it('retains ownership when admission confirmation fails after dispatch', async () => {
const methods = batchMethods();
methods.confirmAgentBackgroundToolResultBatch.mockRejectedValueOnce(new Error('lost reply'));
const resolve = createBackgroundToolCompletionWakeupResolver({
methods: methods as never,
getGenerationJob: async () => null,
});
const prepared = await resolve(await envelope(), { idempotencyKey: 'root' });
if (prepared?.status !== 'ready') throw new Error('Expected ready');
await expect(
prepared.settleOnAdmission?.({
mode: 'continue',
status: 'started',
conversationId: 'conversation-1',
}),
).rejects.toThrow('lost reply');
expect(methods.releaseAgentBackgroundToolResultClaims).not.toHaveBeenCalled();
});
it('still detects a lost producer while a v3 receipt is not ready', async () => {
const methods = batchMethods();
methods.claimBackgroundToolResults.mockResolvedValueOnce({ status: 'not_ready', results: [] });
methods.claimAgentBackgroundToolResultBatch.mockResolvedValueOnce({
status: 'not_ready',
waitingForResult: true,
} as never);
methods.getAgentTriggerDeliveryProducerLease.mockResolvedValueOnce({
status: 'expired',
leaseUntil: new Date(0),
});
const resolve = createBackgroundToolCompletionWakeupResolver({
methods: methods as never,
getGenerationJob: async () => null,
});
await expect(resolve(await envelope(), { idempotencyKey: 'root' })).rejects.toMatchObject({
code: 'BACKGROUND_TOOL_PRODUCER_LOST',
retryable: false,
});
});
});

View file

@ -1,6 +1,10 @@
import { randomUUID } from 'node:crypto';
import { backgroundResultMetadata, isEphemeralAgentId } from 'librechat-data-provider';
import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 } from '@librechat/data-schemas';
import {
AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
} from '@librechat/data-schemas';
import type {
AgentTriggerProducerLeaseStatus,
AgentTriggerDeliveryMethods,
@ -80,6 +84,10 @@ type WakeupMethods = Pick<ConversationMethods, 'getConvo'> &
output: string;
settledAt: Date;
} | null>;
persistAgentBackgroundToolResult?: AgentTriggerDeliveryMethods['persistAgentBackgroundToolResult'];
beginAgentBackgroundToolResultBatchDispatch?: AgentTriggerDeliveryMethods['beginAgentBackgroundToolResultBatchDispatch'];
claimAgentBackgroundToolResultBatch?: AgentTriggerDeliveryMethods['claimAgentBackgroundToolResultBatch'];
confirmAgentBackgroundToolResultBatch?: AgentTriggerDeliveryMethods['confirmAgentBackgroundToolResultBatch'];
claimAgentBackgroundToolResults?: AgentTriggerDeliveryMethods['claimAgentBackgroundToolResults'];
getAgentBackgroundToolResultClaim?: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'];
releaseAgentBackgroundToolResultClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];
@ -98,6 +106,13 @@ export interface BackgroundToolCompletionWakeupResolverDeps {
methods: WakeupMethods;
getGenerationJob: (conversationId: string) => Promise<GenerationState | null>;
getResultBatchSize?: () => number | undefined;
recoverDeadClaim?: BackgroundToolDeadClaimRecovery;
getGenerationAdmissionEvidence?: (
userId: string,
clientRequestId: string,
streamId: string,
conversationId?: string,
) => Promise<{ generationId: string; generationCreatedAt: number } | null>;
/** Longest a waiting delivery re-checks readiness; the backoff default otherwise. */
getWaitMaxIntervalMs?: () => number | undefined;
}
@ -265,12 +280,14 @@ export function createBackgroundToolCompletionWakeupResolver({
getGenerationJob,
getResultBatchSize,
getWaitMaxIntervalMs,
recoverDeadClaim,
getGenerationAdmissionEvidence,
}: BackgroundToolCompletionWakeupResolverDeps): NonNullable<
AgentTriggerExecutionHostDeps['prepareContinue']
> {
const waitingRetry = (receivedAt: number): string =>
waitingRetryAfter(receivedAt, Date.now(), getWaitMaxIntervalMs?.() ?? WAITING_RETRY_CAP_MS);
return async (
const resolve = async (
envelope: AgentContinueTriggerEnvelope,
context: AgentTriggerDispatchContext,
): Promise<AgentTriggerContinuePreparation | undefined> => {
@ -332,6 +349,313 @@ export function createBackgroundToolCompletionWakeupResolver({
status: 404,
});
}
const batchScope = {
deliveryKey: context.idempotencyKey,
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
userId,
...(envelope.principal.tenantId != null && { tenantId: envelope.principal.tenantId }),
conversationId: envelope.target.conversationId,
parentMessageId: envelope.target.parentMessageId,
agentId: envelope.target.agentId,
...(context.deliveryClaimToken != null && { deliveryClaimToken: context.deliveryClaimToken }),
};
const batch =
context.requiredWorkerCapability != null &&
context.requiredWorkerCapability !==
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3
? undefined
: await methods.claimAgentBackgroundToolResultBatch?.({
...batchScope,
limit: getResultBatchSize?.() ?? 8,
maxMetadataChars: BACKGROUND_TOOL_WAKEUP_INPUT_MAX_CHARS - 256,
});
if (batch?.status === 'claimed') {
if (batch.ownerStatus === 'applied') return { status: 'settled' };
if (batch.ownerStatus === 'recoverable') {
await recoverDeadClaim?.({
userId,
conversationId: envelope.target.conversationId,
messageId: envelope.target.parentMessageId,
claimId: batch.claimId,
batchId: batch.batchId,
});
}
throw executionError('The background result is owned by another completion delivery.', {
code: 'BACKGROUND_TOOL_BATCH_PENDING',
retryable: true,
deferWithoutAttempt: true,
retryAfter: '1',
});
}
if (batch?.status === 'not_ready') {
if (batch.waitingForResult === true) {
// A producer can fail its receipt write while its terminal message
// projection succeeds. Reconstruct from that already-durable result.
const projection = await methods.claimBackgroundToolResults({
userId,
conversationId: envelope.target.conversationId,
messageId: envelope.target.parentMessageId,
taskId: registration.taskId,
agentId: envelope.target.agentId,
kind: 'wakeup',
claimId: context.idempotencyKey,
limit: 1,
});
if (
projection.status === 'claimed' &&
projection.claim?.kind === 'manual' &&
projection.claim.receiptReconciled === true
)
return { status: 'settled' };
if (projection.status === 'acquired') {
const result = projection.results.find((result) => result.taskId === registration.taskId);
const releaseProjection = async (): Promise<void> => {
if (
!(await methods.releaseBackgroundToolResultClaims({
userId,
conversationId: envelope.target.conversationId,
messageId: envelope.target.parentMessageId,
taskIds: [registration.taskId],
kind: 'wakeup',
claimId: context.idempotencyKey,
allowMissingMessage: true,
}))
) {
throw new Error('Background projection reconstruction release was not confirmed');
}
};
let restored = false;
try {
if (result != null)
restored =
(await methods.persistAgentBackgroundToolResult?.({
deliveryKey: context.idempotencyKey,
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
result: {
status: result.status,
output: truncateMiddle(
result.output,
AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
),
settledAt: result.settledAt ?? new Date(envelope.event.occurredAt),
},
})) === true;
} finally {
await releaseProjection();
}
if (restored) return resolve(envelope, context);
throw executionError('The background result receipt is being reconstructed.', {
code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
retryable: true,
deferWithoutAttempt: true,
retryAfter: '1',
});
}
if (
projection.status === 'claimed' &&
projection.claim?.kind === 'manual' &&
projection.claim.generationId != null
) {
await recoverDeadClaim?.({
userId,
conversationId: envelope.target.conversationId,
messageId: envelope.target.parentMessageId,
claimId: projection.claim.claimId,
kind: 'manual',
generationId: projection.claim.generationId,
onlyIfUnreconciled: true,
});
}
if (projection.status === 'claimed')
throw executionError('The background projection is being reconciled.', {
code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
retryable: true,
deferWithoutAttempt: true,
retryAfter: '1',
});
const producer = await methods.getAgentTriggerDeliveryProducerLease({
deliveryKey: context.idempotencyKey,
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
now: new Date(),
});
if (producer.status === 'expired') {
throw executionError('The process-local background tool executor was lost.', {
code: 'BACKGROUND_TOOL_PRODUCER_LOST',
retryable: false,
});
}
}
throw executionError('The background result batch is not durable yet.', {
code: 'BACKGROUND_TOOL_RESULT_NOT_READY',
retryable: true,
deferWithoutAttempt: true,
retryAfter: waitingRetry(envelope.receivedAt),
});
}
if (batch?.status === 'acquired') {
const owned = { ...batchScope, batchId: batch.batchId };
const dispatchId = randomUUID();
let dispatchBegan = false;
const release = async (): Promise<boolean> => {
const admitted = await getGenerationAdmissionEvidence?.(
userId,
context.idempotencyKey,
envelope.target.conversationId,
envelope.target.conversationId,
);
if (admitted != null) {
await methods.confirmAgentBackgroundToolResultBatch?.(owned);
return false;
}
// Absence is not proof. The storage CAS permits only a first definite
// failure or a never-dispatched plan, never a retry of an ambiguous POST.
const released = await methods.releaseAgentBackgroundToolResultClaims?.({
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
userId,
conversationId: envelope.target.conversationId,
parentMessageId: envelope.target.parentMessageId,
claimId: context.idempotencyKey,
batchId: batch.batchId,
...(dispatchBegan && { dispatchId }),
...(context.deliveryClaimToken != null && {
deliveryClaimToken: context.deliveryClaimToken,
}),
});
return released === true;
};
// A receipt may precede its projection. Reconcile every member, not just
// the root, against a concurrent manual poll or late message persistence.
const projections = await Promise.all(
batch.results.map((result) =>
methods.claimBackgroundToolResults({
userId,
conversationId: envelope.target.conversationId,
messageId: envelope.target.parentMessageId,
taskId: result.taskId,
agentId: envelope.target.agentId,
kind: 'wakeup',
claimId: context.idempotencyKey,
batchId: batch.batchId,
limit: 1,
}),
),
);
if (
projections.some(
(projection) =>
projection.status === 'claimed' || projection.status === 'outcome_unknown',
)
) {
for (let index = 0; index < projections.length; index++) {
const stale = projections[index];
if (stale?.status !== 'claimed' || stale.claim?.kind !== 'wakeup') continue;
const taskId = batch.results[index]!.taskId;
const receipt = await methods.getAgentBackgroundToolResultClaim?.({
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
userId,
conversationId: envelope.target.conversationId,
parentMessageId: envelope.target.parentMessageId,
taskId,
});
if (receipt?.claimId !== context.idempotencyKey || receipt.batchId !== batch.batchId)
continue;
await methods.releaseBackgroundToolResultClaims({
userId,
conversationId: envelope.target.conversationId,
messageId: envelope.target.parentMessageId,
taskIds: [taskId],
kind: 'wakeup',
claimId: stale.claim.claimId,
...(stale.claim.batchId != null && { batchId: stale.claim.batchId }),
allowMissingMessage: true,
});
throw executionError('A stale background projection is being reconciled.', {
code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
retryable: true,
deferWithoutAttempt: true,
retryAfter: '1',
});
}
const released = await release();
const rootProjection =
projections[batch.results.findIndex((result) => result.taskId === registration.taskId)];
if (
released &&
rootProjection?.status === 'claimed' &&
rootProjection.claim?.kind === 'manual' &&
rootProjection.claim.receiptReconciled === true
) {
return { status: 'settled' };
}
if (released && recoverDeadClaim != null) {
await Promise.all(
projections.map(async (projection) => {
if (
projection.status !== 'claimed' ||
projection.claim?.kind !== 'manual' ||
projection.claim.receiptReconciled === true ||
projection.claim.generationId == null
)
return;
await recoverDeadClaim({
userId,
conversationId: envelope.target.conversationId,
messageId: envelope.target.parentMessageId,
claimId: projection.claim.claimId,
kind: 'manual',
generationId: projection.claim.generationId,
onlyIfUnreconciled: true,
});
}),
);
}
throw executionError('Background result ownership is being reconciled.', {
code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
retryable: true,
deferWithoutAttempt: true,
retryAfter: '1',
});
}
return {
status: 'ready',
parentMessageId,
...(parent.codeApprovalMode != null && { codeApprovalMode: parent.codeApprovalMode }),
input: buildWakeupInput(batch.results),
releaseOnDefiniteFailure: async () => {
await release();
},
beginDispatch: async () => {
dispatchBegan = true;
if (
!(await methods.beginAgentBackgroundToolResultBatchDispatch?.({ ...owned, dispatchId }))
) {
throw executionError('The background batch preparation was superseded.', {
code: 'BACKGROUND_TOOL_BATCH_SUPERSEDED',
retryable: true,
deferWithoutAttempt: true,
});
}
},
settleOnAdmission: async (result) => {
if (
result.status === 'settled' &&
getGenerationAdmissionEvidence != null &&
(await getGenerationAdmissionEvidence(
userId,
context.idempotencyKey,
envelope.target.conversationId,
envelope.target.conversationId,
)) == null
) {
throw new Error('A recovery fence does not prove background admission');
}
if (!(await methods.confirmAgentBackgroundToolResultBatch?.(owned))) {
throw new Error('Background result batch admission was not confirmed');
}
},
};
}
const claim = await methods.claimBackgroundToolResults({
userId,
conversationId: envelope.target.conversationId,
@ -545,6 +869,7 @@ export function createBackgroundToolCompletionWakeupResolver({
deferWithoutAttempt: true,
});
};
return resolve;
}
/** Lists and discards a conversation's undelivered background completions from the
@ -652,6 +977,7 @@ export function createBackgroundToolCompletionWakeupHandler(
renewProducerLease: RenewBackgroundToolCompletionProducerLease,
persistResult?: PersistBackgroundToolCompletionResult,
expedite?: (deliveryKey: string) => void,
getReceiptBatchingEnabled?: () => boolean,
): (
registration: BackgroundToolWakeupRegistration,
) => Promise<BackgroundToolWakeupAdmission | false> {
@ -695,7 +1021,10 @@ export function createBackgroundToolCompletionWakeupHandler(
availableAt: new Date(
Math.max(Date.now(), registration.createdAt) + WAKEUP_ADMISSION_DELAY_MS,
),
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
requiredWorkerCapability:
getReceiptBatchingEnabled?.() === true
? AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3
: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
producerLeaseUntil: new Date(Date.now() + BACKGROUND_TOOL_PRODUCER_LEASE_MS),
});
return {
@ -735,8 +1064,22 @@ export function createBackgroundToolDeadClaimRecovery(
claimId: string;
}) => Promise<'fenced' | 'started' | 'unavailable'>,
releaseReceiptClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'],
batchRecovery?: Pick<
AgentTriggerDeliveryMethods,
'getAgentBackgroundToolResultBatch' | 'confirmAgentBackgroundToolResultBatch'
>,
getAdmissionEvidence?: BackgroundToolCompletionWakeupResolverDeps['getGenerationAdmissionEvidence'],
): BackgroundToolDeadClaimRecovery {
return async ({ userId, conversationId, messageId, claimId, kind, generationId }) => {
return async ({
userId,
conversationId,
messageId,
claimId,
kind,
generationId,
batchId,
onlyIfUnreconciled,
}) => {
if (kind === 'manual') {
if (generationId == null || generationId.length === 0) {
return false;
@ -753,8 +1096,41 @@ export function createBackgroundToolDeadClaimRecovery(
messageId,
kind: 'manual',
claimId,
...(onlyIfUnreconciled === true && { onlyIfUnreconciled }),
});
}
const owner = {
deliveryKey: claimId,
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
userId,
conversationId,
parentMessageId: messageId,
};
const batch = await batchRecovery?.getAgentBackgroundToolResultBatch(owner);
if (batchId != null && batch != null && batch.batchId !== batchId) return false;
if (batch?.appliedAt != null) {
await batchRecovery?.confirmAgentBackgroundToolResultBatch({
...owner,
batchId: batch.batchId,
});
return false;
}
const confirmAdmission = async (): Promise<boolean> => {
const evidence = await getAdmissionEvidence?.(
userId,
claimId,
conversationId,
conversationId,
);
if (evidence == null) return false;
if (batch != null)
await batchRecovery?.confirmAgentBackgroundToolResultBatch({
...owner,
batchId: batch.batchId,
});
return true;
};
if (batch != null && (await confirmAdmission())) return false;
const claimGenerationIsActive = async (): Promise<boolean> => {
const generation = await getGenerationJob(conversationId);
return (
@ -778,30 +1154,39 @@ export function createBackgroundToolDeadClaimRecovery(
* installs a started tombstone that invalidates a delayed creator's token,
* or observes that job creation already won. */
const generationFence = await fenceGenerationClaim({ userId, conversationId, claimId });
if (generationFence === 'unavailable') {
if (generationFence === 'unavailable') return false;
if (generationFence === 'started' && batch != null) {
// The native claim CAS proves admission even when a custom store cannot
// read historical claims or the live generation has been cleaned up.
await batchRecovery?.confirmAgentBackgroundToolResultBatch({
...owner,
batchId: batch.batchId,
});
return false;
}
if (await claimGenerationIsActive()) {
return false;
}
const released = await releaseClaims({
const receiptBatchId = batch?.batchId ?? batchId;
const receiptReleased = await releaseReceiptClaims?.({
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
userId,
conversationId,
parentMessageId: messageId,
claimId,
...(receiptBatchId != null && { batchId: receiptBatchId }),
recoveryFenced: true,
});
if (receiptReleased === false) return false;
if (batch != null) return receiptReleased === true;
return releaseClaims({
userId,
conversationId,
messageId,
allowMissingMessage: true,
kind: 'wakeup',
claimId,
...(receiptBatchId != null && { batchId: receiptBatchId }),
});
if (!released) {
return false;
}
return (
(await releaseReceiptClaims?.({
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
userId,
conversationId,
parentMessageId: messageId,
claimId,
})) ?? true
);
};
}

View file

@ -457,6 +457,59 @@ describe('createToolExecuteHandler — background tool calls', () => {
expect(retire).toHaveBeenCalledWith('background tool result was not persisted', undefined);
});
it('copies committed manual reconciliation into the final background projection retry', async () => {
let finishPersistence: (persisted: boolean) => void = () => undefined;
let resolveProjection: (() => { resultClaim?: { receiptReconciled?: true } }) | undefined;
const tool = makeSearchTool({ calls: 0 });
const handler = createToolExecuteHandler({
loadTools: async () => ({ loadedTools: [tool] }),
backgroundToolCompletion: {
preregister: async () => ({
renew: async () => true,
retire: async () => true,
persistResult: async () => true,
}),
claim: async () => ({ status: 'not_ready' }),
persist: (params) => {
resolveProjection = params.resolveBackgroundTask;
return new Promise<boolean>((resolve) => {
finishPersistence = resolve;
});
},
},
});
const [dispatch] = await runBatch(handler, {
toolCalls: [
{
id: 'manual-marker-dispatch',
stepId: 'manual-marker-step',
name: tool.name,
args: { q: 'marker', run_in_background: true },
},
],
agentId: 'agent_parent_1',
configurable: buildConfig([tool.name]),
metadata: { thread_id: 'manual-marker-convo', run_id: 'manual-marker-parent' },
});
await flushMicrotasks();
const taskId = JSON.parse(dispatch.content).background_task_id as string;
try {
expect(resolveProjection?.().resultClaim).toBeUndefined();
expect(
backgroundTaskRegistry.claimResult('exec_user', 'manual-marker-convo', taskId, {
kind: 'manual',
claimId: 'manual-marker-poll',
receiptReconciled: true,
generationId: 'manual-marker-generation',
}),
).toBe('acquired');
expect(resolveProjection?.().resultClaim).toMatchObject({ receiptReconciled: true });
} finally {
finishPersistence(true);
await flushMicrotasks();
}
});
it('falls back to the settled local result when a poll retired delivery before persistence failed', async () => {
let finishPersistence: ((persisted: boolean) => void) | undefined;
const persist = jest.fn(

View file

@ -6446,6 +6446,9 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
kind: current.resultClaim.kind,
claimId: current.resultClaim.claimId,
claimedAt: new Date(current.resultClaim.claimedAt),
...(current.resultClaim.receiptReconciled === true && {
receiptReconciled: true,
}),
...(current.resultClaim.generationId == null
? {}
: { generationId: current.resultClaim.generationId }),

View file

@ -51,6 +51,7 @@ export interface BackgroundToolResultState {
claimId: string;
claimedAt: Date;
generationId?: string;
receiptReconciled?: true;
};
}

View file

@ -269,3 +269,75 @@ event whose individual timing or acknowledgment is actionable. `fire`, `steer`,
`continue` deliveries reject the option instead of silently weakening their semantics. Deliveries
with `expectedAction` also reject coalescing because one generation cannot prove several distinct
action fences.
## Background receipt batches
Ordinary background tool and code completions use `background_tool_completion_batch_v3`.
Older workers cannot claim these rows; v2 receipts retain task-local delivery. Subagent
completions do not join receipt batches. The existing `completionResultBatchSize` bounds
selection, with no collection window. Scope is owner, tenant, conversation, launching
parent message, and target agent. Results launched on different parent messages remain
separate even when their branches later converge.
The root persists candidate keys before per-result claims, then freezes only successful
claims in `backgroundToolResultBatch.members`. Retries use that same list. Each member's
message projection is reconciled before dispatch. A first definite rejection releases the unadmitted plan. Once a handoff may have been
admitted, later transport failures retain frozen ownership and the same idempotency key. Admission proof is stored before followers settle, and copied to each
receipt so root retention cannot strand a follower.
Followers defer without consuming attempts. A dead or retired root is recovered through
the generation-admission fence before claims are released. Losing collectors release
claims they cannot dispatch. Concurrent roots can still split the ready set; batching is
opportunistic, not one-turn-per-conversation election.
`triggerDelivery.spec.ts` includes a real-Mongo collecting barrier, crash/lost-reply
injection, rolling-upgrade isolation, and an 8-conversation × 4-result storage stress
harness. Its latency measures receipt admission, not model turns or deployment latency.
Receipt batching defaults off. Deploy compatible receipt and manual-poll consumers on
**every replica**, then enable:
```yaml
endpoints:
agents:
backgroundTasks:
completionReceiptBatching: true
```
Older manual-poll workers cannot read v3 ownership. Queue capability fencing alone does
not protect that path. Disabling batching stops new v3 production, not existing
ownership. Do not downgrade poll consumers while v3 receipts remain, including delivered
receipts. A successor lease resumes interrupted cleanup before dispatch.
Every plan, receipt claim, and message claim carries a physical batch identity. Cleanup
matches that identity; its final plan deletion also matches a release identity and the
queue lease. A durable dispatch counter prevents a definite retry failure from releasing
an earlier ambiguous handoff. Native recovery fences distinguish unadmitted tombstones
from actual generation admission. Proven admissions settle followers, never re-present
results.
Retired owners do not expire while a batch still needs claim release or per-receipt
admission proof. Cleanup restores the ordinary retention deadline only after every
member is safe without the owner. Retirement also closes publication and dispatch.
A failed receipt write is repaired from its durable terminal message projection.
Manual polling marks receipt reconciliation before automatic delivery can settle that
handoff; speculative manual ownership only defers the automatic delivery.
Manual confirmation resolves a lost reply by reading the exact committed claim.
Failed confirmation rolls back only unreconciled ownership. If rollback is unavailable,
automatic delivery invokes manual-generation recovery after that owner is no longer active.
Its release CAS excludes committed manual handoffs, including confirmation after a stale
snapshot. Explicit manual recovery keeps its existing generation-fenced contract.
Manual reconciliation survives registry restoration and final response persistence.
Collecting plans acquire the root before siblings, so empty retirement can clear an
unowned plan without orphaning claims. Native `started` recovery fences confirm admission
even when a custom store cannot read historical claims after job cleanup.
Exact durable replays read the current claim without rewriting its handoff proof.
Applied followers finish interrupted owner proof copying before they settle, including
retired roots removed from the delivery queue.
Manual polls carry the observed physical batch ID through recovery. A released owner’s
late projection is cleared by that exact ID; successor epochs and manual claims remain owned.

View file

@ -11,6 +11,9 @@ export interface AgentTriggerDispatchContext {
/** Durable delivery attempt metadata, when dispatched by the queue engine. */
attempt?: number;
maxAttempts?: number;
/** Exact queue lease that owns this preparation. */
deliveryClaimToken?: string;
requiredWorkerCapability?: string;
signal?: AbortSignal;
}
@ -44,7 +47,13 @@ export interface AgentTriggerDispatchHandlers<FireResult, ContinueResult, SteerR
export function dispatchAgentTrigger<FireResult, ContinueResult, SteerResult>(
envelope: unknown,
handlers: AgentTriggerDispatchHandlers<FireResult, ContinueResult, SteerResult>,
options?: { signal?: AbortSignal; attempt?: number; maxAttempts?: number },
options?: {
signal?: AbortSignal;
attempt?: number;
maxAttempts?: number;
deliveryClaimToken?: string;
requiredWorkerCapability?: string;
},
): Promise<ContinueResult | FireResult | SteerResult> {
let normalized: AgentTriggerEnvelope;
try {
@ -56,6 +65,10 @@ export function dispatchAgentTrigger<FireResult, ContinueResult, SteerResult>(
idempotencyKey: getAgentTriggerIdempotencyKey(normalized),
...(options?.attempt != null && { attempt: options.attempt }),
...(options?.maxAttempts != null && { maxAttempts: options.maxAttempts }),
...(options?.deliveryClaimToken != null && { deliveryClaimToken: options.deliveryClaimToken }),
...(options?.requiredWorkerCapability != null && {
requiredWorkerCapability: options.requiredWorkerCapability,
}),
...(options?.signal != null && { signal: options.signal }),
};
if (normalized.mode === 'fire') {

View file

@ -74,7 +74,12 @@ describe('createAgentTriggerDeliveryEngine', () => {
expect(dispatch).toHaveBeenCalledWith(
{ version: 1 },
{ signal: expect.any(AbortSignal), attempt: 1, maxAttempts: 8 },
{
signal: expect.any(AbortSignal),
attempt: 1,
maxAttempts: 8,
deliveryClaimToken: 'claim-1',
},
);
expect(store.beginAttempt).toHaveBeenCalledWith({
id: 'delivery-row-1',

View file

@ -92,6 +92,7 @@ export interface AgentTriggerDeliveryRecord {
orderingKey: string;
laneSequence: number;
envelope: unknown;
requiredWorkerCapability?: string;
status: AgentTriggerDeliveryStatus;
attempts: number;
availableAt: Date;
@ -204,7 +205,13 @@ export interface AgentTriggerDeliveryEngineDeps {
store: AgentTriggerDeliveryStore;
dispatch: (
envelope: unknown,
options?: { signal?: AbortSignal; attempt?: number; maxAttempts?: number },
options?: {
signal?: AbortSignal;
attempt?: number;
maxAttempts?: number;
deliveryClaimToken?: string;
requiredWorkerCapability?: string;
},
) => Promise<AgentTriggerExecutionResult>;
/** Source-owned terminalization must commit before its delivery can become
* dead, including recovery after a crash that exhausted the attempt budget. */
@ -489,6 +496,10 @@ export function createAgentTriggerDeliveryEngine(
signal: controller.signal,
attempt,
maxAttempts,
deliveryClaimToken: delivery.claimToken,
...(delivery.requiredWorkerCapability != null && {
requiredWorkerCapability: delivery.requiredWorkerCapability,
}),
});
} catch (error) {
const attemptedAt = now();

View file

@ -70,6 +70,8 @@ export type AgentTriggerContinuePreparation =
releaseOnDefiniteFailure?: (error?: AgentTriggerExecutionError) => MaybePromise<void>;
/** Commits the source handoff after generation admission. Failure is
* outcome-ambiguous: the same delivery retries with the same request id. */
/** Durable possible-handoff marker written immediately before transport. */
beginDispatch?: () => MaybePromise<void>;
settleOnAdmission?: (result: AgentTriggerContinueResult) => MaybePromise<void>;
}
| { status: 'settled' };
@ -172,7 +174,13 @@ export interface AgentTriggerExecutionHostDeps {
export interface AgentTriggerExecutionHost {
dispatch: (
envelope: unknown,
options?: { signal?: AbortSignal; attempt?: number; maxAttempts?: number },
options?: {
signal?: AbortSignal;
attempt?: number;
maxAttempts?: number;
deliveryClaimToken?: string;
requiredWorkerCapability?: string;
},
) => Promise<AgentTriggerExecutionResult>;
}
@ -636,6 +644,7 @@ async function startRun(
const url = mode === 'fire' ? fireUrl(baseUrl) : continueUrl(baseUrl);
const fetcher: AgentTriggerFetch = deps.fetch ?? globalThis.fetch;
let response: Response;
await readyPreparation?.beginDispatch?.();
try {
response = await fetcher(url, {
method: 'POST',

View file

@ -1,5 +1,6 @@
import {
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
@ -185,6 +186,7 @@ describe('durable agent trigger service', () => {
expect.objectContaining({
workerCapabilities: [
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2,
@ -211,6 +213,14 @@ describe('durable agent trigger service', () => {
await service.stop();
});
it('keeps receipt batching disabled until the deployment gate is explicitly enabled', async () => {
const service = createAgentTriggerService({ mintToken: () => 'test' });
expect(service.getBackgroundCompletionReceiptBatching()).toBe(false);
await service.initialize({ address: '127.0.0.1', completionReceiptBatching: true });
expect(service.getBackgroundCompletionReceiptBatching()).toBe(true);
await service.stop();
});
describe('waiting completion deliveries', () => {
const address = { address: '127.0.0.1', family: 'IPv4' as const, port: 3080 };
const flush = () => new Promise((resolve) => setImmediate(resolve));
@ -407,6 +417,7 @@ describe('durable agent trigger service', () => {
expect.objectContaining({
workerCapabilities: [
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2,

View file

@ -1,5 +1,6 @@
import {
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
@ -50,6 +51,7 @@ const DEFAULT_PURGE_RECOVERY_LIMIT = 25;
export interface AgentTriggerServiceOptions {
completionResultBatchSize?: number;
completionReceiptBatching?: boolean;
address?: BoundAddress | string | null;
idlePolling?: {
queuedTurnMaxIntervalMs?: number;
@ -230,6 +232,7 @@ export interface AgentTriggerService {
input: Parameters<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>[0],
) => ReturnType<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>;
getBackgroundCompletionResultBatchSize: () => number;
getBackgroundCompletionReceiptBatching: () => boolean;
/** Longest a waiting completion delivery re-checks readiness. */
getCompletionWaitMaxIntervalMs: () => number;
/** Best effort: moves waiting completion deliveries forward after what they wait on changed. */
@ -271,6 +274,7 @@ function createDeliveryStore(
...input,
workerCapabilities: [
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2,
@ -352,6 +356,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
}
let boundOrigin: string | undefined;
let backgroundCompletionResultBatchSize = 8;
let backgroundCompletionReceiptBatching = false;
let completionWaitMaxIntervalMs = WAITING_RETRY_CAP_MS;
let deliveryEngine: AgentTriggerDeliveryEngine | undefined;
let initializePromise: Promise<void> | undefined;
@ -410,7 +415,13 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
const dispatchForActivePrincipal = async (
envelope: unknown,
options?: { signal?: AbortSignal; attempt?: number; maxAttempts?: number },
options?: {
signal?: AbortSignal;
attempt?: number;
maxAttempts?: number;
deliveryClaimToken?: string;
requiredWorkerCapability?: string;
},
): Promise<AgentTriggerExecutionResult> => {
const parsed = parseAgentTriggerEnvelope(envelope);
await requireActivePrincipal(parsed.principal.userId);
@ -619,6 +630,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
return {
initialize: (options = {}) => {
backgroundCompletionResultBatchSize = options.completionResultBatchSize ?? 8;
backgroundCompletionReceiptBatching = options.completionReceiptBatching === true;
completionWaitMaxIntervalMs =
options.idlePolling?.completionWaitMaxIntervalMs ?? WAITING_RETRY_CAP_MS;
boundOrigin = selfOriginFromAddress(options.address) ?? boundOrigin;
@ -809,6 +821,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
return getClaim == null ? null : getClaim(input);
}),
getBackgroundCompletionResultBatchSize: () => backgroundCompletionResultBatchSize,
getBackgroundCompletionReceiptBatching: () => backgroundCompletionReceiptBatching,
getCompletionWaitMaxIntervalMs: () => completionWaitMaxIntervalMs,
expediteCompletionWakeups: (input) => expediteCompletions(input),
releaseBackgroundToolResultClaims: (input) =>

View file

@ -270,7 +270,8 @@ function claimsMirrorExactly(left: TokenIdempotencyClaim, right: TokenIdempotenc
left.claimToken === right.claimToken &&
left.previousClaimToken === right.previousClaimToken &&
left.generationProtocolVersion === right.generationProtocolVersion &&
left.startedAt === right.startedAt
left.startedAt === right.startedAt &&
left.recoveryFence === right.recoveryFence
);
}
@ -3339,7 +3340,7 @@ class GenerationJobManagerClass {
}
const normalized = normalizeTokenClaim(claim, 'admission evidence');
assertClaimMatchesRequest(normalized, streamId, conversationId);
if (normalized.startedAt == null) {
if (normalized.startedAt == null || normalized.recoveryFence === true) {
return null;
}
return {
@ -3380,14 +3381,14 @@ class GenerationJobManagerClass {
return 'unavailable';
}
if (observed.existing.startedAt != null) {
return 'started';
return observed.existing.recoveryFence === true ? 'fenced' : 'started';
}
let owned = observed;
if (!observed.claimed) {
owned = await this.takeoverGeneration(userId, clientRequestId, streamId, observed.existing);
if (owned.existing?.startedAt != null) {
return 'started';
return owned.existing.recoveryFence === true ? 'fenced' : 'started';
}
if (!owned.claimed || owned.existing == null) {
return 'unavailable';
@ -3396,7 +3397,7 @@ class GenerationJobManagerClass {
const claim = normalizeTokenClaim(owned.existing, 'background completion recovery fence');
if (claim.startedAt != null) {
return 'started';
return claim.recoveryFence === true ? 'fenced' : 'started';
}
await this.tombstoneObservedGenerationClaim(
userId,
@ -3405,6 +3406,7 @@ class GenerationJobManagerClass {
claim,
Date.now(),
claim.generationProtocolVersion === 2 ? 2 : 1,
true,
);
return 'fenced';
}
@ -3477,11 +3479,13 @@ class GenerationJobManagerClass {
claim: TokenIdempotencyClaim,
createdAt: number,
generationProtocolVersion: 1 | 2,
recoveryFence?: true,
): Promise<TokenIdempotencyClaim> {
const tombstone: TokenIdempotencyClaim = {
...claim,
startedAt: createdAt,
generationProtocolVersion,
...(recoveryFence === true && { recoveryFence }),
};
const primaryKey = this.generationClaimKey(userId, clientRequestId, streamId);
let markError: unknown;

View file

@ -394,6 +394,13 @@ describe('GenerationJobManager start-generation claim', () => {
});
});
it('does not mistake a replayed recovery tombstone for actual admission proof', async () => {
const args = ['user-1', 'req-recovery-proof', 'stream-proof', 'stream-proof'] as const;
expect(await manager.fenceGenerationClaimForRecovery(...args)).toBe('fenced');
expect(await manager.fenceGenerationClaimForRecovery(...args)).toBe('fenced');
expect(await manager.getGenerationAdmissionEvidence(...args)).toBeNull();
});
it('invalidates an unpublished continuation creator before manual recovery', async () => {
const original = await manager.claimGeneration(
'user-1',

View file

@ -714,6 +714,8 @@ export interface IdempotencyClaimValue {
* already-started/cleaned generation and can never be taken over as an
* abandoned pre-create lease. */
startedAt?: number;
/** Recovery closed an unpublished claim without admitting a generation. */
recoveryFence?: true;
}
/** Result of an atomic {@link IJobStore.claimIdempotencyKey} attempt. */

View file

@ -1063,6 +1063,13 @@ describe('agent background completion batch config', () => {
endpoints: { agents: { backgroundTasks: {} } },
});
expect(defaults.endpoints?.agents?.backgroundTasks?.completionResultBatchSize).toBe(8);
expect(defaults.endpoints?.agents?.backgroundTasks?.completionReceiptBatching).toBe(false);
expect(
configSchema.parse({
version: '1.0',
endpoints: { agents: { backgroundTasks: { completionReceiptBatching: true } } },
}).endpoints?.agents?.backgroundTasks?.completionReceiptBatching,
).toBe(true);
for (const completionResultBatchSize of [0, 17, 1.5]) {
expect(
@ -1171,6 +1178,7 @@ describe('agent background task config', () => {
}
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
completionResultBatchSize: 8,
completionReceiptBatching: false,
completionWakeups: true,
completionResultMaxChars: 24 * 1024,
ordinaryToolCancellation: false,
@ -1190,6 +1198,7 @@ describe('agent background task config', () => {
}
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
completionResultBatchSize: 8,
completionReceiptBatching: false,
completionWakeups: false,
completionResultMaxChars: 24 * 1024,
ordinaryToolCancellation: false,
@ -1209,6 +1218,7 @@ describe('agent background task config', () => {
}
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
completionResultBatchSize: 8,
completionReceiptBatching: false,
completionWakeups: true,
completionResultMaxChars: 24 * 1024,
ordinaryToolCancellation: true,

View file

@ -1736,8 +1736,9 @@ export const agentsEndpointSchema = baseEndpointSchema
.max(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX)
.optional()
.default(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT),
/** Maximum message-backed sibling results in one continuation.
* Independent receipts retain task-local delivery ownership. */
/** Enable only after every replica has compatible receipt/poll consumers. */
completionReceiptBatching: z.boolean().optional().default(false),
/** Maximum compatible sibling results in one continuation. */
completionResultBatchSize: z.number().int().min(1).max(16).optional().default(8),
/** Cooperative cancellation for process-local ordinary tools. Off
* by default so existing deployments opt into the new control. */

View file

@ -76,6 +76,7 @@ export {
export {
AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,

View file

@ -347,6 +347,7 @@ export function createMethods(
const agentQueuedTurnMethods = createAgentQueuedTurnMethods(mongoose);
const agentTriggerDeliveryMethods = createAgentTriggerDeliveryMethods(mongoose, {
releaseBatchProjections: messageMethods.releaseBackgroundToolResultClaims,
purgeQueuedTurnsForUser: (user) =>
agentQueuedTurnMethods.deleteAllAgentQueuedTurnsForUser({
user: typeof user === 'string' ? new mongoose.Types.ObjectId(user) : user,

View file

@ -5991,3 +5991,94 @@ describe('Message Operations', () => {
});
});
});
describe('receipt-backed claim release before projection', () => {
it('allows a missing message only when the receipt owner explicitly opts in', async () => {
const input = {
userId: new mongoose.Types.ObjectId().toString(),
conversationId: 'missing-conversation',
messageId: 'missing-parent',
kind: 'wakeup' as const,
claimId: 'batch-root',
};
await expect(releaseBackgroundToolResultClaims(input)).resolves.toBe(false);
await expect(
releaseBackgroundToolResultClaims({ ...input, allowMissingMessage: true }),
).resolves.toBe(true);
});
});
describe('receipt batch projection epochs', () => {
it('does not let delayed cleanup remove a successor claim on the same logical request', async () => {
await Message.create({
messageId: 'epoch-parent',
conversationId: 'epoch-conversation',
user: 'epoch-owner',
isCreatedByUser: false,
unfinished: false,
content: [
{
type: 'tool_call',
tool_call: {
id: 'epoch-call',
output: 'done',
backgroundTask: {
taskId: 'epoch-task',
toolName: 'tool',
status: 'completed',
completionWakeup: true,
completionReceipt: true,
},
},
},
],
});
const input = {
userId: 'epoch-owner',
conversationId: 'epoch-conversation',
messageId: 'epoch-parent',
taskId: 'epoch-task',
kind: 'wakeup' as const,
claimId: 'logical-root',
limit: 1,
};
expect((await claimBackgroundToolResults({ ...input, batchId: 'old' })).status).toBe(
'acquired',
);
let pause: () => void = () => undefined;
let resume: () => void = () => undefined;
const paused = new Promise<void>((resolve) => {
pause = resolve;
});
const barrier = new Promise<void>((resolve) => {
resume = resolve;
});
const update = Message.findOneAndUpdate.bind(Message);
const oldCleanup = jest.spyOn(Message, 'findOneAndUpdate').mockImplementationOnce((...args) => {
const query = update(...args);
const execute = query.exec.bind(query);
jest.spyOn(query, 'exec').mockImplementationOnce(async () => {
pause();
await barrier;
return execute();
});
return query;
});
const predecessor = releaseBackgroundToolResultClaims({ ...input, batchId: 'old' });
await paused;
expect(await releaseBackgroundToolResultClaims({ ...input, batchId: 'old' })).toBe(true);
expect((await claimBackgroundToolResults({ ...input, batchId: 'new' })).status).toBe(
'acquired',
);
resume();
expect(await predecessor).toBe(true);
oldCleanup.mockRestore();
expect((await claimBackgroundToolResults({ ...input, batchId: 'new' })).status).toBe(
'acquired',
);
expect(await claimBackgroundToolResults({ ...input, batchId: 'old' })).toMatchObject({
status: 'claimed',
claim: { batchId: 'new' },
});
});
});

View file

@ -567,7 +567,13 @@ export type BackgroundToolResultClaim =
| { status: 'outcome_unknown'; toolName: string }
| {
status: 'claimed';
claim?: { kind: 'manual' | 'wakeup'; claimId: string; generationId?: string };
claim?: {
kind: 'manual' | 'wakeup';
claimId: string;
generationId?: string;
batchId?: string;
receiptReconciled?: true;
};
messageId?: string;
}
| { status: 'acquired'; results: BackgroundToolResultRecord[]; messageId?: string };
@ -846,6 +852,7 @@ export interface MessageMethods {
claimId: string;
claimedAt: Date;
generationId?: string;
receiptReconciled?: true;
};
};
}): Promise<{ matched: boolean; unfinished: boolean }>;
@ -858,6 +865,8 @@ export interface MessageMethods {
agentId?: string;
kind: 'manual' | 'wakeup';
claimId: string;
/** Physical receipt-batch identity, independent of its logical request. */
batchId?: string;
/** Response generation that owns this manual result delivery. */
generationId?: string;
/** Manual owner-process takeover after automatic delivery was retired. */
@ -866,12 +875,24 @@ export interface MessageMethods {
/** Includes JSON escaping, delimiters, and empty result fields. */
maxMetadataChars?: number;
}): Promise<BackgroundToolResultClaim>;
confirmBackgroundToolResultClaim(params: {
userId: string;
conversationId: string;
messageId: string;
taskId: string;
claimId: string;
}): Promise<boolean>;
releaseBackgroundToolResultClaims(params: {
userId: string;
conversationId: string;
messageId: string;
/** Omit to release every sibling owned by this exact batch claim. */
taskIds?: string[];
/** Receipt-backed results can precede the message projection. */
allowMissingMessage?: true;
/** Manual rollback must not erase a committed receipt handoff. */
onlyIfUnreconciled?: true;
batchId?: string;
kind: 'manual' | 'wakeup';
claimId: string;
}): Promise<boolean>;
@ -1983,7 +2004,15 @@ export function createMessageMethods(
function readBackgroundToolResultClaim(
row: Pick<IMessage, 'content'>,
taskId: string,
): { kind: 'manual' | 'wakeup'; claimId: string; generationId?: string } | undefined {
):
| {
kind: 'manual' | 'wakeup';
claimId: string;
generationId?: string;
batchId?: string;
receiptReconciled?: true;
}
| undefined {
for (const part of row.content ?? []) {
if (part == null || typeof part !== 'object' || Array.isArray(part)) {
continue;
@ -1993,7 +2022,13 @@ export function createMessageMethods(
tool_call?: {
backgroundTask?: {
taskId?: unknown;
resultClaim?: { kind?: unknown; claimId?: unknown; generationId?: unknown };
resultClaim?: {
kind?: unknown;
claimId?: unknown;
generationId?: unknown;
batchId?: unknown;
receiptReconciled?: unknown;
};
};
};
}
@ -2010,6 +2045,8 @@ export function createMessageMethods(
return {
kind: claim.kind,
claimId: claim.claimId,
...(typeof claim.batchId === 'string' && { batchId: claim.batchId }),
...(claim.receiptReconciled === true && { receiptReconciled: true }),
...(typeof claim.generationId === 'string' && claim.generationId.length > 0
? { generationId: claim.generationId }
: {}),
@ -2021,7 +2058,7 @@ export function createMessageMethods(
function parseBackgroundToolResults(
message: IMessage,
claim: { kind: 'manual' | 'wakeup'; claimId: string },
claim: { kind: 'manual' | 'wakeup'; claimId: string; batchId?: string },
): BackgroundToolResultRecord[] {
const results: BackgroundToolResultRecord[] = [];
for (const part of message.content ?? []) {
@ -2040,7 +2077,7 @@ export function createMessageMethods(
status?: unknown;
cancelled?: unknown;
settledAt?: unknown;
resultClaim?: { kind?: unknown; claimId?: unknown };
resultClaim?: { kind?: unknown; claimId?: unknown; batchId?: unknown };
};
};
};
@ -2052,7 +2089,8 @@ export function createMessageMethods(
typeof task.toolName !== 'string' ||
(task.status !== 'completed' && task.status !== 'error') ||
task.resultClaim?.kind !== claim.kind ||
task.resultClaim.claimId !== claim.claimId
task.resultClaim.claimId !== claim.claimId ||
task.resultClaim.batchId !== claim.batchId
) {
continue;
}
@ -2133,6 +2171,7 @@ export function createMessageMethods(
kind,
claimId,
generationId,
batchId,
allowUnfinished = false,
limit = kind === 'wakeup' ? 8 : 1,
maxMetadataChars,
@ -2144,6 +2183,7 @@ export function createMessageMethods(
if (
(requestedMessageId != null &&
(requestedMessageId.length === 0 || requestedMessageId.length > 256)) ||
(batchId != null && (batchId.length === 0 || batchId.length > 128 || kind !== 'wakeup')) ||
taskId.length === 0 ||
taskId.length > 256 ||
claimId.length === 0 ||
@ -2214,7 +2254,10 @@ export function createMessageMethods(
}
const recoveredSource = requestedMessageId == null ? { messageId: resolvedMessageId } : {};
const requestedClaim = readBackgroundToolResultClaim(row, taskId);
const replaying = requestedClaim?.kind === kind && requestedClaim.claimId === claimId;
const replaying =
requestedClaim?.kind === kind &&
requestedClaim.claimId === claimId &&
requestedClaim.batchId === batchId;
const candidates: string[] = [];
const costs = new Map<string, number>();
let receiptBacked = false;
@ -2233,8 +2276,13 @@ export function createMessageMethods(
}
const terminal = task?.status === 'completed' || task?.status === 'error';
const wakeupEligible = kind !== 'wakeup' || task?.completionWakeup === true;
const resultClaim = task?.resultClaim as { kind?: unknown; claimId?: unknown } | undefined;
const replay = resultClaim?.kind === kind && resultClaim.claimId === claimId;
const resultClaim = task?.resultClaim as
| { kind?: unknown; claimId?: unknown; batchId?: unknown }
| undefined;
const replay =
resultClaim?.kind === kind &&
resultClaim.claimId === claimId &&
resultClaim.batchId === batchId;
const sameAgent =
agentId == null ||
partAgentId == null ||
@ -2315,105 +2363,103 @@ export function createMessageMethods(
kind,
claimId,
claimedAt,
...(batchId != null && { batchId }),
...(kind === 'manual' && requestedGenerationId != null
? { generationId: requestedGenerationId }
: {}),
};
const updated = await Message.findOneAndUpdate(
{
user: userId,
conversationId,
messageId: resolvedMessageId,
...(kind === 'manual' && allowUnfinished ? {} : { unfinished: { $ne: true } }),
content: {
$elemMatch: {
type: 'tool_call',
'tool_call.backgroundTask.taskId': taskId,
'tool_call.backgroundTask.status': { $in: ['completed', 'error'] },
...(kind === 'wakeup' ? { 'tool_call.backgroundTask.completionWakeup': true } : {}),
...(replaying
? {
'tool_call.backgroundTask.resultClaim.kind': kind,
'tool_call.backgroundTask.resultClaim.claimId': claimId,
}
: /** Missing OR stored null: the in-memory claimable scan, the
* claim arrayFilters, and the settle stamp all treat a null
* claim as unclaimed, and the subfield-preserving settle write
* keeps a persisted null a whole-object rewrite used to drop.
* `$exists: false` here would strand such a part as terminal
* but permanently unclaimable. */
{ 'tool_call.backgroundTask.resultClaim': null }),
},
const claimFilter: FilterQuery<IMessage> = {
user: userId,
conversationId,
messageId: resolvedMessageId,
...(kind === 'manual' && allowUnfinished ? {} : { unfinished: { $ne: true } }),
content: {
$elemMatch: {
type: 'tool_call',
'tool_call.backgroundTask.taskId': taskId,
'tool_call.backgroundTask.status': { $in: ['completed', 'error'] },
...(kind === 'wakeup' ? { 'tool_call.backgroundTask.completionWakeup': true } : {}),
...(replaying
? {
'tool_call.backgroundTask.resultClaim.kind': kind,
'tool_call.backgroundTask.resultClaim.claimId': claimId,
'tool_call.backgroundTask.resultClaim.batchId': batchId ?? null,
}
: /** Missing OR stored null: the in-memory claimable scan, the
* claim arrayFilters, and the settle stamp all treat a null
* claim as unclaimed, and the subfield-preserving settle write
* keeps a persisted null a whole-object rewrite used to drop.
* `$exists: false` here would strand such a part as terminal
* but permanently unclaimable. */
{ 'tool_call.backgroundTask.resultClaim': null }),
},
...(agentId != null
? {
$expr: {
$anyElementTrue: {
$map: {
input: { $ifNull: ['$content', []] },
as: 'candidate',
in: {
$and: [
{ $eq: ['$$candidate.tool_call.backgroundTask.taskId', taskId] },
{
$in: [
{
$ifNull: [
{
$ifNull: ['$$candidate.agentId', '$$candidate.tool_call.agentId'],
},
null,
],
},
[null, agentId],
],
},
],
},
},
...(agentId != null
? {
$expr: {
$anyElementTrue: {
$map: {
input: { $ifNull: ['$content', []] },
as: 'candidate',
in: {
$and: [
{ $eq: ['$$candidate.tool_call.backgroundTask.taskId', taskId] },
{
$in: [
{
$ifNull: [
{
$ifNull: ['$$candidate.agentId', '$$candidate.tool_call.agentId'],
},
null,
],
},
[null, agentId],
],
},
],
},
},
},
}
: {}),
},
/** Stamps the claim onto every part this pass admitted. The filtered
* positional operator selects those parts by predicate, so the write
* touches only them instead of re-emitting the whole content array, and
* needs no read-modify-write. Amazon DocumentDB rejects the
* aggregation-pipeline form this replaces. */
{ $set: { 'content.$[part].tool_call.backgroundTask.resultClaim': claimStamp } },
{
new: true,
projection: { content: 1 },
arrayFilters: [
},
}
: {}),
};
const updated = replaying
? await Message.findOne(claimFilter).select({ content: 1 }).lean<IMessage | null>()
: await Message.findOneAndUpdate(
claimFilter,
/** Stamps the claim onto every part this pass admitted. The filtered
* positional operator selects those parts by predicate, so the write
* touches only them instead of re-emitting the whole content array, and
* needs no read-modify-write. Amazon DocumentDB rejects the
* aggregation-pipeline form this replaces. */
{ $set: { 'content.$[part].tool_call.backgroundTask.resultClaim': claimStamp } },
{
'part.type': 'tool_call',
'part.tool_call.backgroundTask.taskId': { $in: candidates },
'part.tool_call.backgroundTask.status': { $in: ['completed', 'error'] },
...(kind === 'wakeup'
? { 'part.tool_call.backgroundTask.completionWakeup': true }
: {}),
$and: [
new: true,
projection: { content: 1 },
arrayFilters: [
{
/** Unclaimed, or already held by this exact claimant (replay). */
$or: [
{ 'part.tool_call.backgroundTask.resultClaim': null },
'part.type': 'tool_call',
'part.tool_call.backgroundTask.taskId': { $in: candidates },
'part.tool_call.backgroundTask.status': { $in: ['completed', 'error'] },
...(kind === 'wakeup'
? { 'part.tool_call.backgroundTask.completionWakeup': true }
: {}),
$and: [
{
'part.tool_call.backgroundTask.resultClaim.kind': kind,
'part.tool_call.backgroundTask.resultClaim.claimId': claimId,
'part.tool_call.backgroundTask.resultClaim': null,
},
...(agentId == null ? [] : [agentOwnershipFilter('part.', agentId)]),
],
},
...(agentId == null ? [] : [agentOwnershipFilter('part.', agentId)]),
],
},
],
},
).lean<IMessage | null>();
).lean<IMessage | null>();
if (updated == null) {
return { status: 'not_ready' };
}
const results = parseBackgroundToolResults(updated, { kind, claimId });
const results = parseBackgroundToolResults(updated, { kind, claimId, batchId });
const competingClaim = readBackgroundToolResultClaim(updated, taskId);
return results.some((result) => result.taskId === taskId)
? { status: 'acquired', results, ...recoveredSource }
@ -2424,21 +2470,82 @@ export function createMessageMethods(
};
}
/** Manual ownership is committed only after receipt arbitration completes. */
async function confirmBackgroundToolResultClaim(
input: Parameters<MessageMethods['confirmBackgroundToolResultClaim']>[0],
): Promise<boolean> {
const Message = mongoose.models.Message as Model<IMessage>;
const identity = {
'tool_call.backgroundTask.taskId': input.taskId,
'tool_call.backgroundTask.resultClaim.kind': 'manual',
'tool_call.backgroundTask.resultClaim.claimId': input.claimId,
};
try {
const confirmed = await Message.updateOne(
{
user: input.userId,
conversationId: input.conversationId,
messageId: input.messageId,
content: { $elemMatch: identity },
},
{
$set: { 'content.$[part].tool_call.backgroundTask.resultClaim.receiptReconciled': true },
},
{
arrayFilters: [
{
'part.tool_call.backgroundTask.taskId': input.taskId,
'part.tool_call.backgroundTask.resultClaim.kind': 'manual',
'part.tool_call.backgroundTask.resultClaim.claimId': input.claimId,
},
],
},
);
return confirmed.matchedCount === 1;
} catch (error) {
// A lost write reply is not an uncommitted handoff. Read only the exact
// manual claim, never manufacture success from another claimant.
const committed = await Message.exists({
user: input.userId,
conversationId: input.conversationId,
messageId: input.messageId,
content: {
$elemMatch: {
...identity,
'tool_call.backgroundTask.resultClaim.receiptReconciled': true,
},
},
});
if (committed != null) return true;
throw error;
}
}
async function releaseBackgroundToolResultClaims({
userId,
conversationId,
messageId,
taskIds,
allowMissingMessage,
onlyIfUnreconciled,
kind,
claimId,
batchId,
}: {
userId: string;
conversationId: string;
messageId: string;
taskIds?: string[];
/** Receipt-backed results can precede the message projection. */
allowMissingMessage?: true;
/** Manual rollback must not erase a committed receipt handoff. */
onlyIfUnreconciled?: true;
batchId?: string;
kind: 'manual' | 'wakeup';
claimId: string;
}): Promise<boolean> {
if (onlyIfUnreconciled === true && kind !== 'manual')
throw new TypeError('Unreconciled rollback requires a manual claim');
if (taskIds?.length === 0) {
return true;
}
@ -2460,6 +2567,10 @@ export function createMessageMethods(
{
'part.tool_call.backgroundTask.resultClaim.kind': kind,
'part.tool_call.backgroundTask.resultClaim.claimId': claimId,
'part.tool_call.backgroundTask.resultClaim.batchId': batchId ?? null,
...(onlyIfUnreconciled === true && {
'part.tool_call.backgroundTask.resultClaim.receiptReconciled': { $ne: true },
}),
...(taskIds == null
? {}
: { 'part.tool_call.backgroundTask.taskId': { $in: taskIds } }),
@ -2474,9 +2585,13 @@ export function createMessageMethods(
messageId,
content: { $not: { $type: 'array' } },
});
return arraylessRow != null;
if (arraylessRow != null) return true;
return (
allowMissingMessage === true &&
(await Message.exists({ user: userId, conversationId, messageId })) == null
);
}
const remaining = parseBackgroundToolResults(updated, { kind, claimId });
const remaining = parseBackgroundToolResults(updated, { kind, claimId, batchId });
return taskIds == null
? remaining.length === 0
: !remaining.some((result) => taskIds.includes(result.taskId));
@ -4249,6 +4364,7 @@ export function createMessageMethods(
updateMessageText,
updateToolCallResult,
claimBackgroundToolResults,
confirmBackgroundToolResultClaim,
releaseBackgroundToolResultClaims,
updateMessage,
recordSubagentTaskControlReceipt,

File diff suppressed because it is too large Load diff

View file

@ -1,4 +1,5 @@
import { createHash } from 'node:crypto';
import { createHash, randomUUID } from 'node:crypto';
import { backgroundResultMetadata } from 'librechat-data-provider';
import type { FilterQuery, Model, Types } from 'mongoose';
import type {
AgentBackgroundToolResultReceipt,
@ -17,8 +18,10 @@ import type {
IAgentTriggerUserPurge,
IAgentTriggerUserPurgeDocument,
} from '~/types/triggerDelivery';
import type { MessageMethods } from './message';
import {
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
@ -38,6 +41,10 @@ const DEFAULT_PURGE_RECOVERY_LIMIT = 25;
const MAX_PURGE_RECOVERY_LIMIT = 200;
const HISTORY_LIMIT = 64;
const MAX_BATCH_SIZE = 8;
const BACKGROUND_RECEIPT_CAPABILITIES = [
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
];
const MAX_BATCH_BYTES = 512 * 1024;
/** Bounds the claim's compare-and-swap retries. Each lost round means another
* worker claimed the row this one read, so the queue is making progress. */
@ -167,6 +174,48 @@ export type AgentBackgroundToolResultClaim =
}>;
};
export interface AgentBackgroundToolResultBatchInput {
deliveryKey: string;
sourceId: string;
userId: string;
tenantId?: string;
conversationId: string;
parentMessageId: string;
agentId: string;
limit: number;
maxMetadataChars: number;
deliveryClaimToken?: string;
}
export type AgentBackgroundToolResultBatchClaim =
| { status: 'legacy' }
| { status: 'not_ready'; waitingForResult?: true }
| {
status: 'claimed';
claimId: string;
batchId: string;
ownerStatus: 'pending' | 'applied' | 'recoverable';
}
| (Extract<AgentBackgroundToolResultClaim, { status: 'acquired' }> & { batchId: string });
export type AgentBackgroundToolResultBatchOwnerInput = Omit<
AgentBackgroundToolResultBatchInput,
'limit' | 'maxMetadataChars' | 'agentId'
> & { agentId?: string };
export type AgentBackgroundToolResultBatchReleaseInput = {
sourceId: string;
userId: string;
conversationId: string;
parentMessageId: string;
claimId: string;
deliveryClaimToken?: string;
batchId?: string;
dispatchId?: string;
/** Retirement and generation-claim CAS proved that no admission can occur. */
recoveryFenced?: true;
};
export interface AgentTriggerDeliveryFence {
id: string;
workerId: string;
@ -414,13 +463,21 @@ export interface AgentTriggerDeliveryMethods {
claimId: string;
limit?: number;
}) => Promise<AgentBackgroundToolResultClaim>;
releaseAgentBackgroundToolResultClaims: (input: {
sourceId: string;
userId: string;
conversationId: string;
parentMessageId: string;
claimId: string;
}) => Promise<boolean>;
releaseAgentBackgroundToolResultClaims: (
input: AgentBackgroundToolResultBatchReleaseInput,
) => Promise<boolean>;
claimAgentBackgroundToolResultBatch: (
input: AgentBackgroundToolResultBatchInput,
) => Promise<AgentBackgroundToolResultBatchClaim>;
getAgentBackgroundToolResultBatch: (
input: AgentBackgroundToolResultBatchOwnerInput,
) => Promise<IAgentTriggerDelivery['backgroundToolResultBatch'] | null>;
beginAgentBackgroundToolResultBatchDispatch: (
input: AgentBackgroundToolResultBatchOwnerInput & { batchId: string; dispatchId: string },
) => Promise<boolean>;
confirmAgentBackgroundToolResultBatch: (
input: AgentBackgroundToolResultBatchOwnerInput & { batchId: string },
) => Promise<boolean>;
settleAgentTriggerHandlingOutcome: (
input: SettleAgentTriggerHandlingOutcomeInput,
) => Promise<boolean>;
@ -628,6 +685,7 @@ function requireClaim(delivery: IAgentTriggerDelivery | null): AgentTriggerDeliv
export function createAgentTriggerDeliveryMethods(
mongoose: typeof import('mongoose'),
deps: {
releaseBatchProjections?: MessageMethods['releaseBackgroundToolResultClaims'];
purgeQueuedTurnsForUser?: (user: string | Types.ObjectId) => Promise<unknown>;
} = {},
): AgentTriggerDeliveryMethods {
@ -1125,6 +1183,8 @@ export function createAgentTriggerDeliveryMethods(
input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 &&
input.requiredWorkerCapability !==
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 &&
input.requiredWorkerCapability !==
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3 &&
input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 &&
input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1 &&
input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2
@ -2182,38 +2242,70 @@ export function createAgentTriggerDeliveryMethods(
},
};
})();
const retired = await Delivery()
.findOneAndUpdate(
{
deliveryKey: input.deliveryKey,
'envelope.event.source.type': 'internal',
'envelope.event.source.id': input.sourceId,
...statusFence,
},
{
$set: {
status: 'succeeded',
result,
settledAt: input.settledAt,
expiresAt: new Date(input.settledAt.getTime() + SUCCESS_RETENTION_MS),
laneCleanupPendingAt: input.settledAt,
const retire = (retainBatch: boolean, discardEmptyPlan = false) =>
Delivery()
.findOneAndUpdate(
{
deliveryKey: input.deliveryKey,
'envelope.event.source.type': 'internal',
'envelope.event.source.id': input.sourceId,
...statusFence,
...(discardEmptyPlan && {
backgroundToolResultBatch: { $exists: true },
'backgroundToolResultBatch.members.0': { $exists: false },
'backgroundToolResult.resultClaim.claimId': { $ne: input.deliveryKey },
}),
...(!discardEmptyPlan &&
retainBatch && {
backgroundToolResultBatch: { $exists: true },
'backgroundToolResultBatch.proofCopiedAt': { $exists: false },
}),
...(!discardEmptyPlan &&
!retainBatch && {
$and: [
{
$or: [
{ backgroundToolResultBatch: { $exists: false } },
{ 'backgroundToolResultBatch.proofCopiedAt': { $exists: true } },
],
},
],
}),
...(input.onlyIfUnclaimed === true
? { 'backgroundToolResult.resultClaim': { $exists: false } }
: {}),
},
$unset: {
leaseBy: 1,
leaseUntil: 1,
claimToken: 1,
capabilityStatus: 1,
claimAvailableAt: 1,
capabilityLeaseBy: 1,
capabilityLeaseUntil: 1,
capabilityClaimToken: 1,
lastError: 1,
{
$set: {
status: 'succeeded',
result,
settledAt: input.settledAt,
...(!retainBatch && {
expiresAt: new Date(input.settledAt.getTime() + SUCCESS_RETENTION_MS),
}),
laneCleanupPendingAt: input.settledAt,
},
$unset: {
leaseBy: 1,
leaseUntil: 1,
claimToken: 1,
capabilityStatus: 1,
claimAvailableAt: 1,
capabilityLeaseBy: 1,
capabilityLeaseUntil: 1,
capabilityClaimToken: 1,
lastError: 1,
...(retainBatch && { expiresAt: 1 }),
...(discardEmptyPlan && { backgroundToolResultBatch: 1 }),
},
},
},
{ new: true },
)
.select('_id orderingKey laneCleanupPendingAt')
.lean<Pick<IAgentTriggerDelivery, '_id' | 'orderingKey' | 'laneCleanupPendingAt'>>();
{ new: true },
)
.select('_id orderingKey laneCleanupPendingAt')
.lean<Pick<IAgentTriggerDelivery, '_id' | 'orderingKey' | 'laneCleanupPendingAt'>>();
// A retired owner is its followers' recovery and frozen-membership record.
// Its TTL cannot start while release or receipt-proof copying is incomplete.
const retired = (await retire(false)) ?? (await retire(false, true)) ?? (await retire(true));
if (retired?._id != null) {
try {
await fulfillLaneCleanupRequest(retired);
@ -2268,7 +2360,7 @@ export function createAgentTriggerDeliveryMethods(
requiredWorkerCapability: {
$in: [
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
...BACKGROUND_RECEIPT_CAPABILITIES,
],
},
'envelope.event.source.type': 'internal',
@ -2312,7 +2404,7 @@ export function createAgentTriggerDeliveryMethods(
requiredWorkerCapability: {
$in: [
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
...BACKGROUND_RECEIPT_CAPABILITIES,
],
},
'envelope.event.source.type': 'internal',
@ -2366,7 +2458,7 @@ export function createAgentTriggerDeliveryMethods(
'envelope.target.conversationId': input.conversationId,
...(input.taskId != null && { 'envelope.event.payload.taskId': input.taskId }),
/** Legacy rows keep results only on the parent message. */
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },
};
const read = (filter: FilterQuery<IAgentTriggerDelivery>) =>
Delivery()
@ -2568,7 +2660,7 @@ export function createAgentTriggerDeliveryMethods(
}
const identity = {
deliveryKey: input.deliveryKey,
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },
'envelope.event.source.type': 'internal',
'envelope.event.source.id': input.sourceId,
backgroundToolResultErasedAt: { $exists: false },
@ -2612,7 +2704,7 @@ export function createAgentTriggerDeliveryMethods(
const delivery = await Delivery()
.findOne({
deliveryKey: input.deliveryKey,
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },
'envelope.event.source.type': 'internal',
'envelope.event.source.id': input.sourceId,
})
@ -2648,7 +2740,7 @@ export function createAgentTriggerDeliveryMethods(
}) {
return {
user: input.userId,
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },
'envelope.event.source.type': 'internal',
'envelope.event.source.id': input.sourceId,
'envelope.target.conversationId': input.conversationId,
@ -2769,38 +2861,592 @@ export function createAgentTriggerDeliveryMethods(
return result == null ? { status: 'not_ready' } : { status: 'acquired', results: [result] };
}
async function releaseAgentBackgroundToolResultClaims(input: {
sourceId: string;
userId: string;
conversationId: string;
parentMessageId: string;
claimId: string;
}): Promise<boolean> {
async function releaseAgentBackgroundToolResultClaims(
input: AgentBackgroundToolResultBatchReleaseInput,
): Promise<boolean> {
if (
input.sourceId.length === 0 ||
input.sourceId.length > 256 ||
input.userId.length === 0 ||
input.conversationId.length === 0 ||
input.conversationId.length > 256 ||
input.parentMessageId.length === 0 ||
input.parentMessageId.length > 256 ||
input.claimId.length === 0 ||
[
input.sourceId,
input.userId,
input.conversationId,
input.parentMessageId,
input.claimId,
].some((value) => value.length === 0 || value.length > 256) ||
input.claimId.length > 128
) {
throw new TypeError('Invalid background tool result receipt claim release');
}
const scope = backgroundResultIdentity(input);
const releaseId = randomUUID();
const releasing = await Delivery()
.findOneAndUpdate(
{
...scope,
deliveryKey: input.claimId,
backgroundToolResultBatch: { $exists: true },
'backgroundToolResultBatch.appliedAt': { $exists: false },
...(input.batchId == null ? {} : { 'backgroundToolResultBatch.batchId': input.batchId }),
...(input.deliveryClaimToken == null
? {}
: { capabilityClaimToken: input.deliveryClaimToken }),
...(input.recoveryFenced === true
? {}
: {
$or: [
{ 'backgroundToolResultBatch.releasing': true },
{ 'backgroundToolResultBatch.dispatchCount': 0 },
...(input.dispatchId == null
? []
: [
{
'backgroundToolResultBatch.dispatchCount': 1,
'backgroundToolResultBatch.dispatchId': input.dispatchId,
},
]),
],
}),
},
{
$set: {
'backgroundToolResultBatch.releasing': true,
'backgroundToolResultBatch.releaseId': releaseId,
},
},
{ new: true, timestamps: false },
)
.select('status +backgroundToolResultBatch')
.lean<Pick<IAgentTriggerDelivery, 'status' | 'backgroundToolResultBatch'>>();
if (
releasing == null &&
(await Delivery().exists({
...scope,
deliveryKey: input.claimId,
backgroundToolResultBatch: { $exists: true },
})) != null
)
return false;
const batch = releasing?.backgroundToolResultBatch;
const batchId = batch?.batchId ?? input.batchId;
const receiptScope = {
...scope,
'backgroundToolResult.resultClaim.claimId': input.claimId,
'backgroundToolResult.resultClaim.batchId': batchId ?? null,
};
await Delivery().updateMany(
{ ...scope, 'backgroundToolResult.resultClaim.claimId': input.claimId },
{ ...receiptScope, 'backgroundToolResult.resultClaim.appliedAt': { $exists: false } },
{ $unset: { 'backgroundToolResult.resultClaim': 1 } },
{ timestamps: false },
);
return (
if ((await Delivery().exists(receiptScope)) != null) return false;
if (batch != null) {
if (deps.releaseBatchProjections == null)
throw new Error('Background batch projection cleanup is unavailable');
if (
!(await deps.releaseBatchProjections({
userId: input.userId,
conversationId: input.conversationId,
messageId: input.parentMessageId,
kind: 'wakeup',
claimId: input.claimId,
batchId: batch.batchId,
allowMissingMessage: true,
}))
)
return false;
const clear = (retired: boolean) =>
Delivery().updateOne(
{
...scope,
deliveryKey: input.claimId,
'backgroundToolResultBatch.batchId': batch.batchId,
'backgroundToolResultBatch.releaseId': releaseId,
'backgroundToolResultBatch.appliedAt': { $exists: false },
status: retired ? 'succeeded' : { $ne: 'succeeded' },
...(input.deliveryClaimToken == null
? {}
: { capabilityClaimToken: input.deliveryClaimToken }),
},
{
$unset: { backgroundToolResultBatch: 1 },
...(retired && { $max: { expiresAt: new Date(Date.now() + SUCCESS_RETENTION_MS) } }),
},
{ timestamps: false },
);
const cleared = await clear(releasing?.status === 'succeeded');
if (cleared.matchedCount === 1) return true;
return releasing?.status !== 'succeeded' && (await clear(true)).matchedCount === 1;
}
return true;
}
function backgroundBatchScope(
input:
| AgentBackgroundToolResultBatchInput
| Omit<AgentBackgroundToolResultBatchInput, 'limit' | 'maxMetadataChars'>,
) {
return {
...backgroundResultIdentity(input),
tenantId: input.tenantId ?? null,
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
'envelope.event.type': 'background-tool.completion',
};
}
/** Each result has its own CAS. Candidates survive a collecting crash; only
* confirmed owners enter the frozen member list used by every retry. */
async function claimAgentBackgroundToolResultBatch(
input: AgentBackgroundToolResultBatchInput,
): Promise<AgentBackgroundToolResultBatchClaim> {
if (
[
input.deliveryKey,
input.sourceId,
input.userId,
input.conversationId,
input.parentMessageId,
input.agentId,
].some((value) => value.length === 0 || value.length > 256) ||
input.deliveryKey.length > 128
) {
throw new TypeError('Invalid background result batch identity');
}
if (
!Number.isSafeInteger(input.limit) ||
input.limit < 1 ||
!Number.isSafeInteger(input.maxMetadataChars) ||
input.maxMetadataChars < 1
) {
throw new TypeError('Invalid background result batch bounds');
}
const scope = backgroundBatchScope(input);
const projection =
'deliveryKey requiredWorkerCapability status capabilityStatus result +backgroundToolResult +backgroundToolResultBatch';
type Row = Pick<
IAgentTriggerDelivery,
| 'deliveryKey'
| 'requiredWorkerCapability'
| 'status'
| 'capabilityStatus'
| 'result'
| 'backgroundToolResult'
| 'backgroundToolResultBatch'
> & { envelope?: unknown };
const readRoot = () =>
Delivery()
.findOne({ ...scope, deliveryKey: input.deliveryKey })
.select(projection + ' envelope')
.lean<Row>();
let root = await readRoot();
if (root == null) {
const gated = await Delivery().exists({
user: input.userId,
deliveryKey: input.deliveryKey,
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
});
return { status: gated == null ? 'legacy' : 'not_ready' };
}
if (root.backgroundToolResultBatch?.appliedAt != null) {
if (
root.backgroundToolResultBatch.proofCopiedAt == null &&
!(await confirmAgentBackgroundToolResultBatch({
...input,
batchId: root.backgroundToolResultBatch.batchId,
}))
)
return { status: 'not_ready' };
return {
status: 'claimed',
claimId: input.deliveryKey,
batchId: root.backgroundToolResultBatch.batchId,
ownerStatus: 'applied',
};
}
if (root.backgroundToolResultBatch?.releasing === true) {
const released = await releaseAgentBackgroundToolResultClaims({
...input,
claimId: input.deliveryKey,
batchId: root.backgroundToolResultBatch.batchId,
});
return released ? claimAgentBackgroundToolResultBatch(input) : { status: 'not_ready' };
}
if (root.backgroundToolResult == null) return { status: 'not_ready', waitingForResult: true };
const claim = root.backgroundToolResult.resultClaim;
if (
claim?.claimId === input.deliveryKey &&
claim.batchId !== root.backgroundToolResultBatch?.batchId
) {
await Delivery().updateOne(
{
...scope,
deliveryKey: input.deliveryKey,
'backgroundToolResult.resultClaim.claimId': input.deliveryKey,
'backgroundToolResult.resultClaim.batchId': claim.batchId ?? null,
'backgroundToolResult.resultClaim.appliedAt': { $exists: false },
},
{ $unset: { 'backgroundToolResult.resultClaim': 1 } },
{ timestamps: false },
);
return { status: 'not_ready' };
}
if (claim != null && claim.claimId !== input.deliveryKey) {
if (
root.backgroundToolResultBatch != null &&
root.backgroundToolResultBatch.appliedAt == null
) {
await releaseAgentBackgroundToolResultClaims({
...input,
claimId: input.deliveryKey,
batchId: root.backgroundToolResultBatch.batchId,
});
}
const owner = await Delivery()
.findOne({ ...scope, deliveryKey: claim.claimId })
.select(projection)
.lean<Row>();
const batch = owner?.backgroundToolResultBatch;
const member = batch?.members?.includes(input.deliveryKey);
if (member && batch?.batchId === claim.batchId && batch?.appliedAt != null) {
// Retirement removed this owner from the queue. Its followers must
// finish the proof copy before they can all settle and abandon it.
if (
batch.proofCopiedAt == null &&
!(await confirmAgentBackgroundToolResultBatch({
...input,
deliveryKey: claim.claimId,
batchId: batch.batchId,
}))
)
return { status: 'not_ready' };
return {
status: 'claimed',
claimId: claim.claimId,
batchId: claim.batchId!,
ownerStatus: 'applied',
};
}
if (claim.appliedAt != null)
return {
status: 'claimed',
claimId: claim.claimId,
batchId: claim.batchId!,
ownerStatus: 'applied',
};
// A stale collector may finish a CAS after another collector froze a subset.
if (
(batch?.members != null && !member) ||
(batch != null && batch.batchId !== claim.batchId)
) {
await Delivery().updateOne(
{
...scope,
deliveryKey: input.deliveryKey,
'backgroundToolResult.resultClaim.claimId': claim.claimId,
'backgroundToolResult.resultClaim.batchId': claim.batchId ?? null,
'backgroundToolResult.resultClaim.appliedAt': { $exists: false },
},
{ $unset: { 'backgroundToolResult.resultClaim': 1 } },
{ timestamps: false },
);
return { status: 'not_ready' };
}
const retired =
owner?.result != null &&
typeof owner.result === 'object' &&
'backgroundToolCompletionRetired' in owner.result;
const recoverable =
owner == null ||
DEAD_STATUSES.includes(owner.status) ||
owner.capabilityStatus === 'dead' ||
retired;
return {
status: 'claimed',
claimId: claim.claimId,
batchId: claim.batchId!,
ownerStatus: recoverable ? 'recoverable' : 'pending',
};
}
if (root.backgroundToolResultBatch == null) {
const candidates =
input.limit === 1
? []
: await Delivery()
.find({
...scope,
deliveryKey: { $ne: input.deliveryKey },
status: { $in: UNDELIVERED_STATUSES },
capabilityStatus: { $ne: 'dead' },
backgroundToolResult: { $exists: true },
'backgroundToolResult.resultClaim': { $exists: false },
})
.select('deliveryKey envelope +backgroundToolResult')
.sort({ createdAt: 1, _id: 1 })
.limit(Math.min(input.limit, 32) - 1)
.lean<Row[]>();
const requested = projectBackgroundResult(root);
if (requested == null) return { status: 'not_ready' };
const metadataCost = (result: NonNullable<ReturnType<typeof projectBackgroundResult>>) =>
JSON.stringify(backgroundResultMetadata(result)).length + 1;
let remaining = input.maxMetadataChars - metadataCost(requested) - 1;
if (remaining < 0) throw new TypeError('Background result metadata exceeds its input budget');
const batchId = randomUUID();
const keys = [input.deliveryKey];
for (const candidate of candidates) {
const result = projectBackgroundResult(candidate);
if (result == null) continue;
const cost = metadataCost(result);
if (cost > remaining) continue;
remaining -= cost;
keys.push(candidate.deliveryKey);
}
const planned = await Delivery().updateOne(
{
...scope,
deliveryKey: input.deliveryKey,
backgroundToolResultBatch: { $exists: false },
status: { $in: UNDELIVERED_STATUSES },
capabilityStatus: { $ne: 'dead' },
...(input.deliveryClaimToken == null
? {}
: { capabilityClaimToken: input.deliveryClaimToken }),
$or: [
{ 'backgroundToolResult.resultClaim': { $exists: false } },
{ 'backgroundToolResult.resultClaim.claimId': input.deliveryKey },
],
},
{ $set: { backgroundToolResultBatch: { batchId, dispatchCount: 0, candidates: keys } } },
{ timestamps: false },
);
root =
planned.modifiedCount === 1
? { ...root, backgroundToolResultBatch: { batchId, dispatchCount: 0, candidates: keys } }
: await readRoot();
if (root == null) return { status: 'not_ready' };
}
let batch = root.backgroundToolResultBatch;
if (batch == null) return { status: 'not_ready' };
if (batch.members == null) {
// Retirement of a collecting-only plan contends with this root CAS.
// No sibling can depend on the owner until its own receipt is claimed.
if (root.backgroundToolResult?.resultClaim?.claimId !== input.deliveryKey) {
const claimedRoot = await Delivery().updateOne(
{
...scope,
deliveryKey: input.deliveryKey,
status: { $in: UNDELIVERED_STATUSES },
capabilityStatus: { $ne: 'dead' },
'backgroundToolResultBatch.batchId': batch.batchId,
'backgroundToolResultBatch.releasing': { $ne: true },
'backgroundToolResult.resultClaim': { $exists: false },
...(input.deliveryClaimToken == null
? {}
: { capabilityClaimToken: input.deliveryClaimToken }),
},
{
$set: {
'backgroundToolResult.resultClaim': {
kind: 'wakeup',
claimId: input.deliveryKey,
claimedAt: new Date(),
batchId: batch.batchId,
},
},
},
{ timestamps: false },
);
if (claimedRoot.matchedCount !== 1) return { status: 'not_ready' };
}
await Delivery().updateMany(
{
...scope,
deliveryKey: { $in: batch.candidates },
status: { $in: UNDELIVERED_STATUSES },
capabilityStatus: { $ne: 'dead' },
backgroundToolResult: { $exists: true },
'backgroundToolResult.resultClaim': { $exists: false },
},
{
$set: {
'backgroundToolResult.resultClaim': {
kind: 'wakeup',
claimId: input.deliveryKey,
claimedAt: new Date(),
batchId: batch.batchId,
},
},
},
{ timestamps: false },
);
const owned = await Delivery()
.find({
...scope,
deliveryKey: { $in: batch.candidates },
'backgroundToolResult.resultClaim.claimId': input.deliveryKey,
'backgroundToolResult.resultClaim.batchId': batch.batchId,
})
.select('deliveryKey')
.lean<Pick<IAgentTriggerDelivery, 'deliveryKey'>[]>();
const owners = new Set(owned.map((row) => row.deliveryKey));
const members = batch.candidates.filter((key) => owners.has(key));
if (!owners.has(input.deliveryKey)) {
await releaseAgentBackgroundToolResultClaims({
...input,
claimId: input.deliveryKey,
batchId: batch.batchId,
});
return { status: 'not_ready' };
}
const frozen = await Delivery().updateOne(
{
...scope,
deliveryKey: input.deliveryKey,
'backgroundToolResult.resultClaim.claimId': input.deliveryKey,
'backgroundToolResultBatch.batchId': batch.batchId,
'backgroundToolResultBatch.releasing': { $ne: true },
...(input.deliveryClaimToken == null
? {}
: { capabilityClaimToken: input.deliveryClaimToken }),
'backgroundToolResultBatch.members': { $exists: false },
},
{ $set: { 'backgroundToolResultBatch.members': members } },
{ timestamps: false },
);
if (frozen.modifiedCount === 1) {
batch = { ...batch, members };
} else {
root = await readRoot();
batch = root?.backgroundToolResultBatch;
}
}
if (batch?.members == null || !batch.members.includes(input.deliveryKey))
return { status: 'not_ready' };
const rows = await Delivery()
.find({
...scope,
deliveryKey: { $in: batch.members },
'backgroundToolResult.resultClaim.claimId': input.deliveryKey,
'backgroundToolResult.resultClaim.batchId': batch.batchId,
})
.select('deliveryKey envelope +backgroundToolResult')
.lean<Row[]>();
const byKey = new Map(rows.map((row) => [row.deliveryKey, projectBackgroundResult(row)]));
const results = batch.members.map((key) => byKey.get(key));
if (results.some((result) => result == null)) return { status: 'not_ready' };
if (
input.deliveryClaimToken != null &&
(await Delivery().exists({
...scope,
'backgroundToolResult.resultClaim.claimId': input.claimId,
deliveryKey: input.deliveryKey,
capabilityClaimToken: input.deliveryClaimToken,
'backgroundToolResultBatch.batchId': batch.batchId,
'backgroundToolResultBatch.releasing': { $ne: true },
})) == null
) {
return { status: 'not_ready' };
}
return {
status: 'acquired',
batchId: batch.batchId,
results: results.filter((result): result is NonNullable<typeof result> => result != null),
};
}
async function getAgentBackgroundToolResultBatch(
input: AgentBackgroundToolResultBatchOwnerInput,
): Promise<IAgentTriggerDelivery['backgroundToolResultBatch'] | null> {
const row = await Delivery()
.findOne({ ...backgroundResultIdentity(input), deliveryKey: input.deliveryKey })
.select('+backgroundToolResultBatch')
.lean<Pick<IAgentTriggerDelivery, 'backgroundToolResultBatch'>>();
return row?.backgroundToolResultBatch ?? null;
}
async function beginAgentBackgroundToolResultBatchDispatch(
input: AgentBackgroundToolResultBatchOwnerInput & { batchId: string; dispatchId: string },
): Promise<boolean> {
const marked = await Delivery().updateOne(
{
...backgroundResultIdentity(input),
deliveryKey: input.deliveryKey,
status: { $in: UNDELIVERED_STATUSES },
capabilityStatus: { $ne: 'dead' },
'backgroundToolResultBatch.batchId': input.batchId,
'backgroundToolResultBatch.members.0': { $exists: true },
'backgroundToolResultBatch.releasing': { $ne: true },
'backgroundToolResultBatch.appliedAt': { $exists: false },
...(input.deliveryClaimToken == null
? {}
: { capabilityClaimToken: input.deliveryClaimToken }),
},
{
$inc: { 'backgroundToolResultBatch.dispatchCount': 1 },
$set: { 'backgroundToolResultBatch.dispatchId': input.dispatchId },
},
{ timestamps: false },
);
return marked.matchedCount === 1;
}
/** Admission proof precedes follower settlement, including lost HTTP replies. */
async function confirmAgentBackgroundToolResultBatch(
input: AgentBackgroundToolResultBatchOwnerInput & { batchId: string },
): Promise<boolean> {
const scope = backgroundResultIdentity(input);
const root = await Delivery()
.findOneAndUpdate(
{
...scope,
deliveryKey: input.deliveryKey,
'backgroundToolResultBatch.members.0': { $exists: true },
'backgroundToolResultBatch.batchId': input.batchId,
'backgroundToolResultBatch.releasing': { $ne: true },
},
{ $min: { 'backgroundToolResultBatch.appliedAt': new Date() } },
{ new: true, timestamps: false },
)
.select('status +backgroundToolResultBatch')
.lean<Pick<IAgentTriggerDelivery, 'status' | 'backgroundToolResultBatch'>>();
if (root?.backgroundToolResultBatch?.appliedAt == null) return false;
await Delivery().updateMany(
{
...scope,
deliveryKey: { $in: root.backgroundToolResultBatch.members },
'backgroundToolResult.resultClaim.claimId': input.deliveryKey,
'backgroundToolResult.resultClaim.batchId': input.batchId,
},
{
$set: {
'backgroundToolResult.resultClaim.appliedAt': root.backgroundToolResultBatch.appliedAt,
},
},
{ timestamps: false },
);
if (
(await Delivery().exists({
...scope,
'backgroundToolResult.resultClaim.claimId': input.deliveryKey,
'backgroundToolResult.resultClaim.batchId': input.batchId,
'backgroundToolResult.resultClaim.appliedAt': { $exists: false },
})) != null
)
return false;
const finish = (retired: boolean) =>
Delivery().updateOne(
{
...scope,
deliveryKey: input.deliveryKey,
'backgroundToolResultBatch.batchId': input.batchId,
'backgroundToolResultBatch.appliedAt': { $exists: true },
status: retired ? 'succeeded' : { $ne: 'succeeded' },
},
{
$min: { 'backgroundToolResultBatch.proofCopiedAt': new Date() },
...(retired && { $max: { expiresAt: new Date(Date.now() + SUCCESS_RETENTION_MS) } }),
},
{ timestamps: false },
);
const finished = await finish(root.status === 'succeeded');
if (finished.matchedCount === 1) return true;
return root.status !== 'succeeded' && (await finish(true)).matchedCount === 1;
}
async function settleAgentTriggerHandlingOutcome(
@ -3715,8 +4361,7 @@ export function createAgentTriggerDeliveryMethods(
const error = normalizeFailure(input.error);
const receiptFence = recoverReceipt
? {
requiredWorkerCapability:
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },
backgroundToolResult: { $exists: true },
}
: {};
@ -4273,7 +4918,11 @@ export function createAgentTriggerDeliveryMethods(
},
{
$set: { backgroundToolResultErasedAt: erasedAt },
$unset: { backgroundToolResult: 1, backgroundToolResultDeletionPendingAt: 1 },
$unset: {
backgroundToolResult: 1,
backgroundToolResultBatch: 1,
backgroundToolResultDeletionPendingAt: 1,
},
},
{ timestamps: false },
);
@ -4287,7 +4936,7 @@ export function createAgentTriggerDeliveryMethods(
await Delivery().updateMany(
{
user,
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },
'envelope.target.conversationId': { $in: conversationIds },
backgroundToolResultErasedAt: { $exists: false },
},
@ -4316,6 +4965,10 @@ export function createAgentTriggerDeliveryMethods(
getAgentBackgroundToolResult,
getAgentBackgroundToolResultClaim,
claimAgentBackgroundToolResults,
claimAgentBackgroundToolResultBatch,
confirmAgentBackgroundToolResultBatch,
beginAgentBackgroundToolResultBatchDispatch,
getAgentBackgroundToolResultBatch,
releaseAgentBackgroundToolResultClaims,
settleAgentTriggerHandlingOutcome,
admitAgentEventActorAction,

View file

@ -168,6 +168,8 @@ const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(
kind: { type: String, enum: ['wakeup'], required: true },
claimId: { type: String, required: true, maxlength: 128 },
claimedAt: { type: Date, required: true },
appliedAt: { type: Date },
batchId: { type: String, maxlength: 128 },
},
{ _id: false },
),
@ -178,6 +180,23 @@ const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(
),
select: false,
},
backgroundToolResultBatch: {
type: new Schema(
{
batchId: { type: String, required: true, maxlength: 128 },
dispatchCount: { type: Number, required: true, min: 0 },
dispatchId: { type: String, maxlength: 128 },
releaseId: { type: String, maxlength: 128 },
proofCopiedAt: { type: Date },
candidates: { type: [String], required: true },
releasing: { type: Boolean },
members: { type: [String], default: undefined },
appliedAt: { type: Date },
},
{ _id: false },
),
select: false,
},
backgroundToolResultErasedAt: { type: Date, select: false },
backgroundToolResultDeletionPendingAt: { type: Date, select: false },
attempts: { type: Number, required: true, default: 0, min: 0 },

View file

@ -17,6 +17,8 @@ export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 =
'background_tool_completion_v1';
export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 =
'background_tool_completion_receipt_v2';
export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3 =
'background_tool_completion_batch_v3';
export const AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1 = 'agent_queued_turn_v1';
export const AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2 = 'agent_queued_turn_v2';
export const AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS: number = 64 * 1024;
@ -92,6 +94,8 @@ export interface AgentBackgroundToolResultReceipt {
kind: 'wakeup';
claimId: string;
claimedAt: Date;
appliedAt?: Date;
batchId?: string;
};
}
@ -128,6 +132,18 @@ export interface IAgentTriggerDelivery {
producerLeaseUntil?: Date;
/** Durable source of truth for a background completion. */
backgroundToolResult?: AgentBackgroundToolResultReceipt;
/** Receipt batching is separate from event-envelope coalescing. */
backgroundToolResultBatch?: {
batchId: string;
dispatchCount: number;
dispatchId?: string;
releaseId?: string;
proofCopiedAt?: Date;
candidates: string[];
members?: string[];
appliedAt?: Date;
releasing?: true;
};
/** Conversation-deletion fence preventing a late producer from restoring private output. */
backgroundToolResultErasedAt?: Date;
backgroundToolResultDeletionPendingAt?: Date;