From f88f3fb2febb174c730338726a2b68ec32aeb1d6 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 15:03:22 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=AA=87=20perf:=20Coalesce=20Ready=20Backg?= =?UTF-8?q?round=20Completions=20(#16650)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🪇 perf: Coalesce Ready Background Completions * fix: Recover Receipt Batches and Gate Rolling Deploys * fix: Fence Background Batch Handoffs and Cleanup Epochs * fix: Bind Resumed Cleanup to Its Observed Batch * fix: Preserve Background Results Through Recovery Handoffs * fix: Recover Unconfirmed Manual Background Claims * fix: Fence Automatic Recovery of Manual Handoffs * fix: Preserve Background Reconciliation and Recovery Retention * fix: Preserve Replayed Claims and Finish Follower Proofs * fix: Recover Stale Background Projection Epochs --------- Co-authored-by: Lia --- api/server/experimental.js | 2 + api/server/index.js | 2 + api/server/services/Agents/triggers.js | 18 + api/server/services/Agents/triggers.spec.js | 1 + .../Endpoints/agents/backgroundCompletion.js | 7 + packages/api/src/agents/background.spec.ts | 36 + packages/api/src/agents/background.ts | 24 +- packages/api/src/agents/backgroundClaims.ts | 32 +- .../api/src/agents/backgroundCompletion.ts | 3 + ...groundCompletionWakeup.integration.spec.ts | 1479 +++++++++++++++++ .../agents/backgroundCompletionWakeup.spec.ts | 239 ++- .../src/agents/backgroundCompletionWakeup.ts | 421 ++++- .../src/agents/handlers.background.spec.ts | 53 + packages/api/src/agents/handlers.ts | 3 + packages/api/src/agents/harvest.ts | 1 + packages/api/src/agents/triggers/README.md | 72 + packages/api/src/agents/triggers/dispatch.ts | 15 +- .../api/src/agents/triggers/engine.spec.ts | 7 +- packages/api/src/agents/triggers/engine.ts | 13 +- packages/api/src/agents/triggers/host.ts | 11 +- .../agents/triggers/service.delivery.spec.ts | 11 + packages/api/src/agents/triggers/service.ts | 15 +- .../api/src/stream/GenerationJobManager.ts | 14 +- .../stream/__tests__/idempotencyClaim.spec.ts | 7 + .../api/src/stream/interfaces/IJobStore.ts | 2 + packages/data-provider/src/config.spec.ts | 10 + packages/data-provider/src/config.ts | 5 +- packages/data-schemas/src/index.ts | 1 + packages/data-schemas/src/methods/index.ts | 1 + .../data-schemas/src/methods/message.spec.ts | 91 + packages/data-schemas/src/methods/message.ts | 298 +++- .../src/methods/triggerDelivery.spec.ts | 1092 +++++++++++- .../src/methods/triggerDelivery.ts | 785 ++++++++- .../src/schema/triggerDelivery.ts | 19 + .../data-schemas/src/types/triggerDelivery.ts | 16 + 35 files changed, 4610 insertions(+), 196 deletions(-) create mode 100644 packages/api/src/agents/backgroundCompletionWakeup.integration.spec.ts diff --git a/api/server/experimental.js b/api/server/experimental.js index 18983dafb8..2ad6e559fa 100644 --- a/api/server/experimental.js +++ b/api/server/experimental.js @@ -709,6 +709,8 @@ if (cluster.isMaster) { address: server.address(), completionResultBatchSize: baseAppConfig?.endpoints?.agents?.backgroundTasks?.completionResultBatchSize, + completionReceiptBatching: + baseAppConfig?.endpoints?.agents?.backgroundTasks?.completionReceiptBatching, idlePolling: baseAppConfig?.endpoints?.agents?.eventDriven?.idlePolling, }); } catch (initErr) { diff --git a/api/server/index.js b/api/server/index.js index eec43ca22e..4ceba2caff 100644 --- a/api/server/index.js +++ b/api/server/index.js @@ -508,6 +508,8 @@ const startServer = async () => { address: server.address(), completionResultBatchSize: appConfig?.endpoints?.agents?.backgroundTasks?.completionResultBatchSize, + completionReceiptBatching: + appConfig?.endpoints?.agents?.backgroundTasks?.completionReceiptBatching, idlePolling: appConfig?.endpoints?.agents?.eventDriven?.idlePolling, }); const scheduleEngineArmed = (await initializeScheduleEngine()) != null; diff --git a/api/server/services/Agents/triggers.js b/api/server/services/Agents/triggers.js index f8944c6ea8..3977b8e3f8 100644 --- a/api/server/services/Agents/triggers.js +++ b/api/server/services/Agents/triggers.js @@ -6,6 +6,7 @@ const { createSubagentCompletionWakeupResolver, SUBAGENT_COMPLETION_SOURCE, createBackgroundToolCompletionWakeupResolver, + createBackgroundToolDeadClaimRecovery, BACKGROUND_TOOL_COMPLETION_SOURCE, createAgentQueuedTurnLifecycle, AGENT_QUEUED_TURN_SOURCE, @@ -30,6 +31,22 @@ const backgroundToolCompletionAdapter = createBackgroundToolCompletionWakeupReso methods, getGenerationJob: (conversationId) => GenerationJobManager.getJob(conversationId), getResultBatchSize: () => service.getBackgroundCompletionResultBatchSize(), + getGenerationAdmissionEvidence, + recoverDeadClaim: createBackgroundToolDeadClaimRecovery( + (...args) => service.retire(...args), + methods.releaseBackgroundToolResultClaims, + (conversationId) => GenerationJobManager.getJob(conversationId), + ({ userId, conversationId, claimId }) => + GenerationJobManager.fenceGenerationClaimForRecovery( + userId, + claimId, + conversationId, + conversationId, + ), + methods.releaseAgentBackgroundToolResultClaims, + methods, + getGenerationAdmissionEvidence, + ), getWaitMaxIntervalMs: () => service.getCompletionWaitMaxIntervalMs(), }); const eventActorAdapter = createAgentEventContinueResolver({ @@ -95,6 +112,7 @@ module.exports = { persistAgentBackgroundToolResult: service.persistBackgroundToolResult, expediteCompletionWakeups: service.expediteCompletionWakeups, getAgentBackgroundToolResultClaim: service.getBackgroundToolResultClaim, + getBackgroundCompletionReceiptBatching: service.getBackgroundCompletionReceiptBatching, releaseAgentBackgroundToolResultClaims: service.releaseBackgroundToolResultClaims, drainAgentTriggerDeliveriesForUser: service.drainUser, prepareAgentTriggerUserPurge: service.prepareUserPurge, diff --git a/api/server/services/Agents/triggers.spec.js b/api/server/services/Agents/triggers.spec.js index df3a09b2d6..ed99ccab28 100644 --- a/api/server/services/Agents/triggers.spec.js +++ b/api/server/services/Agents/triggers.spec.js @@ -19,6 +19,7 @@ const mockQueuedTurnLifecycle = { jest.mock('@librechat/api', () => ({ createCheckpointDeletionReclaimer: jest.fn((getJobs) => () => getJobs('owner', 'tenant')), + createBackgroundToolDeadClaimRecovery: jest.fn(() => jest.fn()), createAgentTriggerService: (...args) => mockCreateAgentTriggerService(...args), createAgentContinuationResolver: jest.fn(() => jest.fn()), createAgentEventContinueResolver: jest.fn(() => jest.fn()), diff --git a/api/server/services/Endpoints/agents/backgroundCompletion.js b/api/server/services/Endpoints/agents/backgroundCompletion.js index 94d1f52eec..88ea7c4fa8 100644 --- a/api/server/services/Endpoints/agents/backgroundCompletion.js +++ b/api/server/services/Endpoints/agents/backgroundCompletion.js @@ -1,5 +1,6 @@ const { createBackgroundToolCompletionWakeupHandler, + GenerationJobManager, createBackgroundToolDeadClaimRecovery, createPendingBackgroundCompletions, createBackgroundToolResultHandler, @@ -7,12 +8,15 @@ const { } = require('@librechat/api'); const { listPendingAgentBackgroundToolCompletions, + getAgentBackgroundToolResultBatch, + confirmAgentBackgroundToolResultBatch, listUndeliveredAgentTriggerTaskIds, } = require('~/models'); const { enqueueAgentTrigger, persistAgentBackgroundToolResult, getAgentBackgroundToolResultClaim, + getBackgroundCompletionReceiptBatching, releaseAgentBackgroundToolResultClaims, renewAgentTriggerProducerLease, retireAgentTrigger, @@ -26,6 +30,7 @@ const preregisterBackgroundToolCompletion = createBackgroundToolCompletionWakeup (deliveryKey, sourceId, result) => persistAgentBackgroundToolResult({ deliveryKey, sourceId, result }), (deliveryKey) => expediteCompletionWakeups({ deliveryKeys: [deliveryKey] }), + getBackgroundCompletionReceiptBatching, ); const pendingBackgroundToolCompletions = createPendingBackgroundCompletions({ @@ -52,6 +57,8 @@ function createDeadBackgroundToolClaimRecovery( getGenerationJob, fenceGenerationClaim, releaseAgentBackgroundToolResultClaims, + { getAgentBackgroundToolResultBatch, confirmAgentBackgroundToolResultBatch }, + (...args) => GenerationJobManager.getGenerationAdmissionEvidence(...args), ); } diff --git a/packages/api/src/agents/background.spec.ts b/packages/api/src/agents/background.spec.ts index 571dca62f6..ead7e37788 100644 --- a/packages/api/src/agents/background.spec.ts +++ b/packages/api/src/agents/background.spec.ts @@ -53,6 +53,42 @@ async function waitForSubagentTaskToSettle( throw new Error('Timed out waiting for the detached subagent task.'); } +describe('manual reconciliation restoration', () => { + it('upgrades only the confirmed local owner and preserves the marker on replay', () => { + const registry = new BackgroundTaskRegistryClass(); + const created = registry.create({ + userId: 'marker-user', + conversationId: 'marker-convo', + toolCallId: 'marker-call', + toolName: 'tool', + }); + if ('atCapacity' in created) throw new Error('Unexpected capacity'); + registry.complete('marker-user', 'marker-convo', created.task.id, { content: 'done' }); + const claim = { kind: 'manual' as const, claimId: 'poll' }; + expect(registry.claimResult('marker-user', 'marker-convo', created.task.id, claim)).toBe( + 'acquired', + ); + expect( + registry.claimResult('marker-user', 'marker-convo', created.task.id, { + ...claim, + claimId: 'foreign', + receiptReconciled: true, + }), + ).toBe('claimed'); + expect(created.task.resultClaim?.receiptReconciled).toBeUndefined(); + expect( + registry.claimResult('marker-user', 'marker-convo', created.task.id, { + ...claim, + receiptReconciled: true, + }), + ).toBe('replay'); + expect(registry.claimResult('marker-user', 'marker-convo', created.task.id, claim)).toBe( + 'replay', + ); + expect(created.task.resultClaim?.receiptReconciled).toBe(true); + }); +}); + describe('isBackgroundEligibleToolName', () => { it('excludes direct-path, host-special, and machinery tools', () => { for (const name of [ diff --git a/packages/api/src/agents/background.ts b/packages/api/src/agents/background.ts index f98f0b3406..43cfb15893 100644 --- a/packages/api/src/agents/background.ts +++ b/packages/api/src/agents/background.ts @@ -668,6 +668,7 @@ export interface BackgroundTask { claimId: string; claimedAt: number; generationId?: string; + receiptReconciled?: true; }; /** The declared tool may return a process-local live artifact. A terminal * same-generation poll may therefore deliver from the local claim after it @@ -1823,7 +1824,12 @@ export class BackgroundTaskRegistryClass { userId: string, conversationId: string, taskId: string, - claim: { kind: 'manual' | 'wakeup'; claimId: string; generationId?: string }, + claim: { + kind: 'manual' | 'wakeup'; + claimId: string; + generationId?: string; + receiptReconciled?: true; + }, ): 'acquired' | 'replay' | 'claimed' | 'not_ready' { const task = this.get(userId, conversationId, taskId); if (task == null || task.status === 'running') { @@ -1834,9 +1840,14 @@ export class BackgroundTaskRegistryClass { task.updatedAt = Date.now(); return 'acquired'; } - return task.resultClaim.kind === claim.kind && task.resultClaim.claimId === claim.claimId - ? 'replay' - : 'claimed'; + if (task.resultClaim.kind !== claim.kind || task.resultClaim.claimId !== claim.claimId) { + return 'claimed'; + } + if (claim.receiptReconciled === true && task.resultClaim.receiptReconciled !== true) { + task.resultClaim.receiptReconciled = true; + task.updatedAt = Date.now(); + } + return 'replay'; } releaseResultClaim( @@ -2554,6 +2565,7 @@ export async function runCheckBackgroundTask(params: { conversationId, messageId: task.messageId, claimId: existingClaim.claimId, + ...(existingClaim.batchId != null && { batchId: existingClaim.batchId }), ...(existingClaim.kind === 'manual' ? { kind: 'manual' as const, @@ -2674,6 +2686,7 @@ export async function runCheckBackgroundTask(params: { taskId, { kind: 'manual', + receiptReconciled: true, claimId: invocationId, ...(params.generationId == null ? {} : { generationId: params.generationId }), }, @@ -2744,6 +2757,7 @@ export async function runCheckBackgroundTask(params: { taskId, { kind: 'manual', + receiptReconciled: true, claimId: invocationId, ...(params.generationId == null ? {} : { generationId: params.generationId }), }, @@ -2854,6 +2868,7 @@ export async function runCheckBackgroundTask(params: { conversationId, messageId: claimedResult.messageId, claimId: existingClaim.claimId, + ...(existingClaim.batchId != null && { batchId: existingClaim.batchId }), ...(existingClaim.kind === 'manual' ? { kind: 'manual' as const, @@ -3242,6 +3257,7 @@ export function getBackgroundCodeDelivery(params: { kind: task.resultClaim.kind, claimId: task.resultClaim.claimId, claimedAt: new Date(task.resultClaim.claimedAt), + ...(task.resultClaim.receiptReconciled === true && { receiptReconciled: true }), ...(task.resultClaim.generationId == null ? {} : { generationId: task.resultClaim.generationId }), diff --git a/packages/api/src/agents/backgroundClaims.ts b/packages/api/src/agents/backgroundClaims.ts index 6f92bca763..320aab3994 100644 --- a/packages/api/src/agents/backgroundClaims.ts +++ b/packages/api/src/agents/backgroundClaims.ts @@ -4,7 +4,11 @@ import { BACKGROUND_TOOL_COMPLETION_SOURCE } from './backgroundCompletionWakeup' /** Reconciles manual polling with the independent automatic-delivery receipt, * including polls reconstructed after the process-local registry was lost. */ export async function claimBackgroundToolResult( - methods: Pick, + methods: Pick< + MessageMethods, + 'claimBackgroundToolResults' | 'releaseBackgroundToolResultClaims' + > & + Partial>, getReceiptClaim: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'], input: Parameters[0], ): ReturnType { @@ -30,7 +34,7 @@ export async function claimBackgroundToolResult( if (messageId == null) { throw new Error('The background result claim has no parent message identity'); } - const release = async () => { + const release = async (allowReconciled = false) => { const released = await methods.releaseBackgroundToolResultClaims({ userId: input.userId, conversationId: input.conversationId, @@ -38,8 +42,10 @@ export async function claimBackgroundToolResult( taskIds: messageClaim.results.map((result) => result.taskId), kind: input.kind, claimId: input.claimId, + ...(input.batchId != null && { batchId: input.batchId }), + ...(input.kind === 'manual' && { onlyIfUnreconciled: true }), }); - if (!released) { + if (!released && !allowReconciled) { throw new Error('The background result claim could not be released'); } }; @@ -51,6 +57,26 @@ export async function claimBackgroundToolResult( throw error; } if (receiptClaim == null || receiptClaim.claimId === input.claimId) { + if (input.kind === 'manual' && methods.confirmBackgroundToolResultClaim != null) { + try { + if ( + !(await methods.confirmBackgroundToolResultClaim({ + userId: input.userId, + conversationId: input.conversationId, + messageId, + taskId: input.taskId, + claimId: input.claimId, + })) + ) { + throw new Error('The reconciled background result claim was superseded'); + } + } catch (error) { + // This CAS removes only speculative ownership. A committed or unknown + // handoff remains owned for generation-fenced recovery. + await release(true); + throw error; + } + } return messageClaim; } await release(); diff --git a/packages/api/src/agents/backgroundCompletion.ts b/packages/api/src/agents/backgroundCompletion.ts index 8886ea89c0..e04f82a262 100644 --- a/packages/api/src/agents/backgroundCompletion.ts +++ b/packages/api/src/agents/backgroundCompletion.ts @@ -65,6 +65,9 @@ export interface BackgroundToolDeadClaimRecoveryInput { conversationId: string; messageId: string; claimId: string; + batchId?: string; + /** Automatic repair cannot reopen a committed manual handoff. */ + onlyIfUnreconciled?: true; /** Omitted for the legacy automatic-wakeup recovery path. */ kind?: 'manual' | 'wakeup'; /** Required to prove that a manual claim's owning generation is no longer active. */ diff --git a/packages/api/src/agents/backgroundCompletionWakeup.integration.spec.ts b/packages/api/src/agents/backgroundCompletionWakeup.integration.spec.ts new file mode 100644 index 0000000000..5b06bdcb73 --- /dev/null +++ b/packages/api/src/agents/backgroundCompletionWakeup.integration.spec.ts @@ -0,0 +1,1479 @@ +import mongoose from 'mongoose'; +import { MongoMemoryServer } from 'mongodb-memory-server'; +import { + createMethods, + createModels, + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3, +} from '@librechat/data-schemas'; +import type { IMessage } from '@librechat/data-schemas'; +import type { AgentTriggerFetch } from './triggers/host'; +import { + createBackgroundToolCompletionWakeupResolver, + createBackgroundToolDeadClaimRecovery, + BACKGROUND_TOOL_COMPLETION_SOURCE, +} from './backgroundCompletionWakeup'; +import { + backgroundTaskRegistry, + runCheckBackgroundTask, + getBackgroundCodeDelivery, +} from './background'; +import { createAgentTriggerEnvelope, getAgentTriggerIdempotencyKey } from './triggers/envelope'; +import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport'; +import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore'; +import { GenerationJobManagerClass } from '~/stream/GenerationJobManager'; +import { createAgentTriggerExecutionHost } from './triggers/host'; +import { prepareAgentTriggerDelivery } from './triggers/delivery'; +import { claimBackgroundToolResult } from './backgroundClaims'; + +let mongo: MongoMemoryServer; +let methods: ReturnType; +let userId: string; +const conversationId = 'receipt-replay'; +const parentMessageId = 'receipt-parent'; +const agentId = 'agent_parent_1'; +const capability = AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3; + +beforeAll(async () => { + mongo = await MongoMemoryServer.create(); + await mongoose.connect(mongo.getUri()); + createModels(mongoose); + methods = createMethods(mongoose); +}, 60_000); + +afterAll(async () => { + await mongoose.disconnect(); + await mongo.stop(); +}); + +beforeEach(async () => { + await Promise.all( + ['AgentTriggerDelivery', 'AgentTriggerLaneSequence', 'Conversation', 'Message'].map((name) => + mongoose.models[name].deleteMany({}), + ), + ); + userId = new mongoose.Types.ObjectId().toString(); + await mongoose.models.Conversation.create({ + conversationId, + user: userId, + tenantId: 'tenant-1', + endpoint: 'agents', + agent_id: agentId, + }); + await mongoose.models.Message.create({ + messageId: parentMessageId, + conversationId, + user: userId, + tenantId: 'tenant-1', + parentMessageId: 'user-parent', + isCreatedByUser: false, + unfinished: false, + endpoint: 'agents', + content: [], + }); +}); + +async function ready(taskId: string, persistReceipt = true) { + const envelope = createAgentTriggerEnvelope({ + mode: 'continue', + requestId: taskId, + deliveryId: taskId, + receivedAt: Date.now(), + principal: { id: userId, tenantId: 'tenant-1' }, + event: { + id: taskId, + type: 'background-tool.completion', + occurredAt: Date.now(), + source: { id: BACKGROUND_TOOL_COMPLETION_SOURCE, type: 'internal' }, + payload: { taskId, toolCallId: taskId, toolName: 'tool' }, + }, + target: { conversationId, parentMessageId, agentId }, + input: 'waiting', + }); + const row = prepareAgentTriggerDelivery(envelope, { + orderingKey: taskId, + requiredWorkerCapability: capability, + }); + await methods.enqueueAgentTriggerDelivery(row); + if (persistReceipt) + await methods.persistAgentBackgroundToolResult({ + deliveryKey: row.deliveryKey, + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + result: { status: 'completed', output: taskId, settledAt: new Date() }, + }); + return envelope; +} + +function owner(deliveryKey: string) { + return { + deliveryKey, + userId, + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + tenantId: 'tenant-1', + conversationId, + parentMessageId, + agentId, + }; +} + +it('replays exactly the admitted input after a lost response and a definitely rejected retry', async () => { + const root = await ready('one'); + const sibling = await ready('two'); + const inputs: string[] = []; + const consumed: string[] = []; + let firstInput: string | undefined; + let calls = 0; + const fetcher: AgentTriggerFetch = async (_url, init) => { + const body = JSON.parse(String(init?.body)) as { text: string }; + inputs.push(body.text); + if (++calls === 1) { + firstInput = body.text; + consumed.push('one', 'two'); + throw Object.assign(new Error('lost admitted response'), { code: 'ECONNRESET' }); + } + if (calls === 2) + throw Object.assign(new Error('retry did not connect'), { code: 'ECONNREFUSED' }); + if (body.text !== firstInput) consumed.push('three'); + return new Response( + JSON.stringify({ + status: 'started', + streamId: conversationId, + conversationId, + generationCreatedAt: 100, + }), + { status: 200 }, + ); + }; + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods, + getGenerationJob: async () => null, + }); + const host = createAgentTriggerExecutionHost({ + prepareContinue: resolve, + fetch: fetcher, + mintToken: () => 'test', + getBaseUrl: () => 'http://localhost', + }); + const options = { requiredWorkerCapability: capability }; + await expect(host.dispatch(root, options)).rejects.toMatchObject({ certainty: 'ambiguous' }); + const late = await ready('three'); + await expect(host.dispatch(root, options)).rejects.toMatchObject({ certainty: 'definite' }); + await expect(host.dispatch(root, options)).resolves.toMatchObject({ status: 'started' }); + expect(inputs).toEqual([firstInput, firstInput, firstInput]); + expect(consumed).toEqual(['one', 'two']); + await expect(host.dispatch(sibling, options)).resolves.toMatchObject({ status: 'settled' }); + const lateEnvelope = late.mode === 'continue' ? late : undefined; + if (lateEnvelope == null) throw new Error('Expected continuation'); + const prepared = await resolve(lateEnvelope, { + idempotencyKey: getAgentTriggerIdempotencyKey(late), + ...options, + }); + expect(prepared?.status === 'ready' && prepared.input).toContain('three'); +}); + +it('cleans a late predecessor projection without clearing its successor receipt batch', async () => { + const root = await ready('one'); + const deliveryKey = getAgentTriggerIdempotencyKey(root); + const scope = owner(deliveryKey); + const old = await methods.claimAgentBackgroundToolResultBatch({ + ...scope, + limit: 8, + maxMetadataChars: 16000, + }); + if (old.status !== 'acquired') throw new Error('Expected batch'); + expect( + await methods.releaseAgentBackgroundToolResultClaims({ + ...scope, + claimId: deliveryKey, + batchId: old.batchId, + }), + ).toBe(true); + await mongoose.models.Message.updateOne( + { messageId: parentMessageId, user: userId }, + { + $set: { + content: [ + { + type: 'tool_call', + tool_call: { + id: 'one', + output: 'one', + backgroundTask: { + taskId: 'one', + toolName: 'tool', + status: 'completed', + completionWakeup: true, + completionReceipt: true, + }, + }, + }, + ], + }, + }, + ); + // A predecessor's delayed projection CAS lands after its cleanup completed. + await methods.claimBackgroundToolResults({ + userId, + conversationId, + messageId: parentMessageId, + taskId: 'one', + kind: 'wakeup', + claimId: deliveryKey, + batchId: old.batchId, + limit: 1, + }); + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods, + getGenerationJob: async () => null, + }); + if (root.mode !== 'continue') throw new Error('Expected continuation'); + await expect( + resolve(root, { idempotencyKey: deliveryKey, requiredWorkerCapability: capability }), + ).rejects.toMatchObject({ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING' }); + const successor = await methods.getAgentBackgroundToolResultBatch(scope); + expect(successor?.batchId).not.toBe(old.batchId); + await expect( + resolve(root, { idempotencyKey: deliveryKey, requiredWorkerCapability: capability }), + ).resolves.toMatchObject({ status: 'ready' }); + expect((await methods.getAgentBackgroundToolResultBatch(scope))?.batchId).toBe( + successor?.batchId, + ); +}); + +it('confirms a dead owner with native admission proof instead of re-presenting its results', async () => { + const root = await ready('one'); + const sibling = await ready('two'); + const deliveryKey = getAgentTriggerIdempotencyKey(root); + const scope = owner(deliveryKey); + await methods.claimAgentBackgroundToolResultBatch({ + ...scope, + limit: 8, + maxMetadataChars: 16000, + }); + await mongoose.models.AgentTriggerDelivery.updateOne( + { deliveryKey }, + { $set: { capabilityStatus: 'dead' } }, + ); + const recover = createBackgroundToolDeadClaimRecovery( + async (key, sourceId, reason, options) => + methods.retireAgentTriggerDelivery({ + deliveryKey: key, + sourceId, + reason, + settledAt: new Date(), + ...options, + }), + methods.releaseBackgroundToolResultClaims, + async () => null, + async () => 'started', + methods.releaseAgentBackgroundToolResultClaims, + methods, + async () => ({ generationId: conversationId, generationCreatedAt: 100 }), + ); + expect( + await recover({ userId, conversationId, messageId: parentMessageId, claimId: deliveryKey }), + ).toBe(false); + const follower = await methods.claimAgentBackgroundToolResultBatch({ + ...owner(getAgentTriggerIdempotencyKey(sibling)), + limit: 8, + maxMetadataChars: 16000, + }); + expect(follower).toMatchObject({ status: 'claimed', ownerStatus: 'applied' }); +}); + +async function project(taskId: string) { + await mongoose.models.Message.updateOne( + { messageId: parentMessageId, user: userId }, + { + $push: { + content: { + type: 'tool_call', + tool_call: { + id: taskId, + output: `durable-${taskId}`, + backgroundTask: { + taskId, + toolName: 'tool', + status: 'completed', + settledAt: new Date(), + completionWakeup: true, + completionReceipt: true, + }, + }, + }, + }, + }, + ); +} + +it('reconstructs a failed receipt write from the successful terminal message projection', async () => { + const root = await ready('one', false); + const deliveryKey = getAgentTriggerIdempotencyKey(root); + await project('one'); + await mongoose.models.AgentTriggerDelivery.updateOne( + { deliveryKey }, + { $set: { producerLeaseUntil: new Date(0) } }, + ); + const fetcher: AgentTriggerFetch = async (_url, init) => { + expect(JSON.parse(String(init?.body)).text).toContain('durable-one'); + return new Response( + JSON.stringify({ + status: 'started', + conversationId, + streamId: conversationId, + generationCreatedAt: 100, + }), + { status: 200 }, + ); + }; + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods, + getGenerationJob: async () => null, + }); + const host = createAgentTriggerExecutionHost({ + prepareContinue: resolve, + fetch: fetcher, + mintToken: () => 'test', + getBaseUrl: () => 'http://localhost', + }); + await expect( + host.dispatch(root, { requiredWorkerCapability: capability }), + ).resolves.toMatchObject({ status: 'started' }); + expect( + await methods.getAgentBackgroundToolResult({ + deliveryKey, + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + }), + ).toMatchObject({ output: 'durable-one', resultClaim: { appliedAt: expect.any(Date) } }); +}); + +it('does not settle a root when a speculative manual poll and automatic owner mutually yield', async () => { + const root = await ready('one'); + await ready('two'); + await project('one'); + const key = getAgentTriggerIdempotencyKey(root); + let entered: () => void = () => undefined; + let captured: () => void = () => undefined; + let resume: () => void = () => undefined; + const automaticEntered = new Promise((resolve) => { + entered = resolve; + }); + const snapshotTaken = new Promise((resolve) => { + captured = resolve; + }); + const manualMayYield = new Promise((resolve) => { + resume = resolve; + }); + let lookups = 0; + const getter = async (input: Parameters[0]) => { + if (++lookups === 1) return methods.getAgentBackgroundToolResultClaim(input); + await automaticEntered; + const snapshot = await methods.getAgentBackgroundToolResultClaim(input); + captured(); + await manualMayYield; + return snapshot; + }; + const claim = methods.claimBackgroundToolResults.bind(methods); + const automatic = jest + .spyOn(methods, 'claimBackgroundToolResults') + .mockImplementation(async (input) => { + if (input.kind === 'wakeup' && input.taskId === 'one') { + entered(); + await snapshotTaken; + } + return claim(input); + }); + const manual = claimBackgroundToolResult(methods, getter, { + userId, + conversationId, + messageId: parentMessageId, + taskId: 'one', + kind: 'manual', + claimId: 'manual-poll', + generationId: 'manual-generation', + }); + // Allow the initial manual projection CAS to commit before receipt selection. + const deadline = Date.now() + 3000; + while ( + (await mongoose.models.Message.findOne({ + messageId: parentMessageId, + 'content.tool_call.backgroundTask.resultClaim.claimId': 'manual-poll', + }).lean()) == null + ) { + if (Date.now() > deadline) throw new Error('Manual claim did not arrive'); + await new Promise((resolve) => setTimeout(resolve, 5)); + } + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods, + getGenerationJob: async () => null, + }); + if (root.mode !== 'continue') throw new Error('Expected continuation'); + await expect( + resolve(root, { idempotencyKey: key, requiredWorkerCapability: capability }), + ).rejects.toMatchObject({ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING', deferWithoutAttempt: true }); + resume(); + expect(await manual).toMatchObject({ status: 'claimed' }); + automatic.mockRestore(); + const retry = await resolve(root, { idempotencyKey: key, requiredWorkerCapability: capability }); + expect(retry?.status === 'ready' && retry.input).toContain('one'); + expect(retry?.status === 'ready' && retry.input).toContain('two'); +}); + +it('settles a root only after a manual poll has completed receipt reconciliation', async () => { + const root = await ready('one'); + await project('one'); + expect( + ( + await claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, { + userId, + conversationId, + messageId: parentMessageId, + taskId: 'one', + kind: 'manual', + claimId: 'manual-poll', + }) + ).status, + ).toBe('acquired'); + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods, + getGenerationJob: async () => null, + }); + if (root.mode !== 'continue') throw new Error('Expected continuation'); + await expect( + resolve(root, { + idempotencyKey: getAgentTriggerIdempotencyKey(root), + requiredWorkerCapability: capability, + }), + ).resolves.toEqual({ status: 'settled' }); +}); + +it.each([false, true])( + 'rolls back an unconfirmed manual claim with generation identity %s', + async (withGeneration) => { + const root = await ready('one'); + await project('one'); + const confirmation = jest + .spyOn(mongoose.models.Message, 'updateOne') + .mockImplementationOnce(() => { + throw new Error('confirmation unavailable'); + }); + await expect( + claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, { + userId, + conversationId, + messageId: parentMessageId, + taskId: 'one', + kind: 'manual', + claimId: 'manual-poll', + ...(withGeneration && { generationId: 'manual-generation' }), + }), + ).rejects.toThrow('confirmation unavailable'); + confirmation.mockRestore(); + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods, + getGenerationJob: async () => null, + }); + if (root.mode !== 'continue') throw new Error('Expected continuation'); + const prepared = await resolve(root, { + idempotencyKey: getAgentTriggerIdempotencyKey(root), + requiredWorkerCapability: capability, + }); + expect(prepared?.status === 'ready' && prepared.input).toContain('"result":"one"'); + }, +); + +it('resolves a committed manual confirmation whose write reply was lost', async () => { + const root = await ready('one'); + await project('one'); + const update = mongoose.models.Message.updateOne.bind(mongoose.models.Message); + const lostReply = jest + .spyOn(mongoose.models.Message, 'updateOne') + .mockImplementationOnce((...args) => { + const query = update(...args); + const execute = query.exec.bind(query); + jest.spyOn(query, 'exec').mockImplementationOnce(async () => { + await execute(); + throw new Error('lost confirmation reply'); + }); + return query; + }); + const consumed = await claimBackgroundToolResult( + methods, + methods.getAgentBackgroundToolResultClaim, + { + userId, + conversationId, + messageId: parentMessageId, + taskId: 'one', + kind: 'manual', + claimId: 'manual-poll', + generationId: 'manual-generation', + }, + ); + expect(consumed).toMatchObject({ status: 'acquired', results: [{ taskId: 'one' }] }); + lostReply.mockRestore(); + expect( + await methods.releaseBackgroundToolResultClaims({ + userId, + conversationId, + messageId: parentMessageId, + taskIds: ['one'], + kind: 'manual', + claimId: 'manual-poll', + onlyIfUnreconciled: true, + }), + ).toBe(false); + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods, + getGenerationJob: async () => null, + }); + if (root.mode !== 'continue') throw new Error('Expected continuation'); + await expect( + resolve(root, { + idempotencyKey: getAgentTriggerIdempotencyKey(root), + requiredWorkerCapability: capability, + }), + ).resolves.toEqual({ status: 'settled' }); +}); + +it('recovers abandoned manual ownership after both confirmation and rollback failed', async () => { + const root = await ready('one'); + await ready('two'); + await project('one'); + const confirmation = jest + .spyOn(methods, 'confirmBackgroundToolResultClaim') + .mockRejectedValueOnce(new Error('confirmation unavailable')); + const rollback = jest + .spyOn(methods, 'releaseBackgroundToolResultClaims') + .mockRejectedValueOnce(new Error('rollback unavailable')); + await expect( + claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, { + userId, + conversationId, + messageId: parentMessageId, + taskId: 'one', + kind: 'manual', + claimId: 'manual-poll', + generationId: 'manual-generation', + }), + ).rejects.toThrow('rollback unavailable'); + confirmation.mockRestore(); + rollback.mockRestore(); + let active = true; + const getGenerationJob = async () => + active ? { status: 'running', metadata: { responseMessageId: 'manual-generation' } } : null; + const recover = createBackgroundToolDeadClaimRecovery( + async () => false, + methods.releaseBackgroundToolResultClaims, + getGenerationJob, + async () => 'unavailable', + ); + const recoverDeadClaim = jest.fn(recover); + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods, + getGenerationJob, + recoverDeadClaim, + }); + if (root.mode !== 'continue') throw new Error('Expected continuation'); + const key = getAgentTriggerIdempotencyKey(root); + await mongoose.models.AgentTriggerDelivery.updateOne( + { deliveryKey: key }, + { $set: { capabilityClaimToken: 'exact-queue-lease' } }, + ); + const context = { + idempotencyKey: key, + requiredWorkerCapability: capability, + deliveryClaimToken: 'exact-queue-lease', + }; + await expect(resolve(root, context)).rejects.toMatchObject({ code: 'PARENT_NOT_READY' }); + expect(recoverDeadClaim).not.toHaveBeenCalled(); + active = false; + await expect(resolve(root, context)).rejects.toMatchObject({ + code: 'BACKGROUND_TOOL_CLAIM_RECONCILING', + deferWithoutAttempt: true, + }); + expect(recoverDeadClaim).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'manual', + claimId: 'manual-poll', + generationId: 'manual-generation', + }), + ); + const prepared = await resolve(root, context); + expect(prepared?.status === 'ready' && prepared.input).toContain('one'); + expect(prepared?.status === 'ready' && prepared.input).toContain('two'); +}); + +it('does not use a different manual owner as confirmation read-back proof', async () => { + await ready('one'); + await project('one'); + const update = mongoose.models.Message.updateOne.bind(mongoose.models.Message); + const replaced = jest + .spyOn(mongoose.models.Message, 'updateOne') + .mockImplementationOnce((...args) => { + const query = update(...args); + jest.spyOn(query, 'exec').mockImplementationOnce(async () => { + await update( + { user: userId, messageId: parentMessageId }, + { + $set: { + 'content.0.tool_call.backgroundTask.resultClaim': { + kind: 'manual', + claimId: 'successor-poll', + receiptReconciled: true, + }, + }, + }, + ); + throw new Error('predecessor confirmation failed'); + }); + return query; + }); + await expect( + claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, { + userId, + conversationId, + messageId: parentMessageId, + taskId: 'one', + kind: 'manual', + claimId: 'manual-poll', + }), + ).rejects.toThrow('predecessor confirmation failed'); + replaced.mockRestore(); + expect( + await methods.claimBackgroundToolResults({ + userId, + conversationId, + messageId: parentMessageId, + taskId: 'one', + kind: 'wakeup', + claimId: 'automatic', + }), + ).toMatchObject({ + status: 'claimed', + claim: { claimId: 'successor-poll', receiptReconciled: true }, + }); +}); + +it.each(['missing-receipt', 'batch-root', 'batch-sibling'] as const)( + 'preserves a manual handoff committed after the %s recovery snapshot', + async (scenario) => { + const polled = await ready('one', scenario !== 'missing-receipt'); + const other = scenario === 'batch-sibling' ? await ready('two') : undefined; + const root = other ?? polled; + await project('one'); + let confirmationReached: () => void = () => undefined; + let confirmNow: () => void = () => undefined; + let recoveryReached: () => void = () => undefined; + let recoverNow: () => void = () => undefined; + const confirmationEntered = new Promise((resolve) => { + confirmationReached = resolve; + }); + const confirmationBarrier = new Promise((resolve) => { + confirmNow = resolve; + }); + const recoveryEntered = new Promise((resolve) => { + recoveryReached = resolve; + }); + const recoveryBarrier = new Promise((resolve) => { + recoverNow = resolve; + }); + const confirm = methods.confirmBackgroundToolResultClaim.bind(methods); + const pausedConfirmation = jest + .spyOn(methods, 'confirmBackgroundToolResultClaim') + .mockImplementationOnce(async (input) => { + confirmationReached(); + await confirmationBarrier; + return confirm(input); + }); + const manual = claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, { + userId, + conversationId, + messageId: parentMessageId, + taskId: 'one', + kind: 'manual', + claimId: 'manual-poll', + generationId: 'manual-generation', + }); + await confirmationEntered; + let manualActive = true; + const release = jest.fn(methods.releaseBackgroundToolResultClaims); + const recover = createBackgroundToolDeadClaimRecovery( + async () => false, + release, + async () => + manualActive + ? { status: 'running', metadata: { responseMessageId: 'manual-generation' } } + : null, + async () => 'unavailable', + ); + const recoverDeadClaim = jest.fn(async (input: Parameters[0]) => { + recoveryReached(); + await recoveryBarrier; + return recover(input); + }); + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods, + getGenerationJob: async () => null, + recoverDeadClaim, + }); + if (root.mode !== 'continue') throw new Error('Expected continuation'); + const key = getAgentTriggerIdempotencyKey(root); + await mongoose.models.AgentTriggerDelivery.updateMany( + { deliveryKey: { $in: [key, getAgentTriggerIdempotencyKey(polled)] } }, + { + $set: { + capabilityStatus: 'leased', + capabilityClaimToken: 'exact-queue-lease', + capabilityLeaseBy: 'automatic-worker', + capabilityLeaseUntil: new Date(Date.now() + 60_000), + }, + }, + ); + const context = { + idempotencyKey: key, + requiredWorkerCapability: capability, + deliveryClaimToken: 'exact-queue-lease', + }; + const automatic = resolve(root, context); + await recoveryEntered; + confirmNow(); + expect(await manual).toMatchObject({ status: 'acquired', results: [{ taskId: 'one' }] }); + // The automatic queue lease won manual retirement. The poll returned its + // result and its generation ended while recovery held a stale snapshot. + expect( + await methods.retireAgentTriggerDelivery({ + deliveryKey: getAgentTriggerIdempotencyKey(polled), + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + settledAt: new Date(), + reason: 'manual poll', + onlyIfUnclaimed: true, + requireTransition: true, + }), + ).toBe(false); + manualActive = false; + recoverNow(); + await expect(automatic).rejects.toMatchObject({ + code: 'BACKGROUND_TOOL_CLAIM_RECONCILING', + deferWithoutAttempt: true, + }); + expect(recoverDeadClaim).toHaveBeenCalledWith( + expect.objectContaining({ onlyIfUnreconciled: true }), + ); + expect(release).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'manual', claimId: 'manual-poll', onlyIfUnreconciled: true }), + ); + const claim = await methods.claimBackgroundToolResults({ + userId, + conversationId, + messageId: parentMessageId, + taskId: 'one', + kind: 'wakeup', + claimId: 'probe', + }); + expect(claim).toMatchObject({ + status: 'claimed', + claim: { claimId: 'manual-poll', receiptReconciled: true }, + }); + const pollEnvelope = polled.mode === 'continue' ? polled : undefined; + if (pollEnvelope == null) throw new Error('Expected polled continuation'); + await expect( + resolve(pollEnvelope, { + idempotencyKey: getAgentTriggerIdempotencyKey(polled), + requiredWorkerCapability: capability, + }), + ).resolves.toEqual({ status: 'settled' }); + if (other != null) { + const row = await methods.getAgentTriggerDelivery(getAgentTriggerIdempotencyKey(polled)); + if (row == null) throw new Error('Expected polled delivery'); + expect( + await methods.completeAgentTriggerDelivery({ + id: row.id, + workerId: 'automatic-worker', + claimToken: 'exact-queue-lease', + attempt: 1, + settledAt: new Date(), + result: { status: 'settled', mode: 'continue', conversationId }, + }), + ).toBe(true); + const remaining = await resolve(root, context); + if (remaining?.status !== 'ready') throw new Error('Expected undelivered sibling'); + expect(remaining.input).toContain('"background_task_id":"two"'); + expect(remaining.input).not.toContain('"background_task_id":"one"'); + await remaining.beginDispatch?.(); + } + pausedConfirmation.mockRestore(); + }, +); + +it('keeps explicit manual recovery able to reopen a committed claim after its generation ends', async () => { + await ready('one'); + await project('one'); + await claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, { + userId, + conversationId, + messageId: parentMessageId, + taskId: 'one', + kind: 'manual', + claimId: 'manual-poll', + generationId: 'manual-generation', + }); + let active = true; + const recover = createBackgroundToolDeadClaimRecovery( + async () => false, + methods.releaseBackgroundToolResultClaims, + async () => + active ? { status: 'running', metadata: { responseMessageId: 'manual-generation' } } : null, + async () => 'unavailable', + ); + const input = { + userId, + conversationId, + messageId: parentMessageId, + claimId: 'manual-poll', + kind: 'manual' as const, + generationId: 'manual-generation', + }; + expect(await recover(input)).toBe(false); + active = false; + expect(await recover({ ...input, onlyIfUnreconciled: true })).toBe(false); + expect(await recover(input)).toBe(true); + expect( + ( + await methods.claimBackgroundToolResults({ + userId, + conversationId, + messageId: parentMessageId, + taskId: 'one', + kind: 'manual', + claimId: 'next-poll', + }) + ).status, + ).toBe('acquired'); +}); + +it.each(['same-generation', 'later-generation'] as const)( + 'restores manual reconciliation after %s final persistence', + async (scenario) => { + const generationId = scenario === 'same-generation' ? parentMessageId : 'manual-generation'; + const created = backgroundTaskRegistry.create({ + userId, + conversationId, + toolCallId: 'manual-final-call', + toolName: 'tool', + messageId: parentMessageId, + harvestStarted: true, + }); + if ('atCapacity' in created) throw new Error('Unexpected task capacity'); + const taskId = created.task.id; + const root = await ready(taskId); + await project(taskId); + await mongoose.models.Message.updateOne( + { user: userId, messageId: parentMessageId }, + { $set: { unfinished: true } }, + ); + const original = await mongoose.models.Message.findOne({ + user: userId, + messageId: parentMessageId, + }).lean>(); + backgroundTaskRegistry.complete(userId, conversationId, taskId, { content: 'manual-result' }); + backgroundTaskRegistry.finishHarvest(userId, conversationId, taskId); + backgroundTaskRegistry.markCompletionWakeup(userId, conversationId, taskId, { + renew: async () => true, + retire: async (reason, options) => + methods.retireAgentTriggerDelivery({ + deliveryKey: getAgentTriggerIdempotencyKey(root), + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + settledAt: new Date(), + reason, + ...options, + }), + }); + const polled = JSON.parse( + await runCheckBackgroundTask({ + userId, + conversationId, + args: { background_task_id: taskId }, + toolCallId: 'confirmed-poll', + runId: 'manual-run', + generationId, + claimBackgroundToolResult: (input) => + claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, input), + }), + ); + expect(polled).toMatchObject({ status: 'completed', result: 'manual-result' }); + expect(backgroundTaskRegistry.get(userId, conversationId, taskId)?.resultClaim).toMatchObject({ + receiptReconciled: true, + }); + // Final full persistence uses content captured before manual confirmation. + await mongoose.models.Message.updateOne( + { user: userId, messageId: parentMessageId }, + { $set: { unfinished: false, content: original?.content } }, + ); + const restored = getBackgroundCodeDelivery({ + userId, + conversationId, + args: { background_task_id: taskId }, + }); + expect(restored?.backgroundTask?.resultClaim).toMatchObject({ receiptReconciled: true }); + await methods.updateToolCallResult({ + userId, + conversationId, + messageId: parentMessageId, + toolCallId: taskId, + output: 'manual-result', + backgroundTask: restored?.backgroundTask, + }); + const release = jest.fn(methods.releaseBackgroundToolResultClaims); + const recover = createBackgroundToolDeadClaimRecovery( + async () => false, + release, + async () => null, + async () => 'unavailable', + ); + const claim = await methods.claimBackgroundToolResults({ + userId, + conversationId, + messageId: parentMessageId, + taskId, + kind: 'wakeup', + claimId: 'automatic-probe', + }); + if (claim.status !== 'claimed' || claim.claim == null) + throw new Error('Expected preserved manual claim'); + expect(claim.claim).toMatchObject({ receiptReconciled: true }); + expect( + await recover({ + userId, + conversationId, + messageId: parentMessageId, + claimId: claim.claim.claimId, + kind: 'manual', + generationId, + onlyIfUnreconciled: true, + }), + ).toBe(false); + expect( + (await methods.getAgentTriggerDelivery(getAgentTriggerIdempotencyKey(root)))?.status, + ).toBe('succeeded'); + }, +); + +it.each(['admitted', 'unpublished'] as const)( + 'uses native recovery proof after custom-store %s job cleanup', + async (state) => { + const root = await ready('one'); + const sibling = await ready('two'); + const key = getAgentTriggerIdempotencyKey(root); + await methods.claimAgentBackgroundToolResultBatch({ + ...owner(key), + limit: 8, + maxMetadataChars: 16000, + }); + const store = new InMemoryJobStore({ ttlAfterComplete: 0 }); + // Historical lookup is optional; the native claim/fence CAS still exists. + Reflect.set(store, 'getIdempotencyClaim', undefined); + const manager = new GenerationJobManagerClass(); + manager.configure({ + jobStore: store, + eventTransport: new InMemoryEventTransport(), + isRedis: false, + }); + manager.initialize(); + try { + const claimed = await manager.claimGeneration(userId, key, conversationId, conversationId, 2); + if (state === 'admitted') { + const job = await manager.createJob(conversationId, userId, conversationId, { + idempotencyClientRequestId: key, + idempotencyClaimToken: claimed.existing!.claimToken, + initialMetadata: { generationProtocolVersion: 2 }, + }); + expect(await store.deleteJob(conversationId, job.createdAt)).toBe(true); + } + expect( + await manager.getGenerationAdmissionEvidence(userId, key, conversationId, conversationId), + ).toBeNull(); + await mongoose.models.AgentTriggerDelivery.updateOne( + { deliveryKey: key }, + { $set: { capabilityStatus: 'dead' } }, + ); + const release = jest.fn(methods.releaseAgentBackgroundToolResultClaims); + const recover = createBackgroundToolDeadClaimRecovery( + async (deliveryKey, sourceId, reason, options) => + methods.retireAgentTriggerDelivery({ + deliveryKey, + sourceId, + reason, + settledAt: new Date(), + ...options, + }), + methods.releaseBackgroundToolResultClaims, + async () => null, + ({ userId, conversationId, claimId }) => + manager.fenceGenerationClaimForRecovery(userId, claimId, conversationId, conversationId), + release, + methods, + (...args) => manager.getGenerationAdmissionEvidence(...args), + ); + expect( + await recover({ userId, conversationId, messageId: parentMessageId, claimId: key }), + ).toBe(state === 'unpublished'); + if (state === 'admitted') { + expect(release).not.toHaveBeenCalled(); + expect( + await methods.claimAgentBackgroundToolResultBatch({ + ...owner(getAgentTriggerIdempotencyKey(sibling)), + limit: 8, + maxMetadataChars: 16000, + }), + ).toMatchObject({ status: 'claimed', ownerStatus: 'applied' }); + const batch = await methods.getAgentBackgroundToolResultBatch(owner(key)); + expect(batch?.proofCopiedAt).toBeInstanceOf(Date); + } else { + expect(release).toHaveBeenCalledWith(expect.objectContaining({ recoveryFenced: true })); + expect( + ( + await methods.claimAgentBackgroundToolResultBatch({ + ...owner(getAgentTriggerIdempotencyKey(sibling)), + limit: 8, + maxMetadataChars: 16000, + }) + ).status, + ).toBe('acquired'); + } + } finally { + await manager.destroy(); + } + }, +); + +it.each(['lookup', 'confirmation', 'confirmation-unknown'] as const)( + 'preserves a committed manual replay when %s is unavailable', + async (phase) => { + const root = await ready('one'); + await project('one'); + const key = getAgentTriggerIdempotencyKey(root); + await mongoose.models.AgentTriggerDelivery.updateOne( + { deliveryKey: key }, + { $set: { capabilityStatus: 'leased', capabilityClaimToken: 'automatic-lease' } }, + ); + const input = { + userId, + conversationId, + taskId: 'one', + kind: 'manual' as const, + claimId: 'committed-poll', + generationId: 'manual-generation', + }; + const delivered = await claimBackgroundToolResult( + methods, + methods.getAgentBackgroundToolResultClaim, + input, + ); + expect(delivered).toMatchObject({ status: 'acquired' }); + let restore: () => void = () => undefined; + try { + if (phase === 'lookup') { + await expect( + claimBackgroundToolResult( + methods, + async () => { + throw new Error('receipt lookup unavailable'); + }, + input, + ), + ).rejects.toThrow(); + } else if (phase === 'confirmation') { + const failure = jest + .spyOn(mongoose.models.Message, 'updateOne') + .mockImplementationOnce(() => { + throw new Error('confirmation write unavailable'); + }); + restore = () => { + failure.mockRestore(); + }; + // Exact committed read-back succeeds even though this replay's write fails. + await expect( + claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, { + ...input, + generationId: undefined, + }), + ).resolves.toEqual(delivered); + } else { + const failure = jest + .spyOn(methods, 'confirmBackgroundToolResultClaim') + .mockRejectedValueOnce(new Error('confirmation outcome unavailable')); + restore = () => { + failure.mockRestore(); + }; + await expect( + claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, input), + ).rejects.toThrow('confirmation outcome unavailable'); + } + } finally { + restore(); + } + const competitor = await methods.claimBackgroundToolResults({ + userId, + conversationId, + messageId: parentMessageId, + taskId: 'one', + kind: 'wakeup', + claimId: 'automatic-probe', + }); + expect(competitor).toMatchObject({ + status: 'claimed', + claim: { claimId: input.claimId, generationId: 'manual-generation', receiptReconciled: true }, + }); + expect( + await methods.releaseBackgroundToolResultClaims({ + userId, + conversationId, + messageId: parentMessageId, + taskIds: ['one'], + kind: 'manual', + claimId: input.claimId, + onlyIfUnreconciled: true, + }), + ).toBe(false); + if (root.mode !== 'continue') throw new Error('Expected continuation'); + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods, + getGenerationJob: async () => null, + }); + await expect( + resolve(root, { + idempotencyKey: key, + requiredWorkerCapability: capability, + deliveryClaimToken: 'automatic-lease', + }), + ).resolves.toEqual({ status: 'settled' }); + }, +); + +it('repairs native started-fence confirmation from an applied follower after a retired owner crashed', async () => { + const root = await ready('one'); + const sibling = await ready('two'); + const key = getAgentTriggerIdempotencyKey(root); + const claimed = await methods.claimAgentBackgroundToolResultBatch({ + ...owner(key), + limit: 8, + maxMetadataChars: 16000, + }); + if (claimed.status !== 'acquired') throw new Error('Expected frozen batch'); + await mongoose.models.AgentTriggerDelivery.updateOne( + { deliveryKey: key }, + { $set: { capabilityStatus: 'dead' } }, + ); + const recover = createBackgroundToolDeadClaimRecovery( + async (deliveryKey, sourceId, reason, options) => + methods.retireAgentTriggerDelivery({ + deliveryKey, + sourceId, + reason, + settledAt: new Date(), + ...options, + }), + methods.releaseBackgroundToolResultClaims, + async () => null, + async () => 'started', + methods.releaseAgentBackgroundToolResultClaims, + methods, + async () => null, + ); + const Delivery = mongoose.models.AgentTriggerDelivery; + const copyFailure = jest.spyOn(Delivery, 'updateMany').mockImplementationOnce(() => { + throw new Error('proof copy lost'); + }); + await expect( + recover({ userId, conversationId, messageId: parentMessageId, claimId: key }), + ).rejects.toThrow('proof copy lost'); + copyFailure.mockRestore(); + expect(await Delivery.findOne({ deliveryKey: key }).lean()).toMatchObject({ + status: 'succeeded', + }); + expect(await Delivery.findOne({ deliveryKey: key }).lean()).not.toHaveProperty('expiresAt'); + if (sibling.mode !== 'continue') throw new Error('Expected sibling'); + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods, + getGenerationJob: async () => null, + }); + await expect( + resolve(sibling, { + idempotencyKey: getAgentTriggerIdempotencyKey(sibling), + requiredWorkerCapability: capability, + }), + ).resolves.toEqual({ status: 'settled' }); + expect( + (await methods.getAgentBackgroundToolResultBatch(owner(key)))?.proofCopiedAt, + ).toBeInstanceOf(Date); + expect(await Delivery.findOne({ deliveryKey: key }).lean()).toHaveProperty('expiresAt'); + expect( + await methods.getAgentBackgroundToolResultClaim({ ...owner(key), taskId: 'one' }), + ).toMatchObject({ appliedAt: expect.any(Date) }); + expect( + await methods.getAgentBackgroundToolResultClaim({ ...owner(key), taskId: 'two' }), + ).toMatchObject({ appliedAt: expect.any(Date) }); +}); + +it.each([ + ['local', false], + ['reconstructed', false], + ['local', true], + ['reconstructed', true], +] as const)( + 'recovers a released physical projection through a %s poll after successor retirement %s', + async (pollPath, replaceOwner) => { + const root = await ready('stale-task'); + const key = getAgentTriggerIdempotencyKey(root); + const scope = owner(key); + await project('stale-task'); + const old = await methods.claimAgentBackgroundToolResultBatch({ + ...scope, + limit: 8, + maxMetadataChars: 16000, + }); + if (old.status !== 'acquired') throw new Error('Expected predecessor batch'); + let pause: () => void = () => undefined; + let resume: () => void = () => undefined; + const paused = new Promise((resolve) => { + pause = resolve; + }); + const barrier = new Promise((resolve) => { + resume = resolve; + }); + const Message = mongoose.models.Message; + const update = Message.findOneAndUpdate.bind(Message); + const lateProjection = jest + .spyOn(Message, 'findOneAndUpdate') + .mockImplementationOnce((...args) => { + const query = update(...args); + const execute = query.exec.bind(query); + jest.spyOn(query, 'exec').mockImplementationOnce(async () => { + pause(); + await barrier; + return execute(); + }); + return query; + }); + const delayed = methods.claimBackgroundToolResults({ + userId, + conversationId, + messageId: parentMessageId, + taskId: 'stale-task', + kind: 'wakeup', + claimId: key, + batchId: old.batchId, + limit: 1, + }); + await paused; + expect( + await methods.releaseAgentBackgroundToolResultClaims({ + ...scope, + claimId: key, + batchId: old.batchId, + }), + ).toBe(true); + if (replaceOwner) { + const successor = await methods.claimAgentBackgroundToolResultBatch({ + ...scope, + limit: 8, + maxMetadataChars: 16000, + }); + if (successor.status !== 'acquired') throw new Error('Expected successor batch'); + expect(successor.batchId).not.toBe(old.batchId); + expect( + await methods.releaseAgentBackgroundToolResultClaims({ + ...scope, + claimId: key, + batchId: successor.batchId, + }), + ).toBe(true); + } + expect( + await methods.retireAgentTriggerDelivery({ + deliveryKey: key, + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + reason: 'unadmitted owner retired', + settledAt: new Date(), + onlyIfUnclaimed: true, + }), + ).toBe(true); + resume(); + await expect(delayed).resolves.toMatchObject({ status: 'acquired' }); + lateProjection.mockRestore(); + expect(await methods.getAgentBackgroundToolResultBatch(scope)).toBeNull(); + + // A broad logical-owner release would also erase these unrelated claims. + await project('other-epoch'); + await project('manual-task'); + await methods.claimBackgroundToolResults({ + userId, + conversationId, + messageId: parentMessageId, + taskId: 'other-epoch', + kind: 'wakeup', + claimId: key, + batchId: 'unrelated-epoch', + limit: 1, + }); + await claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, { + userId, + conversationId, + messageId: parentMessageId, + taskId: 'manual-task', + kind: 'manual', + claimId: key, + }); + if (pollPath === 'local') { + const created = backgroundTaskRegistry.create({ + taskId: 'stale-task', + userId, + conversationId, + messageId: parentMessageId, + toolCallId: 'stale-task', + toolName: 'tool', + }); + if ('atCapacity' in created) throw new Error('Unexpected capacity'); + backgroundTaskRegistry.complete(userId, conversationId, created.task.id, { + content: 'durable-stale-task', + }); + backgroundTaskRegistry.markCompletionWakeup(userId, conversationId, created.task.id); + } + const store = new InMemoryJobStore({ ttlAfterComplete: 0 }); + const manager = new GenerationJobManagerClass(); + manager.configure({ + jobStore: store, + eventTransport: new InMemoryEventTransport(), + isRedis: false, + }); + manager.initialize(); + try { + const release = jest.fn(methods.releaseBackgroundToolResultClaims); + const recover = createBackgroundToolDeadClaimRecovery( + async (deliveryKey, sourceId, reason, options) => + methods.retireAgentTriggerDelivery({ + deliveryKey, + sourceId, + reason, + settledAt: new Date(), + ...options, + }), + release, + async () => null, + ({ userId, conversationId, claimId }) => + manager.fenceGenerationClaimForRecovery(userId, claimId, conversationId, conversationId), + methods.releaseAgentBackgroundToolResultClaims, + methods, + (...args) => manager.getGenerationAdmissionEvidence(...args), + ); + const recoverDeadBackgroundToolClaim = jest.fn(recover); + const polled = JSON.parse( + await runCheckBackgroundTask({ + userId, + conversationId, + args: { background_task_id: 'stale-task' }, + toolCallId: 'stale-poll', + runId: 'recovery-run', + generationId: 'recovery-generation', + claimBackgroundToolResult: (input) => + claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, input), + recoverDeadBackgroundToolClaim, + }), + ); + expect(polled).toMatchObject({ status: 'completed', result: 'durable-stale-task' }); + expect(recoverDeadBackgroundToolClaim).toHaveBeenCalledWith( + expect.objectContaining({ claimId: key, batchId: old.batchId }), + ); + expect(release).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'wakeup', claimId: key, batchId: old.batchId }), + ); + for (const [taskId, claim] of [ + ['other-epoch', { kind: 'wakeup', claimId: key, batchId: 'unrelated-epoch' }], + ['manual-task', { kind: 'manual', claimId: key, receiptReconciled: true }], + ] as const) { + expect( + await methods.claimBackgroundToolResults({ + userId, + conversationId, + messageId: parentMessageId, + taskId, + kind: 'wakeup', + claimId: 'probe', + }), + ).toMatchObject({ status: 'claimed', claim }); + } + } finally { + await manager.destroy(); + } + }, +); + +it('does not release a stale epoch while a successor receipt batch owns the logical delivery', async () => { + const root = await ready('stale-task'); + const key = getAgentTriggerIdempotencyKey(root); + const scope = owner(key); + await project('stale-task'); + const old = await methods.claimAgentBackgroundToolResultBatch({ + ...scope, + limit: 8, + maxMetadataChars: 16000, + }); + if (old.status !== 'acquired') throw new Error('Expected predecessor'); + await methods.releaseAgentBackgroundToolResultClaims({ + ...scope, + claimId: key, + batchId: old.batchId, + }); + const successor = await methods.claimAgentBackgroundToolResultBatch({ + ...scope, + limit: 8, + maxMetadataChars: 16000, + }); + if (successor.status !== 'acquired') throw new Error('Expected successor'); + await methods.claimBackgroundToolResults({ + userId, + conversationId, + messageId: parentMessageId, + taskId: 'stale-task', + kind: 'wakeup', + claimId: key, + batchId: successor.batchId, + limit: 1, + }); + const release = jest.fn(methods.releaseBackgroundToolResultClaims); + const retire = jest.fn(async () => true); + const fence = jest.fn(async () => 'fenced' as const); + const recover = createBackgroundToolDeadClaimRecovery( + retire, + release, + async () => null, + fence, + methods.releaseAgentBackgroundToolResultClaims, + methods, + ); + expect( + await recover({ + userId, + conversationId, + messageId: parentMessageId, + claimId: key, + batchId: old.batchId, + }), + ).toBe(false); + expect(retire).not.toHaveBeenCalled(); + expect(fence).not.toHaveBeenCalled(); + expect(release).not.toHaveBeenCalled(); + expect( + await methods.getAgentBackgroundToolResultClaim({ ...scope, taskId: 'stale-task' }), + ).toMatchObject({ batchId: successor.batchId }); + expect( + await methods.claimBackgroundToolResults({ + userId, + conversationId, + messageId: parentMessageId, + taskId: 'stale-task', + kind: 'manual', + claimId: 'probe', + }), + ).toMatchObject({ status: 'claimed', claim: { claimId: key, batchId: successor.batchId } }); +}); diff --git a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts index 3d8c9460a1..f599dad7a6 100644 --- a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts +++ b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts @@ -1,4 +1,7 @@ -import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 } from '@librechat/data-schemas'; +import { + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3, + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, +} from '@librechat/data-schemas'; import type { AgentTriggerProducerLeaseStatus } from '@librechat/data-schemas'; import type { CodeApprovalMode } from 'librechat-data-provider'; import type { EnqueueBackgroundToolCompletion } from './backgroundCompletionWakeup'; @@ -113,6 +116,37 @@ describe('background tool completion wakeups', () => { jest.useRealTimers(); }); + it('admits v3 receipts only when the deployment rollout gate is enabled', async () => { + const enqueue = jest.fn< + ReturnType, + Parameters + >(async () => ({ deliveryKey: 'key' })); + let enabled = false; + const notify = createBackgroundToolCompletionWakeupHandler( + enqueue, + async () => true, + async () => true, + undefined, + undefined, + () => enabled, + ); + await notify(registration()); + expect(enqueue).toHaveBeenLastCalledWith( + expect.anything(), + expect.objectContaining({ + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + }), + ); + enabled = true; + await notify(registration({ taskId: 'task-2' })); + expect(enqueue).toHaveBeenLastCalledWith( + expect.anything(), + expect.objectContaining({ + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3, + }), + ); + }); + it('expedites its own delivery when a result it can consume appears', async () => { const expedite = jest.fn(); const notify = createBackgroundToolCompletionWakeupHandler( @@ -265,6 +299,7 @@ describe('background tool completion wakeups', () => { userId: 'user-1', conversationId: 'conversation-1', messageId: 'response-1', + allowMissingMessage: true, kind: 'wakeup', claimId: 'batch-root-delivery', }); @@ -274,6 +309,7 @@ describe('background tool completion wakeups', () => { conversationId: 'conversation-1', parentMessageId: 'response-1', claimId: 'batch-root-delivery', + recoveryFenced: true, }); }); @@ -1034,3 +1070,204 @@ describe('pending background completions', () => { } }); }); + +describe('capability-gated receipt batch resolution', () => { + function batchMethods() { + const { methods } = resolverMethods(); + return { + ...methods, + claimAgentBackgroundToolResultBatch: jest.fn(async () => ({ + status: 'acquired' as const, + batchId: 'batch-1', + results: [ + { + taskId: 'task-1', + toolCallId: 'call-1', + toolName: 'tool', + status: 'completed' as const, + output: 'one', + }, + { + taskId: 'task-2', + toolCallId: 'call-2', + toolName: 'tool', + status: 'completed' as const, + output: 'two', + }, + ], + })), + beginAgentBackgroundToolResultBatchDispatch: jest.fn(async () => true), + confirmAgentBackgroundToolResultBatch: jest.fn(async () => true), + }; + } + + it('does not probe batch storage for rollout-default v2 deliveries', async () => { + const methods = batchMethods(); + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods: methods as never, + getGenerationJob: async () => null, + }); + await resolve(await envelope(), { + idempotencyKey: 'delivery-1', + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + }); + expect(methods.claimAgentBackgroundToolResultBatch).not.toHaveBeenCalled(); + }); + + it('reconciles every member before dispatch and confirms only after admission', async () => { + const methods = batchMethods(); + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods: methods as never, + getGenerationJob: async () => null, + }); + const prepared = await resolve(await envelope(), { idempotencyKey: 'delivery-1' }); + expect(methods.claimBackgroundToolResults).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ taskId: 'task-1' }), + ); + expect(methods.claimBackgroundToolResults).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ taskId: 'task-2' }), + ); + expect(methods.confirmAgentBackgroundToolResultBatch).not.toHaveBeenCalled(); + expect(methods.claimAgentBackgroundToolResultBatch).toHaveBeenCalledWith( + expect.objectContaining({ + tenantId: 'tenant-1', + limit: 8, + maxMetadataChars: BACKGROUND_TOOL_WAKEUP_INPUT_MAX_CHARS - 256, + }), + ); + if (prepared?.status !== 'ready') throw new Error('Expected ready'); + expect(prepared.input).toContain('task-2'); + await prepared.settleOnAdmission?.({ + mode: 'continue', + status: 'started', + conversationId: 'conversation-1', + }); + expect(methods.confirmAgentBackgroundToolResultBatch).toHaveBeenCalledTimes(1); + }); + + it('defers followers until the root has confirmed admission', async () => { + const methods = batchMethods(); + methods.claimAgentBackgroundToolResultBatch + .mockResolvedValueOnce({ + status: 'claimed', + claimId: 'root', + ownerStatus: 'pending', + } as never) + .mockResolvedValueOnce({ + status: 'claimed', + claimId: 'root', + ownerStatus: 'applied', + } as never); + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods: methods as never, + getGenerationJob: async () => null, + }); + await expect(resolve(await envelope(), { idempotencyKey: 'follower' })).rejects.toMatchObject({ + code: 'BACKGROUND_TOOL_BATCH_PENDING', + deferWithoutAttempt: true, + }); + await expect(resolve(await envelope(), { idempotencyKey: 'follower' })).resolves.toEqual({ + status: 'settled', + }); + expect(methods.claimBackgroundToolResults).not.toHaveBeenCalled(); + }); + + it('uses fenced dead-owner recovery before retrying a follower as its own root', async () => { + const methods = batchMethods(); + const recoverDeadClaim = jest.fn(async () => true); + methods.claimAgentBackgroundToolResultBatch.mockResolvedValueOnce({ + status: 'claimed', + claimId: 'root', + ownerStatus: 'recoverable', + } as never); + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods: methods as never, + getGenerationJob: async () => null, + recoverDeadClaim, + }); + await expect(resolve(await envelope(), { idempotencyKey: 'follower' })).rejects.toMatchObject({ + code: 'BACKGROUND_TOOL_BATCH_PENDING', + }); + expect(recoverDeadClaim).toHaveBeenCalledWith( + expect.objectContaining({ claimId: 'root', messageId: 'response-1' }), + ); + expect((await resolve(await envelope(), { idempotencyKey: 'follower' }))?.status).toBe('ready'); + }); + + it('releases the entire frozen batch when a sibling manual poll wins', async () => { + const methods = batchMethods(); + methods.claimBackgroundToolResults + .mockResolvedValueOnce({ status: 'acquired', results: [] }) + .mockResolvedValueOnce({ + status: 'claimed', + claim: { kind: 'manual', claimId: 'poll' }, + } as never); + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods: methods as never, + getGenerationJob: async () => null, + }); + await expect(resolve(await envelope(), { idempotencyKey: 'root' })).rejects.toMatchObject({ + code: 'BACKGROUND_TOOL_CLAIM_RECONCILING', + }); + expect(methods.releaseAgentBackgroundToolResultClaims).toHaveBeenCalledWith( + expect.objectContaining({ claimId: 'root' }), + ); + expect(methods.confirmAgentBackgroundToolResultBatch).not.toHaveBeenCalled(); + }); + + it('releases receipt ownership even when no message projection has landed', async () => { + const methods = batchMethods(); + methods.claimBackgroundToolResults.mockResolvedValue({ status: 'not_found' } as never); + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods: methods as never, + getGenerationJob: async () => null, + }); + const prepared = await resolve(await envelope(), { idempotencyKey: 'root' }); + if (prepared?.status !== 'ready') throw new Error('Expected ready'); + await prepared.releaseOnDefiniteFailure?.(); + expect(methods.releaseBackgroundToolResultClaims).not.toHaveBeenCalled(); + expect(methods.releaseAgentBackgroundToolResultClaims).toHaveBeenCalled(); + }); + + it('retains ownership when admission confirmation fails after dispatch', async () => { + const methods = batchMethods(); + methods.confirmAgentBackgroundToolResultBatch.mockRejectedValueOnce(new Error('lost reply')); + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods: methods as never, + getGenerationJob: async () => null, + }); + const prepared = await resolve(await envelope(), { idempotencyKey: 'root' }); + if (prepared?.status !== 'ready') throw new Error('Expected ready'); + await expect( + prepared.settleOnAdmission?.({ + mode: 'continue', + status: 'started', + conversationId: 'conversation-1', + }), + ).rejects.toThrow('lost reply'); + expect(methods.releaseAgentBackgroundToolResultClaims).not.toHaveBeenCalled(); + }); + + it('still detects a lost producer while a v3 receipt is not ready', async () => { + const methods = batchMethods(); + methods.claimBackgroundToolResults.mockResolvedValueOnce({ status: 'not_ready', results: [] }); + methods.claimAgentBackgroundToolResultBatch.mockResolvedValueOnce({ + status: 'not_ready', + waitingForResult: true, + } as never); + methods.getAgentTriggerDeliveryProducerLease.mockResolvedValueOnce({ + status: 'expired', + leaseUntil: new Date(0), + }); + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods: methods as never, + getGenerationJob: async () => null, + }); + await expect(resolve(await envelope(), { idempotencyKey: 'root' })).rejects.toMatchObject({ + code: 'BACKGROUND_TOOL_PRODUCER_LOST', + retryable: false, + }); + }); +}); diff --git a/packages/api/src/agents/backgroundCompletionWakeup.ts b/packages/api/src/agents/backgroundCompletionWakeup.ts index e3df97eb04..56a057f95d 100644 --- a/packages/api/src/agents/backgroundCompletionWakeup.ts +++ b/packages/api/src/agents/backgroundCompletionWakeup.ts @@ -1,6 +1,10 @@ import { randomUUID } from 'node:crypto'; import { backgroundResultMetadata, isEphemeralAgentId } from 'librechat-data-provider'; -import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 } from '@librechat/data-schemas'; +import { + AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS, + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3, + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, +} from '@librechat/data-schemas'; import type { AgentTriggerProducerLeaseStatus, AgentTriggerDeliveryMethods, @@ -80,6 +84,10 @@ type WakeupMethods = Pick & output: string; settledAt: Date; } | null>; + persistAgentBackgroundToolResult?: AgentTriggerDeliveryMethods['persistAgentBackgroundToolResult']; + beginAgentBackgroundToolResultBatchDispatch?: AgentTriggerDeliveryMethods['beginAgentBackgroundToolResultBatchDispatch']; + claimAgentBackgroundToolResultBatch?: AgentTriggerDeliveryMethods['claimAgentBackgroundToolResultBatch']; + confirmAgentBackgroundToolResultBatch?: AgentTriggerDeliveryMethods['confirmAgentBackgroundToolResultBatch']; claimAgentBackgroundToolResults?: AgentTriggerDeliveryMethods['claimAgentBackgroundToolResults']; getAgentBackgroundToolResultClaim?: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']; releaseAgentBackgroundToolResultClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims']; @@ -98,6 +106,13 @@ export interface BackgroundToolCompletionWakeupResolverDeps { methods: WakeupMethods; getGenerationJob: (conversationId: string) => Promise; getResultBatchSize?: () => number | undefined; + recoverDeadClaim?: BackgroundToolDeadClaimRecovery; + getGenerationAdmissionEvidence?: ( + userId: string, + clientRequestId: string, + streamId: string, + conversationId?: string, + ) => Promise<{ generationId: string; generationCreatedAt: number } | null>; /** Longest a waiting delivery re-checks readiness; the backoff default otherwise. */ getWaitMaxIntervalMs?: () => number | undefined; } @@ -265,12 +280,14 @@ export function createBackgroundToolCompletionWakeupResolver({ getGenerationJob, getResultBatchSize, getWaitMaxIntervalMs, + recoverDeadClaim, + getGenerationAdmissionEvidence, }: BackgroundToolCompletionWakeupResolverDeps): NonNullable< AgentTriggerExecutionHostDeps['prepareContinue'] > { const waitingRetry = (receivedAt: number): string => waitingRetryAfter(receivedAt, Date.now(), getWaitMaxIntervalMs?.() ?? WAITING_RETRY_CAP_MS); - return async ( + const resolve = async ( envelope: AgentContinueTriggerEnvelope, context: AgentTriggerDispatchContext, ): Promise => { @@ -332,6 +349,313 @@ export function createBackgroundToolCompletionWakeupResolver({ status: 404, }); } + const batchScope = { + deliveryKey: context.idempotencyKey, + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + userId, + ...(envelope.principal.tenantId != null && { tenantId: envelope.principal.tenantId }), + conversationId: envelope.target.conversationId, + parentMessageId: envelope.target.parentMessageId, + agentId: envelope.target.agentId, + ...(context.deliveryClaimToken != null && { deliveryClaimToken: context.deliveryClaimToken }), + }; + const batch = + context.requiredWorkerCapability != null && + context.requiredWorkerCapability !== + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3 + ? undefined + : await methods.claimAgentBackgroundToolResultBatch?.({ + ...batchScope, + limit: getResultBatchSize?.() ?? 8, + maxMetadataChars: BACKGROUND_TOOL_WAKEUP_INPUT_MAX_CHARS - 256, + }); + if (batch?.status === 'claimed') { + if (batch.ownerStatus === 'applied') return { status: 'settled' }; + if (batch.ownerStatus === 'recoverable') { + await recoverDeadClaim?.({ + userId, + conversationId: envelope.target.conversationId, + messageId: envelope.target.parentMessageId, + claimId: batch.claimId, + batchId: batch.batchId, + }); + } + throw executionError('The background result is owned by another completion delivery.', { + code: 'BACKGROUND_TOOL_BATCH_PENDING', + retryable: true, + deferWithoutAttempt: true, + retryAfter: '1', + }); + } + if (batch?.status === 'not_ready') { + if (batch.waitingForResult === true) { + // A producer can fail its receipt write while its terminal message + // projection succeeds. Reconstruct from that already-durable result. + const projection = await methods.claimBackgroundToolResults({ + userId, + conversationId: envelope.target.conversationId, + messageId: envelope.target.parentMessageId, + taskId: registration.taskId, + agentId: envelope.target.agentId, + kind: 'wakeup', + claimId: context.idempotencyKey, + limit: 1, + }); + if ( + projection.status === 'claimed' && + projection.claim?.kind === 'manual' && + projection.claim.receiptReconciled === true + ) + return { status: 'settled' }; + if (projection.status === 'acquired') { + const result = projection.results.find((result) => result.taskId === registration.taskId); + const releaseProjection = async (): Promise => { + if ( + !(await methods.releaseBackgroundToolResultClaims({ + userId, + conversationId: envelope.target.conversationId, + messageId: envelope.target.parentMessageId, + taskIds: [registration.taskId], + kind: 'wakeup', + claimId: context.idempotencyKey, + allowMissingMessage: true, + })) + ) { + throw new Error('Background projection reconstruction release was not confirmed'); + } + }; + let restored = false; + try { + if (result != null) + restored = + (await methods.persistAgentBackgroundToolResult?.({ + deliveryKey: context.idempotencyKey, + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + result: { + status: result.status, + output: truncateMiddle( + result.output, + AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS, + ), + settledAt: result.settledAt ?? new Date(envelope.event.occurredAt), + }, + })) === true; + } finally { + await releaseProjection(); + } + if (restored) return resolve(envelope, context); + throw executionError('The background result receipt is being reconstructed.', { + code: 'BACKGROUND_TOOL_CLAIM_RECONCILING', + retryable: true, + deferWithoutAttempt: true, + retryAfter: '1', + }); + } + if ( + projection.status === 'claimed' && + projection.claim?.kind === 'manual' && + projection.claim.generationId != null + ) { + await recoverDeadClaim?.({ + userId, + conversationId: envelope.target.conversationId, + messageId: envelope.target.parentMessageId, + claimId: projection.claim.claimId, + kind: 'manual', + generationId: projection.claim.generationId, + onlyIfUnreconciled: true, + }); + } + if (projection.status === 'claimed') + throw executionError('The background projection is being reconciled.', { + code: 'BACKGROUND_TOOL_CLAIM_RECONCILING', + retryable: true, + deferWithoutAttempt: true, + retryAfter: '1', + }); + + const producer = await methods.getAgentTriggerDeliveryProducerLease({ + deliveryKey: context.idempotencyKey, + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + now: new Date(), + }); + if (producer.status === 'expired') { + throw executionError('The process-local background tool executor was lost.', { + code: 'BACKGROUND_TOOL_PRODUCER_LOST', + retryable: false, + }); + } + } + throw executionError('The background result batch is not durable yet.', { + code: 'BACKGROUND_TOOL_RESULT_NOT_READY', + retryable: true, + deferWithoutAttempt: true, + retryAfter: waitingRetry(envelope.receivedAt), + }); + } + if (batch?.status === 'acquired') { + const owned = { ...batchScope, batchId: batch.batchId }; + const dispatchId = randomUUID(); + let dispatchBegan = false; + const release = async (): Promise => { + const admitted = await getGenerationAdmissionEvidence?.( + userId, + context.idempotencyKey, + envelope.target.conversationId, + envelope.target.conversationId, + ); + if (admitted != null) { + await methods.confirmAgentBackgroundToolResultBatch?.(owned); + return false; + } + // Absence is not proof. The storage CAS permits only a first definite + // failure or a never-dispatched plan, never a retry of an ambiguous POST. + const released = await methods.releaseAgentBackgroundToolResultClaims?.({ + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + userId, + conversationId: envelope.target.conversationId, + parentMessageId: envelope.target.parentMessageId, + claimId: context.idempotencyKey, + batchId: batch.batchId, + ...(dispatchBegan && { dispatchId }), + ...(context.deliveryClaimToken != null && { + deliveryClaimToken: context.deliveryClaimToken, + }), + }); + return released === true; + }; + // A receipt may precede its projection. Reconcile every member, not just + // the root, against a concurrent manual poll or late message persistence. + const projections = await Promise.all( + batch.results.map((result) => + methods.claimBackgroundToolResults({ + userId, + conversationId: envelope.target.conversationId, + messageId: envelope.target.parentMessageId, + taskId: result.taskId, + agentId: envelope.target.agentId, + kind: 'wakeup', + claimId: context.idempotencyKey, + batchId: batch.batchId, + limit: 1, + }), + ), + ); + if ( + projections.some( + (projection) => + projection.status === 'claimed' || projection.status === 'outcome_unknown', + ) + ) { + for (let index = 0; index < projections.length; index++) { + const stale = projections[index]; + if (stale?.status !== 'claimed' || stale.claim?.kind !== 'wakeup') continue; + const taskId = batch.results[index]!.taskId; + const receipt = await methods.getAgentBackgroundToolResultClaim?.({ + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + userId, + conversationId: envelope.target.conversationId, + parentMessageId: envelope.target.parentMessageId, + taskId, + }); + if (receipt?.claimId !== context.idempotencyKey || receipt.batchId !== batch.batchId) + continue; + await methods.releaseBackgroundToolResultClaims({ + userId, + conversationId: envelope.target.conversationId, + messageId: envelope.target.parentMessageId, + taskIds: [taskId], + kind: 'wakeup', + claimId: stale.claim.claimId, + ...(stale.claim.batchId != null && { batchId: stale.claim.batchId }), + allowMissingMessage: true, + }); + throw executionError('A stale background projection is being reconciled.', { + code: 'BACKGROUND_TOOL_CLAIM_RECONCILING', + retryable: true, + deferWithoutAttempt: true, + retryAfter: '1', + }); + } + const released = await release(); + const rootProjection = + projections[batch.results.findIndex((result) => result.taskId === registration.taskId)]; + if ( + released && + rootProjection?.status === 'claimed' && + rootProjection.claim?.kind === 'manual' && + rootProjection.claim.receiptReconciled === true + ) { + return { status: 'settled' }; + } + if (released && recoverDeadClaim != null) { + await Promise.all( + projections.map(async (projection) => { + if ( + projection.status !== 'claimed' || + projection.claim?.kind !== 'manual' || + projection.claim.receiptReconciled === true || + projection.claim.generationId == null + ) + return; + await recoverDeadClaim({ + userId, + conversationId: envelope.target.conversationId, + messageId: envelope.target.parentMessageId, + claimId: projection.claim.claimId, + kind: 'manual', + generationId: projection.claim.generationId, + onlyIfUnreconciled: true, + }); + }), + ); + } + throw executionError('Background result ownership is being reconciled.', { + code: 'BACKGROUND_TOOL_CLAIM_RECONCILING', + retryable: true, + deferWithoutAttempt: true, + retryAfter: '1', + }); + } + return { + status: 'ready', + parentMessageId, + ...(parent.codeApprovalMode != null && { codeApprovalMode: parent.codeApprovalMode }), + input: buildWakeupInput(batch.results), + releaseOnDefiniteFailure: async () => { + await release(); + }, + beginDispatch: async () => { + dispatchBegan = true; + if ( + !(await methods.beginAgentBackgroundToolResultBatchDispatch?.({ ...owned, dispatchId })) + ) { + throw executionError('The background batch preparation was superseded.', { + code: 'BACKGROUND_TOOL_BATCH_SUPERSEDED', + retryable: true, + deferWithoutAttempt: true, + }); + } + }, + settleOnAdmission: async (result) => { + if ( + result.status === 'settled' && + getGenerationAdmissionEvidence != null && + (await getGenerationAdmissionEvidence( + userId, + context.idempotencyKey, + envelope.target.conversationId, + envelope.target.conversationId, + )) == null + ) { + throw new Error('A recovery fence does not prove background admission'); + } + if (!(await methods.confirmAgentBackgroundToolResultBatch?.(owned))) { + throw new Error('Background result batch admission was not confirmed'); + } + }, + }; + } const claim = await methods.claimBackgroundToolResults({ userId, conversationId: envelope.target.conversationId, @@ -545,6 +869,7 @@ export function createBackgroundToolCompletionWakeupResolver({ deferWithoutAttempt: true, }); }; + return resolve; } /** Lists and discards a conversation's undelivered background completions from the @@ -652,6 +977,7 @@ export function createBackgroundToolCompletionWakeupHandler( renewProducerLease: RenewBackgroundToolCompletionProducerLease, persistResult?: PersistBackgroundToolCompletionResult, expedite?: (deliveryKey: string) => void, + getReceiptBatchingEnabled?: () => boolean, ): ( registration: BackgroundToolWakeupRegistration, ) => Promise { @@ -695,7 +1021,10 @@ export function createBackgroundToolCompletionWakeupHandler( availableAt: new Date( Math.max(Date.now(), registration.createdAt) + WAKEUP_ADMISSION_DELAY_MS, ), - requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + requiredWorkerCapability: + getReceiptBatchingEnabled?.() === true + ? AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3 + : AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, producerLeaseUntil: new Date(Date.now() + BACKGROUND_TOOL_PRODUCER_LEASE_MS), }); return { @@ -735,8 +1064,22 @@ export function createBackgroundToolDeadClaimRecovery( claimId: string; }) => Promise<'fenced' | 'started' | 'unavailable'>, releaseReceiptClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'], + batchRecovery?: Pick< + AgentTriggerDeliveryMethods, + 'getAgentBackgroundToolResultBatch' | 'confirmAgentBackgroundToolResultBatch' + >, + getAdmissionEvidence?: BackgroundToolCompletionWakeupResolverDeps['getGenerationAdmissionEvidence'], ): BackgroundToolDeadClaimRecovery { - return async ({ userId, conversationId, messageId, claimId, kind, generationId }) => { + return async ({ + userId, + conversationId, + messageId, + claimId, + kind, + generationId, + batchId, + onlyIfUnreconciled, + }) => { if (kind === 'manual') { if (generationId == null || generationId.length === 0) { return false; @@ -753,8 +1096,41 @@ export function createBackgroundToolDeadClaimRecovery( messageId, kind: 'manual', claimId, + ...(onlyIfUnreconciled === true && { onlyIfUnreconciled }), }); } + const owner = { + deliveryKey: claimId, + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + userId, + conversationId, + parentMessageId: messageId, + }; + const batch = await batchRecovery?.getAgentBackgroundToolResultBatch(owner); + if (batchId != null && batch != null && batch.batchId !== batchId) return false; + if (batch?.appliedAt != null) { + await batchRecovery?.confirmAgentBackgroundToolResultBatch({ + ...owner, + batchId: batch.batchId, + }); + return false; + } + const confirmAdmission = async (): Promise => { + const evidence = await getAdmissionEvidence?.( + userId, + claimId, + conversationId, + conversationId, + ); + if (evidence == null) return false; + if (batch != null) + await batchRecovery?.confirmAgentBackgroundToolResultBatch({ + ...owner, + batchId: batch.batchId, + }); + return true; + }; + if (batch != null && (await confirmAdmission())) return false; const claimGenerationIsActive = async (): Promise => { const generation = await getGenerationJob(conversationId); return ( @@ -778,30 +1154,39 @@ export function createBackgroundToolDeadClaimRecovery( * installs a started tombstone that invalidates a delayed creator's token, * or observes that job creation already won. */ const generationFence = await fenceGenerationClaim({ userId, conversationId, claimId }); - if (generationFence === 'unavailable') { + if (generationFence === 'unavailable') return false; + if (generationFence === 'started' && batch != null) { + // The native claim CAS proves admission even when a custom store cannot + // read historical claims or the live generation has been cleaned up. + await batchRecovery?.confirmAgentBackgroundToolResultBatch({ + ...owner, + batchId: batch.batchId, + }); return false; } if (await claimGenerationIsActive()) { return false; } - const released = await releaseClaims({ + const receiptBatchId = batch?.batchId ?? batchId; + const receiptReleased = await releaseReceiptClaims?.({ + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + userId, + conversationId, + parentMessageId: messageId, + claimId, + ...(receiptBatchId != null && { batchId: receiptBatchId }), + recoveryFenced: true, + }); + if (receiptReleased === false) return false; + if (batch != null) return receiptReleased === true; + return releaseClaims({ userId, conversationId, messageId, + allowMissingMessage: true, kind: 'wakeup', claimId, + ...(receiptBatchId != null && { batchId: receiptBatchId }), }); - if (!released) { - return false; - } - return ( - (await releaseReceiptClaims?.({ - sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, - userId, - conversationId, - parentMessageId: messageId, - claimId, - })) ?? true - ); }; } diff --git a/packages/api/src/agents/handlers.background.spec.ts b/packages/api/src/agents/handlers.background.spec.ts index 6eac3eedb1..ce75ddac9f 100644 --- a/packages/api/src/agents/handlers.background.spec.ts +++ b/packages/api/src/agents/handlers.background.spec.ts @@ -457,6 +457,59 @@ describe('createToolExecuteHandler — background tool calls', () => { expect(retire).toHaveBeenCalledWith('background tool result was not persisted', undefined); }); + it('copies committed manual reconciliation into the final background projection retry', async () => { + let finishPersistence: (persisted: boolean) => void = () => undefined; + let resolveProjection: (() => { resultClaim?: { receiptReconciled?: true } }) | undefined; + const tool = makeSearchTool({ calls: 0 }); + const handler = createToolExecuteHandler({ + loadTools: async () => ({ loadedTools: [tool] }), + backgroundToolCompletion: { + preregister: async () => ({ + renew: async () => true, + retire: async () => true, + persistResult: async () => true, + }), + claim: async () => ({ status: 'not_ready' }), + persist: (params) => { + resolveProjection = params.resolveBackgroundTask; + return new Promise((resolve) => { + finishPersistence = resolve; + }); + }, + }, + }); + const [dispatch] = await runBatch(handler, { + toolCalls: [ + { + id: 'manual-marker-dispatch', + stepId: 'manual-marker-step', + name: tool.name, + args: { q: 'marker', run_in_background: true }, + }, + ], + agentId: 'agent_parent_1', + configurable: buildConfig([tool.name]), + metadata: { thread_id: 'manual-marker-convo', run_id: 'manual-marker-parent' }, + }); + await flushMicrotasks(); + const taskId = JSON.parse(dispatch.content).background_task_id as string; + try { + expect(resolveProjection?.().resultClaim).toBeUndefined(); + expect( + backgroundTaskRegistry.claimResult('exec_user', 'manual-marker-convo', taskId, { + kind: 'manual', + claimId: 'manual-marker-poll', + receiptReconciled: true, + generationId: 'manual-marker-generation', + }), + ).toBe('acquired'); + expect(resolveProjection?.().resultClaim).toMatchObject({ receiptReconciled: true }); + } finally { + finishPersistence(true); + await flushMicrotasks(); + } + }); + it('falls back to the settled local result when a poll retired delivery before persistence failed', async () => { let finishPersistence: ((persisted: boolean) => void) | undefined; const persist = jest.fn( diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts index 670296a101..bbb1643d7c 100644 --- a/packages/api/src/agents/handlers.ts +++ b/packages/api/src/agents/handlers.ts @@ -6446,6 +6446,9 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand kind: current.resultClaim.kind, claimId: current.resultClaim.claimId, claimedAt: new Date(current.resultClaim.claimedAt), + ...(current.resultClaim.receiptReconciled === true && { + receiptReconciled: true, + }), ...(current.resultClaim.generationId == null ? {} : { generationId: current.resultClaim.generationId }), diff --git a/packages/api/src/agents/harvest.ts b/packages/api/src/agents/harvest.ts index 63cd918861..38e257df91 100644 --- a/packages/api/src/agents/harvest.ts +++ b/packages/api/src/agents/harvest.ts @@ -51,6 +51,7 @@ export interface BackgroundToolResultState { claimId: string; claimedAt: Date; generationId?: string; + receiptReconciled?: true; }; } diff --git a/packages/api/src/agents/triggers/README.md b/packages/api/src/agents/triggers/README.md index 5fe25e09aa..3d266ce654 100644 --- a/packages/api/src/agents/triggers/README.md +++ b/packages/api/src/agents/triggers/README.md @@ -269,3 +269,75 @@ event whose individual timing or acknowledgment is actionable. `fire`, `steer`, `continue` deliveries reject the option instead of silently weakening their semantics. Deliveries with `expectedAction` also reject coalescing because one generation cannot prove several distinct action fences. + +## Background receipt batches + +Ordinary background tool and code completions use `background_tool_completion_batch_v3`. +Older workers cannot claim these rows; v2 receipts retain task-local delivery. Subagent +completions do not join receipt batches. The existing `completionResultBatchSize` bounds +selection, with no collection window. Scope is owner, tenant, conversation, launching +parent message, and target agent. Results launched on different parent messages remain +separate even when their branches later converge. + +The root persists candidate keys before per-result claims, then freezes only successful +claims in `backgroundToolResultBatch.members`. Retries use that same list. Each member's +message projection is reconciled before dispatch. A first definite rejection releases the unadmitted plan. Once a handoff may have been +admitted, later transport failures retain frozen ownership and the same idempotency key. Admission proof is stored before followers settle, and copied to each +receipt so root retention cannot strand a follower. + +Followers defer without consuming attempts. A dead or retired root is recovered through +the generation-admission fence before claims are released. Losing collectors release +claims they cannot dispatch. Concurrent roots can still split the ready set; batching is +opportunistic, not one-turn-per-conversation election. + +`triggerDelivery.spec.ts` includes a real-Mongo collecting barrier, crash/lost-reply +injection, rolling-upgrade isolation, and an 8-conversation × 4-result storage stress +harness. Its latency measures receipt admission, not model turns or deployment latency. + +Receipt batching defaults off. Deploy compatible receipt and manual-poll consumers on +**every replica**, then enable: + +```yaml +endpoints: + agents: + backgroundTasks: + completionReceiptBatching: true +``` + +Older manual-poll workers cannot read v3 ownership. Queue capability fencing alone does +not protect that path. Disabling batching stops new v3 production, not existing +ownership. Do not downgrade poll consumers while v3 receipts remain, including delivered +receipts. A successor lease resumes interrupted cleanup before dispatch. + +Every plan, receipt claim, and message claim carries a physical batch identity. Cleanup +matches that identity; its final plan deletion also matches a release identity and the +queue lease. A durable dispatch counter prevents a definite retry failure from releasing +an earlier ambiguous handoff. Native recovery fences distinguish unadmitted tombstones +from actual generation admission. Proven admissions settle followers, never re-present +results. + +Retired owners do not expire while a batch still needs claim release or per-receipt +admission proof. Cleanup restores the ordinary retention deadline only after every +member is safe without the owner. Retirement also closes publication and dispatch. + +A failed receipt write is repaired from its durable terminal message projection. +Manual polling marks receipt reconciliation before automatic delivery can settle that +handoff; speculative manual ownership only defers the automatic delivery. + +Manual confirmation resolves a lost reply by reading the exact committed claim. +Failed confirmation rolls back only unreconciled ownership. If rollback is unavailable, +automatic delivery invokes manual-generation recovery after that owner is no longer active. +Its release CAS excludes committed manual handoffs, including confirmation after a stale +snapshot. Explicit manual recovery keeps its existing generation-fenced contract. + +Manual reconciliation survives registry restoration and final response persistence. +Collecting plans acquire the root before siblings, so empty retirement can clear an +unowned plan without orphaning claims. Native `started` recovery fences confirm admission +even when a custom store cannot read historical claims after job cleanup. + +Exact durable replays read the current claim without rewriting its handoff proof. +Applied followers finish interrupted owner proof copying before they settle, including +retired roots removed from the delivery queue. + +Manual polls carry the observed physical batch ID through recovery. A released owner’s +late projection is cleared by that exact ID; successor epochs and manual claims remain owned. diff --git a/packages/api/src/agents/triggers/dispatch.ts b/packages/api/src/agents/triggers/dispatch.ts index 09d7607eca..99fe88e14e 100644 --- a/packages/api/src/agents/triggers/dispatch.ts +++ b/packages/api/src/agents/triggers/dispatch.ts @@ -11,6 +11,9 @@ export interface AgentTriggerDispatchContext { /** Durable delivery attempt metadata, when dispatched by the queue engine. */ attempt?: number; maxAttempts?: number; + /** Exact queue lease that owns this preparation. */ + deliveryClaimToken?: string; + requiredWorkerCapability?: string; signal?: AbortSignal; } @@ -44,7 +47,13 @@ export interface AgentTriggerDispatchHandlers( envelope: unknown, handlers: AgentTriggerDispatchHandlers, - options?: { signal?: AbortSignal; attempt?: number; maxAttempts?: number }, + options?: { + signal?: AbortSignal; + attempt?: number; + maxAttempts?: number; + deliveryClaimToken?: string; + requiredWorkerCapability?: string; + }, ): Promise { let normalized: AgentTriggerEnvelope; try { @@ -56,6 +65,10 @@ export function dispatchAgentTrigger( idempotencyKey: getAgentTriggerIdempotencyKey(normalized), ...(options?.attempt != null && { attempt: options.attempt }), ...(options?.maxAttempts != null && { maxAttempts: options.maxAttempts }), + ...(options?.deliveryClaimToken != null && { deliveryClaimToken: options.deliveryClaimToken }), + ...(options?.requiredWorkerCapability != null && { + requiredWorkerCapability: options.requiredWorkerCapability, + }), ...(options?.signal != null && { signal: options.signal }), }; if (normalized.mode === 'fire') { diff --git a/packages/api/src/agents/triggers/engine.spec.ts b/packages/api/src/agents/triggers/engine.spec.ts index 697834d4f8..7c151f1026 100644 --- a/packages/api/src/agents/triggers/engine.spec.ts +++ b/packages/api/src/agents/triggers/engine.spec.ts @@ -74,7 +74,12 @@ describe('createAgentTriggerDeliveryEngine', () => { expect(dispatch).toHaveBeenCalledWith( { version: 1 }, - { signal: expect.any(AbortSignal), attempt: 1, maxAttempts: 8 }, + { + signal: expect.any(AbortSignal), + attempt: 1, + maxAttempts: 8, + deliveryClaimToken: 'claim-1', + }, ); expect(store.beginAttempt).toHaveBeenCalledWith({ id: 'delivery-row-1', diff --git a/packages/api/src/agents/triggers/engine.ts b/packages/api/src/agents/triggers/engine.ts index e6d6f9e8fc..7f1414ba22 100644 --- a/packages/api/src/agents/triggers/engine.ts +++ b/packages/api/src/agents/triggers/engine.ts @@ -92,6 +92,7 @@ export interface AgentTriggerDeliveryRecord { orderingKey: string; laneSequence: number; envelope: unknown; + requiredWorkerCapability?: string; status: AgentTriggerDeliveryStatus; attempts: number; availableAt: Date; @@ -204,7 +205,13 @@ export interface AgentTriggerDeliveryEngineDeps { store: AgentTriggerDeliveryStore; dispatch: ( envelope: unknown, - options?: { signal?: AbortSignal; attempt?: number; maxAttempts?: number }, + options?: { + signal?: AbortSignal; + attempt?: number; + maxAttempts?: number; + deliveryClaimToken?: string; + requiredWorkerCapability?: string; + }, ) => Promise; /** Source-owned terminalization must commit before its delivery can become * dead, including recovery after a crash that exhausted the attempt budget. */ @@ -489,6 +496,10 @@ export function createAgentTriggerDeliveryEngine( signal: controller.signal, attempt, maxAttempts, + deliveryClaimToken: delivery.claimToken, + ...(delivery.requiredWorkerCapability != null && { + requiredWorkerCapability: delivery.requiredWorkerCapability, + }), }); } catch (error) { const attemptedAt = now(); diff --git a/packages/api/src/agents/triggers/host.ts b/packages/api/src/agents/triggers/host.ts index 6bf30f70b8..30c6eb818e 100644 --- a/packages/api/src/agents/triggers/host.ts +++ b/packages/api/src/agents/triggers/host.ts @@ -70,6 +70,8 @@ export type AgentTriggerContinuePreparation = releaseOnDefiniteFailure?: (error?: AgentTriggerExecutionError) => MaybePromise; /** Commits the source handoff after generation admission. Failure is * outcome-ambiguous: the same delivery retries with the same request id. */ + /** Durable possible-handoff marker written immediately before transport. */ + beginDispatch?: () => MaybePromise; settleOnAdmission?: (result: AgentTriggerContinueResult) => MaybePromise; } | { status: 'settled' }; @@ -172,7 +174,13 @@ export interface AgentTriggerExecutionHostDeps { export interface AgentTriggerExecutionHost { dispatch: ( envelope: unknown, - options?: { signal?: AbortSignal; attempt?: number; maxAttempts?: number }, + options?: { + signal?: AbortSignal; + attempt?: number; + maxAttempts?: number; + deliveryClaimToken?: string; + requiredWorkerCapability?: string; + }, ) => Promise; } @@ -636,6 +644,7 @@ async function startRun( const url = mode === 'fire' ? fireUrl(baseUrl) : continueUrl(baseUrl); const fetcher: AgentTriggerFetch = deps.fetch ?? globalThis.fetch; let response: Response; + await readyPreparation?.beginDispatch?.(); try { response = await fetcher(url, { method: 'POST', diff --git a/packages/api/src/agents/triggers/service.delivery.spec.ts b/packages/api/src/agents/triggers/service.delivery.spec.ts index c62b09e8ce..2f0b9fb1d8 100644 --- a/packages/api/src/agents/triggers/service.delivery.spec.ts +++ b/packages/api/src/agents/triggers/service.delivery.spec.ts @@ -1,5 +1,6 @@ import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1, @@ -185,6 +186,7 @@ describe('durable agent trigger service', () => { expect.objectContaining({ workerCapabilities: [ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1, AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2, @@ -211,6 +213,14 @@ describe('durable agent trigger service', () => { await service.stop(); }); + it('keeps receipt batching disabled until the deployment gate is explicitly enabled', async () => { + const service = createAgentTriggerService({ mintToken: () => 'test' }); + expect(service.getBackgroundCompletionReceiptBatching()).toBe(false); + await service.initialize({ address: '127.0.0.1', completionReceiptBatching: true }); + expect(service.getBackgroundCompletionReceiptBatching()).toBe(true); + await service.stop(); + }); + describe('waiting completion deliveries', () => { const address = { address: '127.0.0.1', family: 'IPv4' as const, port: 3080 }; const flush = () => new Promise((resolve) => setImmediate(resolve)); @@ -407,6 +417,7 @@ describe('durable agent trigger service', () => { expect.objectContaining({ workerCapabilities: [ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1, AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2, diff --git a/packages/api/src/agents/triggers/service.ts b/packages/api/src/agents/triggers/service.ts index f2786ed860..1dc249251c 100644 --- a/packages/api/src/agents/triggers/service.ts +++ b/packages/api/src/agents/triggers/service.ts @@ -1,5 +1,6 @@ import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1, @@ -50,6 +51,7 @@ const DEFAULT_PURGE_RECOVERY_LIMIT = 25; export interface AgentTriggerServiceOptions { completionResultBatchSize?: number; + completionReceiptBatching?: boolean; address?: BoundAddress | string | null; idlePolling?: { queuedTurnMaxIntervalMs?: number; @@ -230,6 +232,7 @@ export interface AgentTriggerService { input: Parameters[0], ) => ReturnType; getBackgroundCompletionResultBatchSize: () => number; + getBackgroundCompletionReceiptBatching: () => boolean; /** Longest a waiting completion delivery re-checks readiness. */ getCompletionWaitMaxIntervalMs: () => number; /** Best effort: moves waiting completion deliveries forward after what they wait on changed. */ @@ -271,6 +274,7 @@ function createDeliveryStore( ...input, workerCapabilities: [ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1, AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2, @@ -352,6 +356,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A } let boundOrigin: string | undefined; let backgroundCompletionResultBatchSize = 8; + let backgroundCompletionReceiptBatching = false; let completionWaitMaxIntervalMs = WAITING_RETRY_CAP_MS; let deliveryEngine: AgentTriggerDeliveryEngine | undefined; let initializePromise: Promise | undefined; @@ -410,7 +415,13 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A const dispatchForActivePrincipal = async ( envelope: unknown, - options?: { signal?: AbortSignal; attempt?: number; maxAttempts?: number }, + options?: { + signal?: AbortSignal; + attempt?: number; + maxAttempts?: number; + deliveryClaimToken?: string; + requiredWorkerCapability?: string; + }, ): Promise => { const parsed = parseAgentTriggerEnvelope(envelope); await requireActivePrincipal(parsed.principal.userId); @@ -619,6 +630,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A return { initialize: (options = {}) => { backgroundCompletionResultBatchSize = options.completionResultBatchSize ?? 8; + backgroundCompletionReceiptBatching = options.completionReceiptBatching === true; completionWaitMaxIntervalMs = options.idlePolling?.completionWaitMaxIntervalMs ?? WAITING_RETRY_CAP_MS; boundOrigin = selfOriginFromAddress(options.address) ?? boundOrigin; @@ -809,6 +821,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A return getClaim == null ? null : getClaim(input); }), getBackgroundCompletionResultBatchSize: () => backgroundCompletionResultBatchSize, + getBackgroundCompletionReceiptBatching: () => backgroundCompletionReceiptBatching, getCompletionWaitMaxIntervalMs: () => completionWaitMaxIntervalMs, expediteCompletionWakeups: (input) => expediteCompletions(input), releaseBackgroundToolResultClaims: (input) => diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts index 40869e850f..3d28faabdc 100644 --- a/packages/api/src/stream/GenerationJobManager.ts +++ b/packages/api/src/stream/GenerationJobManager.ts @@ -270,7 +270,8 @@ function claimsMirrorExactly(left: TokenIdempotencyClaim, right: TokenIdempotenc left.claimToken === right.claimToken && left.previousClaimToken === right.previousClaimToken && left.generationProtocolVersion === right.generationProtocolVersion && - left.startedAt === right.startedAt + left.startedAt === right.startedAt && + left.recoveryFence === right.recoveryFence ); } @@ -3339,7 +3340,7 @@ class GenerationJobManagerClass { } const normalized = normalizeTokenClaim(claim, 'admission evidence'); assertClaimMatchesRequest(normalized, streamId, conversationId); - if (normalized.startedAt == null) { + if (normalized.startedAt == null || normalized.recoveryFence === true) { return null; } return { @@ -3380,14 +3381,14 @@ class GenerationJobManagerClass { return 'unavailable'; } if (observed.existing.startedAt != null) { - return 'started'; + return observed.existing.recoveryFence === true ? 'fenced' : 'started'; } let owned = observed; if (!observed.claimed) { owned = await this.takeoverGeneration(userId, clientRequestId, streamId, observed.existing); if (owned.existing?.startedAt != null) { - return 'started'; + return owned.existing.recoveryFence === true ? 'fenced' : 'started'; } if (!owned.claimed || owned.existing == null) { return 'unavailable'; @@ -3396,7 +3397,7 @@ class GenerationJobManagerClass { const claim = normalizeTokenClaim(owned.existing, 'background completion recovery fence'); if (claim.startedAt != null) { - return 'started'; + return claim.recoveryFence === true ? 'fenced' : 'started'; } await this.tombstoneObservedGenerationClaim( userId, @@ -3405,6 +3406,7 @@ class GenerationJobManagerClass { claim, Date.now(), claim.generationProtocolVersion === 2 ? 2 : 1, + true, ); return 'fenced'; } @@ -3477,11 +3479,13 @@ class GenerationJobManagerClass { claim: TokenIdempotencyClaim, createdAt: number, generationProtocolVersion: 1 | 2, + recoveryFence?: true, ): Promise { const tombstone: TokenIdempotencyClaim = { ...claim, startedAt: createdAt, generationProtocolVersion, + ...(recoveryFence === true && { recoveryFence }), }; const primaryKey = this.generationClaimKey(userId, clientRequestId, streamId); let markError: unknown; diff --git a/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts b/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts index 1c8578d86d..117d858e30 100644 --- a/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts +++ b/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts @@ -394,6 +394,13 @@ describe('GenerationJobManager start-generation claim', () => { }); }); + it('does not mistake a replayed recovery tombstone for actual admission proof', async () => { + const args = ['user-1', 'req-recovery-proof', 'stream-proof', 'stream-proof'] as const; + expect(await manager.fenceGenerationClaimForRecovery(...args)).toBe('fenced'); + expect(await manager.fenceGenerationClaimForRecovery(...args)).toBe('fenced'); + expect(await manager.getGenerationAdmissionEvidence(...args)).toBeNull(); + }); + it('invalidates an unpublished continuation creator before manual recovery', async () => { const original = await manager.claimGeneration( 'user-1', diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.ts index 5c705d70bb..467e067985 100644 --- a/packages/api/src/stream/interfaces/IJobStore.ts +++ b/packages/api/src/stream/interfaces/IJobStore.ts @@ -714,6 +714,8 @@ export interface IdempotencyClaimValue { * already-started/cleaned generation and can never be taken over as an * abandoned pre-create lease. */ startedAt?: number; + /** Recovery closed an unpublished claim without admitting a generation. */ + recoveryFence?: true; } /** Result of an atomic {@link IJobStore.claimIdempotencyKey} attempt. */ diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts index 14fae0dc87..8c06a6a0dc 100644 --- a/packages/data-provider/src/config.spec.ts +++ b/packages/data-provider/src/config.spec.ts @@ -1063,6 +1063,13 @@ describe('agent background completion batch config', () => { endpoints: { agents: { backgroundTasks: {} } }, }); expect(defaults.endpoints?.agents?.backgroundTasks?.completionResultBatchSize).toBe(8); + expect(defaults.endpoints?.agents?.backgroundTasks?.completionReceiptBatching).toBe(false); + expect( + configSchema.parse({ + version: '1.0', + endpoints: { agents: { backgroundTasks: { completionReceiptBatching: true } } }, + }).endpoints?.agents?.backgroundTasks?.completionReceiptBatching, + ).toBe(true); for (const completionResultBatchSize of [0, 17, 1.5]) { expect( @@ -1171,6 +1178,7 @@ describe('agent background task config', () => { } expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({ completionResultBatchSize: 8, + completionReceiptBatching: false, completionWakeups: true, completionResultMaxChars: 24 * 1024, ordinaryToolCancellation: false, @@ -1190,6 +1198,7 @@ describe('agent background task config', () => { } expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({ completionResultBatchSize: 8, + completionReceiptBatching: false, completionWakeups: false, completionResultMaxChars: 24 * 1024, ordinaryToolCancellation: false, @@ -1209,6 +1218,7 @@ describe('agent background task config', () => { } expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({ completionResultBatchSize: 8, + completionReceiptBatching: false, completionWakeups: true, completionResultMaxChars: 24 * 1024, ordinaryToolCancellation: true, diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index f64b03666c..3ea0043fa2 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -1736,8 +1736,9 @@ export const agentsEndpointSchema = baseEndpointSchema .max(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX) .optional() .default(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT), - /** Maximum message-backed sibling results in one continuation. - * Independent receipts retain task-local delivery ownership. */ + /** Enable only after every replica has compatible receipt/poll consumers. */ + completionReceiptBatching: z.boolean().optional().default(false), + /** Maximum compatible sibling results in one continuation. */ completionResultBatchSize: z.number().int().min(1).max(16).optional().default(8), /** Cooperative cancellation for process-local ordinary tools. Off * by default so existing deployments opt into the new control. */ diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.ts index e2f7e93ce9..6afad13b80 100644 --- a/packages/data-schemas/src/index.ts +++ b/packages/data-schemas/src/index.ts @@ -76,6 +76,7 @@ export { export { AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1, diff --git a/packages/data-schemas/src/methods/index.ts b/packages/data-schemas/src/methods/index.ts index 825fe5a15d..c5417b8d7a 100644 --- a/packages/data-schemas/src/methods/index.ts +++ b/packages/data-schemas/src/methods/index.ts @@ -347,6 +347,7 @@ export function createMethods( const agentQueuedTurnMethods = createAgentQueuedTurnMethods(mongoose); const agentTriggerDeliveryMethods = createAgentTriggerDeliveryMethods(mongoose, { + releaseBatchProjections: messageMethods.releaseBackgroundToolResultClaims, purgeQueuedTurnsForUser: (user) => agentQueuedTurnMethods.deleteAllAgentQueuedTurnsForUser({ user: typeof user === 'string' ? new mongoose.Types.ObjectId(user) : user, diff --git a/packages/data-schemas/src/methods/message.spec.ts b/packages/data-schemas/src/methods/message.spec.ts index 5e029e423a..3b87bbf478 100644 --- a/packages/data-schemas/src/methods/message.spec.ts +++ b/packages/data-schemas/src/methods/message.spec.ts @@ -5991,3 +5991,94 @@ describe('Message Operations', () => { }); }); }); + +describe('receipt-backed claim release before projection', () => { + it('allows a missing message only when the receipt owner explicitly opts in', async () => { + const input = { + userId: new mongoose.Types.ObjectId().toString(), + conversationId: 'missing-conversation', + messageId: 'missing-parent', + kind: 'wakeup' as const, + claimId: 'batch-root', + }; + await expect(releaseBackgroundToolResultClaims(input)).resolves.toBe(false); + await expect( + releaseBackgroundToolResultClaims({ ...input, allowMissingMessage: true }), + ).resolves.toBe(true); + }); +}); + +describe('receipt batch projection epochs', () => { + it('does not let delayed cleanup remove a successor claim on the same logical request', async () => { + await Message.create({ + messageId: 'epoch-parent', + conversationId: 'epoch-conversation', + user: 'epoch-owner', + isCreatedByUser: false, + unfinished: false, + content: [ + { + type: 'tool_call', + tool_call: { + id: 'epoch-call', + output: 'done', + backgroundTask: { + taskId: 'epoch-task', + toolName: 'tool', + status: 'completed', + completionWakeup: true, + completionReceipt: true, + }, + }, + }, + ], + }); + const input = { + userId: 'epoch-owner', + conversationId: 'epoch-conversation', + messageId: 'epoch-parent', + taskId: 'epoch-task', + kind: 'wakeup' as const, + claimId: 'logical-root', + limit: 1, + }; + expect((await claimBackgroundToolResults({ ...input, batchId: 'old' })).status).toBe( + 'acquired', + ); + let pause: () => void = () => undefined; + let resume: () => void = () => undefined; + const paused = new Promise((resolve) => { + pause = resolve; + }); + const barrier = new Promise((resolve) => { + resume = resolve; + }); + const update = Message.findOneAndUpdate.bind(Message); + const oldCleanup = jest.spyOn(Message, 'findOneAndUpdate').mockImplementationOnce((...args) => { + const query = update(...args); + const execute = query.exec.bind(query); + jest.spyOn(query, 'exec').mockImplementationOnce(async () => { + pause(); + await barrier; + return execute(); + }); + return query; + }); + const predecessor = releaseBackgroundToolResultClaims({ ...input, batchId: 'old' }); + await paused; + expect(await releaseBackgroundToolResultClaims({ ...input, batchId: 'old' })).toBe(true); + expect((await claimBackgroundToolResults({ ...input, batchId: 'new' })).status).toBe( + 'acquired', + ); + resume(); + expect(await predecessor).toBe(true); + oldCleanup.mockRestore(); + expect((await claimBackgroundToolResults({ ...input, batchId: 'new' })).status).toBe( + 'acquired', + ); + expect(await claimBackgroundToolResults({ ...input, batchId: 'old' })).toMatchObject({ + status: 'claimed', + claim: { batchId: 'new' }, + }); + }); +}); diff --git a/packages/data-schemas/src/methods/message.ts b/packages/data-schemas/src/methods/message.ts index 5d0ec86234..66181b1b7c 100644 --- a/packages/data-schemas/src/methods/message.ts +++ b/packages/data-schemas/src/methods/message.ts @@ -567,7 +567,13 @@ export type BackgroundToolResultClaim = | { status: 'outcome_unknown'; toolName: string } | { status: 'claimed'; - claim?: { kind: 'manual' | 'wakeup'; claimId: string; generationId?: string }; + claim?: { + kind: 'manual' | 'wakeup'; + claimId: string; + generationId?: string; + batchId?: string; + receiptReconciled?: true; + }; messageId?: string; } | { status: 'acquired'; results: BackgroundToolResultRecord[]; messageId?: string }; @@ -846,6 +852,7 @@ export interface MessageMethods { claimId: string; claimedAt: Date; generationId?: string; + receiptReconciled?: true; }; }; }): Promise<{ matched: boolean; unfinished: boolean }>; @@ -858,6 +865,8 @@ export interface MessageMethods { agentId?: string; kind: 'manual' | 'wakeup'; claimId: string; + /** Physical receipt-batch identity, independent of its logical request. */ + batchId?: string; /** Response generation that owns this manual result delivery. */ generationId?: string; /** Manual owner-process takeover after automatic delivery was retired. */ @@ -866,12 +875,24 @@ export interface MessageMethods { /** Includes JSON escaping, delimiters, and empty result fields. */ maxMetadataChars?: number; }): Promise; + confirmBackgroundToolResultClaim(params: { + userId: string; + conversationId: string; + messageId: string; + taskId: string; + claimId: string; + }): Promise; releaseBackgroundToolResultClaims(params: { userId: string; conversationId: string; messageId: string; /** Omit to release every sibling owned by this exact batch claim. */ taskIds?: string[]; + /** Receipt-backed results can precede the message projection. */ + allowMissingMessage?: true; + /** Manual rollback must not erase a committed receipt handoff. */ + onlyIfUnreconciled?: true; + batchId?: string; kind: 'manual' | 'wakeup'; claimId: string; }): Promise; @@ -1983,7 +2004,15 @@ export function createMessageMethods( function readBackgroundToolResultClaim( row: Pick, taskId: string, - ): { kind: 'manual' | 'wakeup'; claimId: string; generationId?: string } | undefined { + ): + | { + kind: 'manual' | 'wakeup'; + claimId: string; + generationId?: string; + batchId?: string; + receiptReconciled?: true; + } + | undefined { for (const part of row.content ?? []) { if (part == null || typeof part !== 'object' || Array.isArray(part)) { continue; @@ -1993,7 +2022,13 @@ export function createMessageMethods( tool_call?: { backgroundTask?: { taskId?: unknown; - resultClaim?: { kind?: unknown; claimId?: unknown; generationId?: unknown }; + resultClaim?: { + kind?: unknown; + claimId?: unknown; + generationId?: unknown; + batchId?: unknown; + receiptReconciled?: unknown; + }; }; }; } @@ -2010,6 +2045,8 @@ export function createMessageMethods( return { kind: claim.kind, claimId: claim.claimId, + ...(typeof claim.batchId === 'string' && { batchId: claim.batchId }), + ...(claim.receiptReconciled === true && { receiptReconciled: true }), ...(typeof claim.generationId === 'string' && claim.generationId.length > 0 ? { generationId: claim.generationId } : {}), @@ -2021,7 +2058,7 @@ export function createMessageMethods( function parseBackgroundToolResults( message: IMessage, - claim: { kind: 'manual' | 'wakeup'; claimId: string }, + claim: { kind: 'manual' | 'wakeup'; claimId: string; batchId?: string }, ): BackgroundToolResultRecord[] { const results: BackgroundToolResultRecord[] = []; for (const part of message.content ?? []) { @@ -2040,7 +2077,7 @@ export function createMessageMethods( status?: unknown; cancelled?: unknown; settledAt?: unknown; - resultClaim?: { kind?: unknown; claimId?: unknown }; + resultClaim?: { kind?: unknown; claimId?: unknown; batchId?: unknown }; }; }; }; @@ -2052,7 +2089,8 @@ export function createMessageMethods( typeof task.toolName !== 'string' || (task.status !== 'completed' && task.status !== 'error') || task.resultClaim?.kind !== claim.kind || - task.resultClaim.claimId !== claim.claimId + task.resultClaim.claimId !== claim.claimId || + task.resultClaim.batchId !== claim.batchId ) { continue; } @@ -2133,6 +2171,7 @@ export function createMessageMethods( kind, claimId, generationId, + batchId, allowUnfinished = false, limit = kind === 'wakeup' ? 8 : 1, maxMetadataChars, @@ -2144,6 +2183,7 @@ export function createMessageMethods( if ( (requestedMessageId != null && (requestedMessageId.length === 0 || requestedMessageId.length > 256)) || + (batchId != null && (batchId.length === 0 || batchId.length > 128 || kind !== 'wakeup')) || taskId.length === 0 || taskId.length > 256 || claimId.length === 0 || @@ -2214,7 +2254,10 @@ export function createMessageMethods( } const recoveredSource = requestedMessageId == null ? { messageId: resolvedMessageId } : {}; const requestedClaim = readBackgroundToolResultClaim(row, taskId); - const replaying = requestedClaim?.kind === kind && requestedClaim.claimId === claimId; + const replaying = + requestedClaim?.kind === kind && + requestedClaim.claimId === claimId && + requestedClaim.batchId === batchId; const candidates: string[] = []; const costs = new Map(); let receiptBacked = false; @@ -2233,8 +2276,13 @@ export function createMessageMethods( } const terminal = task?.status === 'completed' || task?.status === 'error'; const wakeupEligible = kind !== 'wakeup' || task?.completionWakeup === true; - const resultClaim = task?.resultClaim as { kind?: unknown; claimId?: unknown } | undefined; - const replay = resultClaim?.kind === kind && resultClaim.claimId === claimId; + const resultClaim = task?.resultClaim as + | { kind?: unknown; claimId?: unknown; batchId?: unknown } + | undefined; + const replay = + resultClaim?.kind === kind && + resultClaim.claimId === claimId && + resultClaim.batchId === batchId; const sameAgent = agentId == null || partAgentId == null || @@ -2315,105 +2363,103 @@ export function createMessageMethods( kind, claimId, claimedAt, + ...(batchId != null && { batchId }), ...(kind === 'manual' && requestedGenerationId != null ? { generationId: requestedGenerationId } : {}), }; - const updated = await Message.findOneAndUpdate( - { - user: userId, - conversationId, - messageId: resolvedMessageId, - ...(kind === 'manual' && allowUnfinished ? {} : { unfinished: { $ne: true } }), - content: { - $elemMatch: { - type: 'tool_call', - 'tool_call.backgroundTask.taskId': taskId, - 'tool_call.backgroundTask.status': { $in: ['completed', 'error'] }, - ...(kind === 'wakeup' ? { 'tool_call.backgroundTask.completionWakeup': true } : {}), - ...(replaying - ? { - 'tool_call.backgroundTask.resultClaim.kind': kind, - 'tool_call.backgroundTask.resultClaim.claimId': claimId, - } - : /** Missing OR stored null: the in-memory claimable scan, the - * claim arrayFilters, and the settle stamp all treat a null - * claim as unclaimed, and the subfield-preserving settle write - * keeps a persisted null a whole-object rewrite used to drop. - * `$exists: false` here would strand such a part as terminal - * but permanently unclaimable. */ - { 'tool_call.backgroundTask.resultClaim': null }), - }, + const claimFilter: FilterQuery = { + user: userId, + conversationId, + messageId: resolvedMessageId, + ...(kind === 'manual' && allowUnfinished ? {} : { unfinished: { $ne: true } }), + content: { + $elemMatch: { + type: 'tool_call', + 'tool_call.backgroundTask.taskId': taskId, + 'tool_call.backgroundTask.status': { $in: ['completed', 'error'] }, + ...(kind === 'wakeup' ? { 'tool_call.backgroundTask.completionWakeup': true } : {}), + ...(replaying + ? { + 'tool_call.backgroundTask.resultClaim.kind': kind, + 'tool_call.backgroundTask.resultClaim.claimId': claimId, + 'tool_call.backgroundTask.resultClaim.batchId': batchId ?? null, + } + : /** Missing OR stored null: the in-memory claimable scan, the + * claim arrayFilters, and the settle stamp all treat a null + * claim as unclaimed, and the subfield-preserving settle write + * keeps a persisted null a whole-object rewrite used to drop. + * `$exists: false` here would strand such a part as terminal + * but permanently unclaimable. */ + { 'tool_call.backgroundTask.resultClaim': null }), }, - ...(agentId != null - ? { - $expr: { - $anyElementTrue: { - $map: { - input: { $ifNull: ['$content', []] }, - as: 'candidate', - in: { - $and: [ - { $eq: ['$$candidate.tool_call.backgroundTask.taskId', taskId] }, - { - $in: [ - { - $ifNull: [ - { - $ifNull: ['$$candidate.agentId', '$$candidate.tool_call.agentId'], - }, - null, - ], - }, - [null, agentId], - ], - }, - ], - }, + }, + ...(agentId != null + ? { + $expr: { + $anyElementTrue: { + $map: { + input: { $ifNull: ['$content', []] }, + as: 'candidate', + in: { + $and: [ + { $eq: ['$$candidate.tool_call.backgroundTask.taskId', taskId] }, + { + $in: [ + { + $ifNull: [ + { + $ifNull: ['$$candidate.agentId', '$$candidate.tool_call.agentId'], + }, + null, + ], + }, + [null, agentId], + ], + }, + ], }, }, }, - } - : {}), - }, - /** Stamps the claim onto every part this pass admitted. The filtered - * positional operator selects those parts by predicate, so the write - * touches only them instead of re-emitting the whole content array, and - * needs no read-modify-write. Amazon DocumentDB rejects the - * aggregation-pipeline form this replaces. */ - { $set: { 'content.$[part].tool_call.backgroundTask.resultClaim': claimStamp } }, - { - new: true, - projection: { content: 1 }, - arrayFilters: [ + }, + } + : {}), + }; + const updated = replaying + ? await Message.findOne(claimFilter).select({ content: 1 }).lean() + : await Message.findOneAndUpdate( + claimFilter, + /** Stamps the claim onto every part this pass admitted. The filtered + * positional operator selects those parts by predicate, so the write + * touches only them instead of re-emitting the whole content array, and + * needs no read-modify-write. Amazon DocumentDB rejects the + * aggregation-pipeline form this replaces. */ + { $set: { 'content.$[part].tool_call.backgroundTask.resultClaim': claimStamp } }, { - 'part.type': 'tool_call', - 'part.tool_call.backgroundTask.taskId': { $in: candidates }, - 'part.tool_call.backgroundTask.status': { $in: ['completed', 'error'] }, - ...(kind === 'wakeup' - ? { 'part.tool_call.backgroundTask.completionWakeup': true } - : {}), - $and: [ + new: true, + projection: { content: 1 }, + arrayFilters: [ { - /** Unclaimed, or already held by this exact claimant (replay). */ - $or: [ - { 'part.tool_call.backgroundTask.resultClaim': null }, + 'part.type': 'tool_call', + 'part.tool_call.backgroundTask.taskId': { $in: candidates }, + 'part.tool_call.backgroundTask.status': { $in: ['completed', 'error'] }, + ...(kind === 'wakeup' + ? { 'part.tool_call.backgroundTask.completionWakeup': true } + : {}), + $and: [ { - 'part.tool_call.backgroundTask.resultClaim.kind': kind, - 'part.tool_call.backgroundTask.resultClaim.claimId': claimId, + 'part.tool_call.backgroundTask.resultClaim': null, }, + ...(agentId == null ? [] : [agentOwnershipFilter('part.', agentId)]), ], }, - ...(agentId == null ? [] : [agentOwnershipFilter('part.', agentId)]), ], }, - ], - }, - ).lean(); + ).lean(); if (updated == null) { return { status: 'not_ready' }; } - const results = parseBackgroundToolResults(updated, { kind, claimId }); + const results = parseBackgroundToolResults(updated, { kind, claimId, batchId }); const competingClaim = readBackgroundToolResultClaim(updated, taskId); return results.some((result) => result.taskId === taskId) ? { status: 'acquired', results, ...recoveredSource } @@ -2424,21 +2470,82 @@ export function createMessageMethods( }; } + /** Manual ownership is committed only after receipt arbitration completes. */ + async function confirmBackgroundToolResultClaim( + input: Parameters[0], + ): Promise { + const Message = mongoose.models.Message as Model; + const identity = { + 'tool_call.backgroundTask.taskId': input.taskId, + 'tool_call.backgroundTask.resultClaim.kind': 'manual', + 'tool_call.backgroundTask.resultClaim.claimId': input.claimId, + }; + try { + const confirmed = await Message.updateOne( + { + user: input.userId, + conversationId: input.conversationId, + messageId: input.messageId, + content: { $elemMatch: identity }, + }, + { + $set: { 'content.$[part].tool_call.backgroundTask.resultClaim.receiptReconciled': true }, + }, + { + arrayFilters: [ + { + 'part.tool_call.backgroundTask.taskId': input.taskId, + 'part.tool_call.backgroundTask.resultClaim.kind': 'manual', + 'part.tool_call.backgroundTask.resultClaim.claimId': input.claimId, + }, + ], + }, + ); + return confirmed.matchedCount === 1; + } catch (error) { + // A lost write reply is not an uncommitted handoff. Read only the exact + // manual claim, never manufacture success from another claimant. + const committed = await Message.exists({ + user: input.userId, + conversationId: input.conversationId, + messageId: input.messageId, + content: { + $elemMatch: { + ...identity, + 'tool_call.backgroundTask.resultClaim.receiptReconciled': true, + }, + }, + }); + if (committed != null) return true; + throw error; + } + } + async function releaseBackgroundToolResultClaims({ userId, conversationId, messageId, taskIds, + allowMissingMessage, + onlyIfUnreconciled, kind, claimId, + batchId, }: { userId: string; conversationId: string; messageId: string; taskIds?: string[]; + /** Receipt-backed results can precede the message projection. */ + allowMissingMessage?: true; + /** Manual rollback must not erase a committed receipt handoff. */ + onlyIfUnreconciled?: true; + batchId?: string; kind: 'manual' | 'wakeup'; claimId: string; }): Promise { + if (onlyIfUnreconciled === true && kind !== 'manual') + throw new TypeError('Unreconciled rollback requires a manual claim'); if (taskIds?.length === 0) { return true; } @@ -2460,6 +2567,10 @@ export function createMessageMethods( { 'part.tool_call.backgroundTask.resultClaim.kind': kind, 'part.tool_call.backgroundTask.resultClaim.claimId': claimId, + 'part.tool_call.backgroundTask.resultClaim.batchId': batchId ?? null, + ...(onlyIfUnreconciled === true && { + 'part.tool_call.backgroundTask.resultClaim.receiptReconciled': { $ne: true }, + }), ...(taskIds == null ? {} : { 'part.tool_call.backgroundTask.taskId': { $in: taskIds } }), @@ -2474,9 +2585,13 @@ export function createMessageMethods( messageId, content: { $not: { $type: 'array' } }, }); - return arraylessRow != null; + if (arraylessRow != null) return true; + return ( + allowMissingMessage === true && + (await Message.exists({ user: userId, conversationId, messageId })) == null + ); } - const remaining = parseBackgroundToolResults(updated, { kind, claimId }); + const remaining = parseBackgroundToolResults(updated, { kind, claimId, batchId }); return taskIds == null ? remaining.length === 0 : !remaining.some((result) => taskIds.includes(result.taskId)); @@ -4249,6 +4364,7 @@ export function createMessageMethods( updateMessageText, updateToolCallResult, claimBackgroundToolResults, + confirmBackgroundToolResultClaim, releaseBackgroundToolResultClaims, updateMessage, recordSubagentTaskControlReceipt, diff --git a/packages/data-schemas/src/methods/triggerDelivery.spec.ts b/packages/data-schemas/src/methods/triggerDelivery.spec.ts index c49c46ea22..51e878d37e 100644 --- a/packages/data-schemas/src/methods/triggerDelivery.spec.ts +++ b/packages/data-schemas/src/methods/triggerDelivery.spec.ts @@ -10,6 +10,7 @@ import type { import type { IUser } from '~/types/user'; import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1, AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2, @@ -28,6 +29,8 @@ import { createAgentTriggerLaneSequenceModel } from '../models/triggerLaneSequen import { createAgentTriggerUserPurgeModel } from '../models/triggerUserPurge'; import { createAgentTriggerDeliveryModel } from '../models/triggerDelivery'; import { createConversationModel } from '../models/convo'; +import { createMessageModel } from '../models/message'; +import { createMessageMethods } from './message'; import { createUserModel } from '../models/user'; jest.mock('~/config/winston', () => ({ @@ -56,8 +59,11 @@ beforeAll(async () => { UserPurge = createAgentTriggerUserPurgeModel(mongoose); User = createUserModel(mongoose); createConversationModel(mongoose); + createMessageModel(mongoose); await Promise.all([Delivery.init(), LaneSequence.init(), UserPurge.init(), User.init()]); - methods = createAgentTriggerDeliveryMethods(mongoose); + methods = createAgentTriggerDeliveryMethods(mongoose, { + releaseBatchProjections: createMessageMethods(mongoose).releaseBackgroundToolResultClaims, + }); }, DB_SETUP_TIMEOUT_MS); afterAll(async () => { @@ -79,6 +85,14 @@ afterAll(() => { setAgentEventActorReceiptMetricObserver(); }); +async function confirmBatch( + input: Parameters[0], +) { + const batch = await methods.getAgentBackgroundToolResultBatch(input); + if (batch == null) return false; + return methods.confirmAgentBackgroundToolResultBatch({ ...input, batchId: batch.batchId }); +} + function enqueueInput( overrides: Partial[0]> = {}, ) { @@ -5554,3 +5568,1079 @@ describe('agent trigger delivery methods', () => { expect(await UserPurge.countDocuments({ _id: user })).toBe(1); }); }); + +describe('background result receipt batches', () => { + const capability = AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3; + const sourceId = 'background-tool-completion'; + let user: mongoose.Types.ObjectId; + beforeEach(() => { + user = new mongoose.Types.ObjectId(); + }); + + async function ready( + key: string, + overrides: Partial[0]> = {}, + ) { + await methods.enqueueAgentTriggerDelivery( + enqueueInput({ + deliveryKey: key, + orderingKey: `lane-${key}`, + user, + tenantId: 'tenant-1', + requiredWorkerCapability: capability, + envelope: { + event: { + type: 'background-tool.completion', + source: { id: sourceId, type: 'internal' }, + payload: { taskId: key, toolCallId: `call-${key}`, toolName: 'tool' }, + }, + target: { conversationId: 'conversation', parentMessageId: 'parent', agentId: 'agent' }, + }, + ...overrides, + }), + ); + await methods.persistAgentBackgroundToolResult({ + deliveryKey: key, + sourceId, + result: { status: 'completed', output: `output-${key}`, settledAt: START }, + }); + } + + function input(deliveryKey: string) { + return { + deliveryKey, + sourceId, + userId: user.toString(), + tenantId: 'tenant-1', + conversationId: 'conversation', + parentMessageId: 'parent', + agentId: 'agent', + limit: 8, + maxMetadataChars: 16000, + }; + } + + it('freezes ready membership, defers followers, and confirms every receipt before settlement', async () => { + await ready('one'); + await ready('two'); + const batch = await methods.claimAgentBackgroundToolResultBatch(input('one')); + expect(batch).toMatchObject({ + status: 'acquired', + results: [{ taskId: 'one' }, { taskId: 'two' }], + }); + await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).resolves.toMatchObject({ + status: 'claimed', + ownerStatus: 'pending', + }); + await ready('late'); + expect(await methods.claimAgentBackgroundToolResultBatch(input('one'))).toEqual(batch); + expect(await confirmBatch(input('one'))).toBe(true); + await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).resolves.toMatchObject({ + status: 'claimed', + ownerStatus: 'applied', + }); + await Delivery.deleteOne({ deliveryKey: 'one' }); + await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).resolves.toMatchObject({ + status: 'claimed', + ownerStatus: 'applied', + }); + expect((await methods.claimAgentBackgroundToolResultBatch(input('late'))).status).toBe( + 'acquired', + ); + }); + + it.each(['claim', 'membership'] as const)( + 'reconstructs the same candidates after a crash during %s', + async (phase) => { + await ready('one'); + await ready('two'); + const updateOne = Delivery.updateOne.bind(Delivery); + const crash = + phase === 'claim' + ? jest.spyOn(Delivery, 'updateMany').mockImplementationOnce(() => { + throw new Error('crash after candidate write'); + }) + : jest.spyOn(Delivery, 'updateOne').mockImplementation((...args) => { + const query = updateOne(...args); + if (JSON.stringify(query.getUpdate()).includes('backgroundToolResultBatch.members')) + throw new Error('crash after result claims'); + return query; + }); + await expect(methods.claimAgentBackgroundToolResultBatch(input('one'))).rejects.toThrow( + 'crash', + ); + crash.mockRestore(); + await ready('late'); + const retry = await methods.claimAgentBackgroundToolResultBatch(input('one')); + expect(retry).toMatchObject({ + status: 'acquired', + results: [{ taskId: 'one' }, { taskId: 'two' }], + }); + expect(JSON.stringify(retry)).not.toContain('late'); + }, + ); + + it('reads back frozen membership after a lost write reply', async () => { + await ready('one'); + await ready('two'); + const updateOne = Delivery.updateOne.bind(Delivery); + const lostReply = jest.spyOn(Delivery, 'updateOne').mockImplementation((...args) => { + const query = updateOne(...args); + if (JSON.stringify(query.getUpdate()).includes('backgroundToolResultBatch.members')) { + const execute = query.exec.bind(query); + jest.spyOn(query, 'exec').mockImplementationOnce(async () => { + await execute(); + throw new Error('lost membership reply'); + }); + } + return query; + }); + await expect(methods.claimAgentBackgroundToolResultBatch(input('one'))).rejects.toThrow( + 'lost membership reply', + ); + lostReply.mockRestore(); + await ready('late'); + expect(await methods.claimAgentBackgroundToolResultBatch(input('one'))).toMatchObject({ + status: 'acquired', + results: [{ taskId: 'one' }, { taskId: 'two' }], + }); + }); + + it('returns only successful per-result claims when another root wins part of the selection', async () => { + await ready('one'); + await ready('two'); + await ready('three'); + const updateMany = Delivery.updateMany.bind(Delivery); + const partial = jest.spyOn(Delivery, 'updateMany').mockImplementationOnce((...args) => { + return updateMany({ ...args[0], deliveryKey: { $in: ['one', 'three'] } }, args[1], args[2]); + }); + expect(await methods.claimAgentBackgroundToolResultBatch(input('one'))).toMatchObject({ + status: 'acquired', + results: [{ taskId: 'one' }, { taskId: 'three' }], + }); + partial.mockRestore(); + expect((await methods.claimAgentBackgroundToolResultBatch(input('two'))).status).toBe( + 'acquired', + ); + }); + + it('releases every member and clears the frozen plan after definite rejection', async () => { + await ready('one'); + await ready('two'); + await methods.claimAgentBackgroundToolResultBatch(input('one')); + expect( + await methods.releaseAgentBackgroundToolResultClaims({ ...input('one'), claimId: 'one' }), + ).toBe(true); + expect( + await methods.getAgentBackgroundToolResultClaim({ ...input('two'), taskId: 'two' }), + ).toBeNull(); + expect((await methods.claimAgentBackgroundToolResultBatch(input('two'))).status).toBe( + 'acquired', + ); + }); + + it.each(['dead', 'retired'] as const)( + 'keeps followers recoverable when their root is %s', + async (terminal) => { + await ready('one'); + await ready('two'); + await methods.claimAgentBackgroundToolResultBatch(input('one')); + await Delivery.updateOne( + { deliveryKey: 'one' }, + terminal === 'dead' + ? { $set: { status: 'leased', capabilityStatus: 'dead' } } + : { $set: { status: 'succeeded', result: { backgroundToolCompletionRetired: true } } }, + ); + await expect( + methods.claimAgentBackgroundToolResultBatch(input('two')), + ).resolves.toMatchObject({ status: 'claimed', ownerStatus: 'recoverable' }); + expect(await Delivery.exists({ deliveryKey: 'two' })).not.toBeNull(); + }, + ); + + it('does not mix tenants, principals, branches, agents, sources, or v2 deliveries', async () => { + await ready('one'); + await ready('v2', { + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + }); + await ready('tenant', { tenantId: 'tenant-2' }); + await ready('user', { user: new mongoose.Types.ObjectId() }); + for (const [key, field, value] of [ + ['branch', 'parentMessageId', 'other-parent'], + ['agent', 'agentId', 'other-agent'], + ['conversation', 'conversationId', 'other-conversation'], + ] as const) { + await ready(key, { + envelope: { + event: { + type: 'background-tool.completion', + source: { id: sourceId, type: 'internal' }, + payload: { taskId: key, toolCallId: key, toolName: 'tool' }, + }, + target: { + conversationId: 'conversation', + parentMessageId: 'parent', + agentId: 'agent', + [field]: value, + }, + }, + }); + } + expect(await methods.claimAgentBackgroundToolResultBatch(input('one'))).toMatchObject({ + status: 'acquired', + results: [{ taskId: 'one' }], + }); + await expect(methods.claimAgentBackgroundToolResultBatch(input('v2'))).resolves.toEqual({ + status: 'legacy', + }); + await expect( + methods.claimAgentBackgroundToolResultBatch({ ...input('tenant'), tenantId: 'tenant-1' }), + ).resolves.toEqual({ status: 'not_ready' }); + }); + + it('prevents v2 workers from acquiring either a root or a follower during a rolling upgrade', async () => { + await ready('one'); + await ready('two'); + await methods.claimAgentBackgroundToolResultBatch(input('one')); + await expect( + methods.claimNextAgentTriggerDelivery({ + workerId: 'old', + claimToken: 'old-token', + now: START, + leaseUntil: new Date(START.getTime() + 60000), + workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2], + }), + ).resolves.toBeNull(); + const next = await methods.claimNextAgentTriggerDelivery({ + workerId: 'new', + claimToken: 'new-token', + now: START, + leaseUntil: new Date(START.getTime() + 60000), + workerCapabilities: [capability], + }); + expect(next).not.toBeNull(); + }); + + it('resumes an interrupted release under the successor queue lease', async () => { + await ready('one'); + await ready('two'); + await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityClaimToken: 'old' } }); + await methods.claimAgentBackgroundToolResultBatch({ + ...input('one'), + deliveryClaimToken: 'old', + }); + const updateOne = Delivery.findOneAndUpdate.bind(Delivery); + const lostReply = jest.spyOn(Delivery, 'findOneAndUpdate').mockImplementationOnce((...args) => { + const query = updateOne(...args); + const execute = query.exec.bind(query); + jest.spyOn(query, 'exec').mockImplementationOnce(async () => { + await execute(); + throw new Error('lost release fence reply'); + }); + return query; + }); + await expect( + methods.releaseAgentBackgroundToolResultClaims({ + ...input('one'), + claimId: 'one', + deliveryClaimToken: 'old', + }), + ).rejects.toThrow('lost release fence reply'); + lostReply.mockRestore(); + await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityClaimToken: 'new' } }); + expect( + await methods.claimAgentBackgroundToolResultBatch({ + ...input('one'), + deliveryClaimToken: 'new', + }), + ).toMatchObject({ status: 'acquired', results: [{ taskId: 'one' }, { taskId: 'two' }] }); + expect(await confirmBatch(input('one'))).toBe(true); + await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).resolves.toMatchObject({ + status: 'claimed', + ownerStatus: 'applied', + }); + }); + + it('cannot discard a follower after a root acquired its receipt', async () => { + await ready('one'); + await ready('two'); + await methods.claimAgentBackgroundToolResultBatch(input('one')); + await expect( + methods.retireAgentTriggerDelivery({ + deliveryKey: 'two', + sourceId, + settledAt: START, + reason: 'discarded', + onlyIfUnclaimed: true, + requireTransition: true, + }), + ).resolves.toBe(false); + expect(await methods.claimAgentBackgroundToolResultBatch(input('one'))).toMatchObject({ + status: 'acquired', + results: [{ taskId: 'one' }, { taskId: 'two' }], + }); + }); + + it('keeps rollout-default receipts visible to old manual polling after late projection', async () => { + await ready('one', { + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + }); + await methods.claimAgentBackgroundToolResults({ ...input('one'), claimId: 'one' }); + const oldLookup = await Delivery.findOne({ + user, + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + 'envelope.target.conversationId': 'conversation', + 'envelope.target.parentMessageId': 'parent', + 'envelope.event.payload.taskId': 'one', + }) + .select('+backgroundToolResult') + .lean(); + expect(oldLookup?.backgroundToolResult?.resultClaim?.claimId).toBe('one'); + }); + + it('retains frozen membership across an ambiguous POST and a definitely failed retry', async () => { + await ready('one'); + await ready('two'); + const batch = await methods.claimAgentBackgroundToolResultBatch(input('one')); + if (batch.status !== 'acquired') throw new Error('Expected acquired batch'); + const owner = { ...input('one'), batchId: batch.batchId }; + await methods.beginAgentBackgroundToolResultBatchDispatch({ ...owner, dispatchId: 'first' }); + await ready('late'); + await methods.beginAgentBackgroundToolResultBatchDispatch({ ...owner, dispatchId: 'retry' }); + expect( + await methods.releaseAgentBackgroundToolResultClaims({ + ...owner, + claimId: 'one', + dispatchId: 'retry', + }), + ).toBe(false); + expect(await methods.claimAgentBackgroundToolResultBatch(input('one'))).toEqual(batch); + expect(await methods.confirmAgentBackgroundToolResultBatch(owner)).toBe(true); + expect( + await methods.getAgentBackgroundToolResultClaim({ ...input('late'), taskId: 'late' }), + ).toBeNull(); + expect((await methods.claimAgentBackgroundToolResultBatch(input('late'))).status).toBe( + 'acquired', + ); + }); + + it('releases only a proven first handoff failure', async () => { + await ready('one'); + await ready('two'); + const batch = await methods.claimAgentBackgroundToolResultBatch(input('one')); + if (batch.status !== 'acquired') throw new Error('Expected acquired batch'); + const owner = { ...input('one'), batchId: batch.batchId }; + await methods.beginAgentBackgroundToolResultBatchDispatch({ ...owner, dispatchId: 'first' }); + expect( + await methods.releaseAgentBackgroundToolResultClaims({ + ...owner, + claimId: 'one', + dispatchId: 'different-attempt', + }), + ).toBe(false); + expect( + await methods.releaseAgentBackgroundToolResultClaims({ + ...owner, + claimId: 'one', + dispatchId: 'first', + }), + ).toBe(true); + }); + + it.each(['receipts', 'plan'] as const)( + 'fences a predecessor already paused before %s cleanup', + async (phase) => { + await ready('one'); + await ready('two'); + await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityClaimToken: 'old' } }); + const original = await methods.claimAgentBackgroundToolResultBatch({ + ...input('one'), + deliveryClaimToken: 'old', + }); + if (original.status !== 'acquired') throw new Error('Expected batch'); + let pause: () => void = () => undefined; + let resume: () => void = () => undefined; + const paused = new Promise((resolve) => { + pause = resolve; + }); + const barrier = new Promise((resolve) => { + resume = resolve; + }); + let intercept = true; + const one = Delivery.updateOne.bind(Delivery); + const many = Delivery.updateMany.bind(Delivery); + const block = (query: ReturnType) => { + const execute = query.exec.bind(query); + jest.spyOn(query, 'exec').mockImplementationOnce(async () => { + pause(); + await barrier; + return execute(); + }); + return query; + }; + const spy = + phase === 'plan' + ? jest.spyOn(Delivery, 'updateOne').mockImplementation((...args) => { + const query = one(...args); + if ( + intercept && + JSON.stringify(query.getUpdate()).includes('"backgroundToolResultBatch":1') + ) { + intercept = false; + return block(query); + } + return query; + }) + : jest.spyOn(Delivery, 'updateMany').mockImplementation((...args) => { + const query = many(...args); + if ( + intercept && + JSON.stringify(query.getUpdate()).includes('"backgroundToolResult.resultClaim":1') + ) { + intercept = false; + return block(query); + } + return query; + }); + const stale = methods.releaseAgentBackgroundToolResultClaims({ + ...input('one'), + batchId: original.batchId, + claimId: 'one', + deliveryClaimToken: 'old', + }); + await paused; + await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityClaimToken: 'new' } }); + const successor = await methods.claimAgentBackgroundToolResultBatch({ + ...input('one'), + deliveryClaimToken: 'new', + }); + if (successor.status !== 'acquired') throw new Error('Expected successor'); + expect(successor.batchId).not.toBe(original.batchId); + resume(); + expect(await stale).toBe(false); + spy.mockRestore(); + expect( + await methods.confirmAgentBackgroundToolResultBatch({ + ...input('one'), + batchId: successor.batchId, + }), + ).toBe(true); + await expect( + methods.claimAgentBackgroundToolResultBatch(input('two')), + ).resolves.toMatchObject({ status: 'claimed', ownerStatus: 'applied' }); + }, + ); + + it('does not let a resumed cleanup snapshot erase a replacement batch without a queue token', async () => { + await ready('one'); + await ready('two'); + const old = await methods.claimAgentBackgroundToolResultBatch(input('one')); + if (old.status !== 'acquired') throw new Error('Expected old batch'); + await Delivery.updateOne( + { deliveryKey: 'one' }, + { $set: { 'backgroundToolResultBatch.releasing': true } }, + ); + let pause: () => void = () => undefined; + let resume: () => void = () => undefined; + const paused = new Promise((resolve) => { + pause = resolve; + }); + const barrier = new Promise((resolve) => { + resume = resolve; + }); + const update = Delivery.findOneAndUpdate.bind(Delivery); + const cleanup = jest.spyOn(Delivery, 'findOneAndUpdate').mockImplementationOnce((...args) => { + const query = update(...args); + const execute = query.exec.bind(query); + jest.spyOn(query, 'exec').mockImplementationOnce(async () => { + pause(); + await barrier; + return execute(); + }); + return query; + }); + const predecessor = methods.claimAgentBackgroundToolResultBatch(input('one')); + await paused; + expect( + await methods.releaseAgentBackgroundToolResultClaims({ + ...input('one'), + claimId: 'one', + batchId: old.batchId, + }), + ).toBe(true); + const current = await methods.claimAgentBackgroundToolResultBatch(input('one')); + if (current.status !== 'acquired') throw new Error('Expected replacement batch'); + resume(); + expect(await predecessor).toEqual({ status: 'not_ready' }); + cleanup.mockRestore(); + expect(await methods.claimAgentBackgroundToolResultBatch(input('one'))).toEqual(current); + }); + + it('clears a collecting-only plan retired by a manual poll after its collector crashes', async () => { + await ready('one'); + await ready('two'); + const write = Delivery.updateOne.bind(Delivery); + const crash = jest.spyOn(Delivery, 'updateOne').mockImplementation((...args) => { + const query = write(...args); + if (JSON.stringify(query.getUpdate()).includes('backgroundToolResult.resultClaim')) + throw new Error('crash before root claim'); + return query; + }); + await expect(methods.claimAgentBackgroundToolResultBatch(input('one'))).rejects.toThrow( + 'crash before root claim', + ); + crash.mockRestore(); + const collecting = await Delivery.findOne({ deliveryKey: 'one' }) + .select('+backgroundToolResultBatch +backgroundToolResult') + .lean(); + expect(collecting?.backgroundToolResultBatch).toMatchObject({ candidates: ['one', 'two'] }); + expect(collecting?.backgroundToolResult?.resultClaim).toBeUndefined(); + expect( + await methods.retireAgentTriggerDelivery({ + deliveryKey: 'one', + sourceId, + settledAt: START, + reason: 'manual poll', + onlyIfUnclaimed: true, + requireTransition: true, + }), + ).toBe(true); + const retired = await Delivery.findOne({ deliveryKey: 'one' }) + .select('+backgroundToolResultBatch') + .lean(); + expect(retired).toMatchObject({ + status: 'succeeded', + expiresAt: new Date(START.getTime() + 90 * 24 * 60 * 60_000), + }); + expect(retired).not.toHaveProperty('backgroundToolResultBatch'); + expect( + await methods.getAgentBackgroundToolResultClaim({ ...input('two'), taskId: 'two' }), + ).toBeNull(); + expect((await methods.claimAgentBackgroundToolResultBatch(input('two'))).status).toBe( + 'acquired', + ); + }); + + it('prevents a stale collector from acquiring siblings after empty-plan retirement', async () => { + await ready('one'); + await ready('two'); + let pause: () => void = () => undefined; + let resume: () => void = () => undefined; + const paused = new Promise((resolve) => { + pause = resolve; + }); + const barrier = new Promise((resolve) => { + resume = resolve; + }); + const write = Delivery.updateOne.bind(Delivery); + const rootClaim = jest.spyOn(Delivery, 'updateOne').mockImplementation((...args) => { + const query = write(...args); + if (JSON.stringify(query.getUpdate()).includes('backgroundToolResult.resultClaim')) { + const execute = query.exec.bind(query); + jest.spyOn(query, 'exec').mockImplementationOnce(async () => { + pause(); + await barrier; + return execute(); + }); + } + return query; + }); + const collecting = methods.claimAgentBackgroundToolResultBatch(input('one')); + await paused; + expect( + await methods.retireAgentTriggerDelivery({ + deliveryKey: 'one', + sourceId, + settledAt: START, + reason: 'manual poll', + onlyIfUnclaimed: true, + }), + ).toBe(true); + resume(); + expect(await collecting).toEqual({ status: 'not_ready' }); + rootClaim.mockRestore(); + expect( + await methods.getAgentBackgroundToolResultClaim({ ...input('two'), taskId: 'two' }), + ).toBeNull(); + expect(await Delivery.findOne({ deliveryKey: 'one' }).lean()).toHaveProperty('expiresAt'); + }); + + it('retains an owned collecting plan before membership publication', async () => { + await ready('one'); + await ready('two'); + const crash = jest.spyOn(Delivery, 'updateMany').mockImplementationOnce(() => { + throw new Error('crash before sibling claims'); + }); + await expect(methods.claimAgentBackgroundToolResultBatch(input('one'))).rejects.toThrow( + 'crash before sibling claims', + ); + crash.mockRestore(); + expect( + await methods.retireAgentTriggerDelivery({ + deliveryKey: 'one', + sourceId, + settledAt: START, + reason: 'manual poll', + onlyIfUnclaimed: true, + }), + ).toBe(false); + await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityStatus: 'dead' } }); + expect( + await methods.retireAgentTriggerDelivery({ + deliveryKey: 'one', + sourceId, + settledAt: START, + reason: 'recovery', + onlyIfDead: true, + }), + ).toBe(true); + const retired = await Delivery.findOne({ deliveryKey: 'one' }) + .select('+backgroundToolResultBatch') + .lean(); + expect(retired?.backgroundToolResultBatch).toBeDefined(); + expect(retired).not.toHaveProperty('expiresAt'); + expect( + await methods.releaseAgentBackgroundToolResultClaims({ + ...input('one'), + claimId: 'one', + recoveryFenced: true, + }), + ).toBe(true); + expect(await Delivery.findOne({ deliveryKey: 'one' }).lean()).toHaveProperty('expiresAt'); + }); + + it('retains a retired owner across the TTL horizon until every claim is released', async () => { + await ready('one'); + await ready('two'); + const batch = await methods.claimAgentBackgroundToolResultBatch(input('one')); + if (batch.status !== 'acquired') throw new Error('Expected batch'); + await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityStatus: 'dead' } }); + expect( + await methods.retireAgentTriggerDelivery({ + deliveryKey: 'one', + sourceId, + reason: 'recovery', + onlyIfDead: true, + settledAt: START, + }), + ).toBe(true); + // The recovering process died before the generation fence or result release. + expect(await Delivery.findOne({ deliveryKey: 'one' }).lean()).not.toHaveProperty('expiresAt'); + const future = new Date(Date.now() + 91 * 24 * 60 * 60_000); + await Delivery.deleteMany({ expiresAt: { $lte: future } }); + await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).resolves.toMatchObject({ + status: 'claimed', + ownerStatus: 'recoverable', + }); + expect( + await methods.releaseAgentBackgroundToolResultClaims({ + ...input('one'), + claimId: 'one', + batchId: batch.batchId, + recoveryFenced: true, + }), + ).toBe(true); + expect(await Delivery.findOne({ deliveryKey: 'one' }).lean()).toHaveProperty('expiresAt'); + expect((await methods.claimAgentBackgroundToolResultBatch(input('two'))).status).toBe( + 'acquired', + ); + }); + + it('retains applied ownership until every receipt has durable proof of admission', async () => { + await ready('one'); + await ready('two'); + const batch = await methods.claimAgentBackgroundToolResultBatch(input('one')); + if (batch.status !== 'acquired') throw new Error('Expected batch'); + await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityStatus: 'dead' } }); + await methods.retireAgentTriggerDelivery({ + deliveryKey: 'one', + sourceId, + reason: 'recovery', + onlyIfDead: true, + settledAt: START, + }); + const lostCopy = jest.spyOn(Delivery, 'updateMany').mockImplementationOnce(() => { + throw new Error('crash before receipt proof copy'); + }); + await expect(confirmBatch(input('one'))).rejects.toThrow('crash before receipt proof copy'); + lostCopy.mockRestore(); + expect(await Delivery.findOne({ deliveryKey: 'one' }).lean()).not.toHaveProperty('expiresAt'); + await Delivery.deleteMany({ + expiresAt: { $lte: new Date(Date.now() + 91 * 24 * 60 * 60_000) }, + }); + expect(await Delivery.exists({ deliveryKey: 'one' })).not.toBeNull(); + // The durable root proof remains sufficient after the native key's horizon. + await expect(methods.claimAgentBackgroundToolResultBatch(input('one'))).resolves.toMatchObject({ + status: 'claimed', + ownerStatus: 'applied', + }); + const root = await Delivery.findOne({ deliveryKey: 'one' }) + .select('+backgroundToolResultBatch') + .lean(); + expect(root?.backgroundToolResultBatch?.proofCopiedAt).toBeInstanceOf(Date); + expect(root?.expiresAt).toBeInstanceOf(Date); + await Delivery.deleteOne({ deliveryKey: 'one' }); + await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).resolves.toMatchObject({ + status: 'claimed', + ownerStatus: 'applied', + }); + }); + + it.each(['before-copy', 'partial-copy', 'before-finish'] as const)( + 'repairs a retired owner from its applied follower after %s failure', + async (phase) => { + await ready('one'); + await ready('two'); + await ready('three'); + const acquired = await methods.claimAgentBackgroundToolResultBatch(input('one')); + if (acquired.status !== 'acquired') throw new Error('Expected batch'); + await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityStatus: 'dead' } }); + await methods.retireAgentTriggerDelivery({ + deliveryKey: 'one', + sourceId, + reason: 'recovery', + onlyIfDead: true, + settledAt: START, + }); + const copy = Delivery.updateMany.bind(Delivery); + const finish = Delivery.updateOne.bind(Delivery); + const failure = + phase === 'before-finish' + ? jest.spyOn(Delivery, 'updateOne').mockImplementation((...args) => { + const query = finish(...args); + if (JSON.stringify(query.getUpdate()).includes('proofCopiedAt')) + throw new Error('proof completion failed'); + return query; + }) + : jest.spyOn(Delivery, 'updateMany').mockImplementationOnce((...args) => { + if (phase === 'before-copy') throw new Error('proof copy failed'); + const query = copy( + { ...args[0], deliveryKey: { $in: ['one', 'two'] } }, + args[1], + args[2], + ); + const execute = query.exec.bind(query); + jest.spyOn(query, 'exec').mockImplementationOnce(async () => { + await execute(); + throw new Error('partial proof copy failed'); + }); + return query; + }); + await expect(confirmBatch(input('one'))).rejects.toThrow(); + failure.mockRestore(); + expect(await Delivery.findOne({ deliveryKey: 'one' }).lean()).not.toHaveProperty('expiresAt'); + const cannotRepair = jest.spyOn(Delivery, 'updateMany').mockImplementationOnce(() => { + throw new Error('retry cannot copy'); + }); + await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).rejects.toThrow( + 'retry cannot copy', + ); + cannotRepair.mockRestore(); + await expect( + methods.claimAgentBackgroundToolResultBatch(input('two')), + ).resolves.toMatchObject({ status: 'claimed', ownerStatus: 'applied' }); + const repaired = await Delivery.findOne({ deliveryKey: 'one' }) + .select('+backgroundToolResultBatch') + .lean(); + expect(repaired?.backgroundToolResultBatch?.proofCopiedAt).toBeInstanceOf(Date); + expect(repaired?.expiresAt).toBeInstanceOf(Date); + for (const taskId of ['one', 'two', 'three']) { + expect( + await methods.getAgentBackgroundToolResultClaim({ ...input(taskId), taskId }), + ).toMatchObject({ appliedAt: expect.any(Date) }); + } + await Delivery.deleteOne({ deliveryKey: 'one' }); + await expect( + methods.claimAgentBackgroundToolResultBatch(input('three')), + ).resolves.toMatchObject({ status: 'claimed', ownerStatus: 'applied' }); + }, + ); + + it('does not publish a late collecting plan onto an already retired owner', async () => { + await ready('one'); + await ready('two'); + let pause: () => void = () => undefined; + let resume: () => void = () => undefined; + const paused = new Promise((resolve) => { + pause = resolve; + }); + const barrier = new Promise((resolve) => { + resume = resolve; + }); + const update = Delivery.updateOne.bind(Delivery); + const publication = jest.spyOn(Delivery, 'updateOne').mockImplementationOnce((...args) => { + const query = update(...args); + const execute = query.exec.bind(query); + jest.spyOn(query, 'exec').mockImplementationOnce(async () => { + pause(); + await barrier; + return execute(); + }); + return query; + }); + const stale = methods.claimAgentBackgroundToolResultBatch(input('one')); + await paused; + expect( + await methods.retireAgentTriggerDelivery({ + deliveryKey: 'one', + sourceId, + reason: 'discarded', + onlyIfUnclaimed: true, + settledAt: START, + }), + ).toBe(true); + const collectingCrash = jest.spyOn(Delivery, 'updateMany').mockImplementationOnce(() => { + throw new Error('crash after late publication'); + }); + resume(); + await expect(stale).resolves.toEqual({ status: 'not_ready' }); + publication.mockRestore(); + collectingCrash.mockRestore(); + expect( + await Delivery.findOne({ deliveryKey: 'one' }).select('+backgroundToolResultBatch').lean(), + ).not.toHaveProperty('backgroundToolResultBatch'); + expect( + await methods.getAgentBackgroundToolResultClaim({ ...input('two'), taskId: 'two' }), + ).toBeNull(); + }); + + it('cannot begin dispatch after the owner is retired, even without a queue token', async () => { + await ready('one'); + await ready('two'); + const batch = await methods.claimAgentBackgroundToolResultBatch(input('one')); + if (batch.status !== 'acquired') throw new Error('Expected batch'); + await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityStatus: 'dead' } }); + await methods.retireAgentTriggerDelivery({ + deliveryKey: 'one', + sourceId, + reason: 'recovery', + onlyIfDead: true, + settledAt: START, + }); + expect( + await methods.beginAgentBackgroundToolResultBatchDispatch({ + ...input('one'), + batchId: batch.batchId, + dispatchId: 'late', + }), + ).toBe(false); + }); + + it('does not let an expired lease release its successor preparation', async () => { + await ready('one'); + await ready('two'); + await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityClaimToken: 'current' } }); + expect( + ( + await methods.claimAgentBackgroundToolResultBatch({ + ...input('one'), + deliveryClaimToken: 'current', + }) + ).status, + ).toBe('acquired'); + await expect( + methods.releaseAgentBackgroundToolResultClaims({ + ...input('one'), + claimId: 'one', + deliveryClaimToken: 'expired', + }), + ).resolves.toBe(false); + expect((await methods.claimAgentBackgroundToolResultBatch(input('two'))).status).toBe( + 'claimed', + ); + expect(await confirmBatch(input('one'))).toBe(true); + await expect( + methods.releaseAgentBackgroundToolResultClaims({ + ...input('one'), + claimId: 'one', + deliveryClaimToken: 'current', + }), + ).resolves.toBe(false); + await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).resolves.toMatchObject({ + status: 'claimed', + ownerStatus: 'applied', + }); + }); + + it('serializes definite release against an admission confirmation', async () => { + await ready('one'); + await ready('two'); + await methods.claimAgentBackgroundToolResultBatch(input('one')); + const [released, confirmed] = await Promise.all([ + methods.releaseAgentBackgroundToolResultClaims({ ...input('one'), claimId: 'one' }), + confirmBatch(input('one')), + ]); + expect(Number(released) + Number(confirmed)).toBe(1); + }); + + it('uses bounded selection, including a batch size of one', async () => { + await ready('one'); + await ready('two'); + expect( + await methods.claimAgentBackgroundToolResultBatch({ ...input('one'), limit: 1 }), + ).toMatchObject({ status: 'acquired', results: [{ taskId: 'one' }] }); + expect( + await methods.getAgentBackgroundToolResultClaim({ ...input('two'), taskId: 'two' }), + ).toBeNull(); + }); + + it('keeps private batch state out of delivery reads and erases it with conversation results', async () => { + await ready('one'); + await ready('two'); + await methods.claimAgentBackgroundToolResultBatch(input('one')); + expect(await methods.getAgentTriggerDelivery('one')).not.toHaveProperty( + 'backgroundToolResultBatch', + ); + await methods.eraseAgentTriggerDeliveryConversationResults(user, ['conversation']); + const row = await Delivery.findOne({ deliveryKey: 'one' }) + .select('+backgroundToolResultBatch +backgroundToolResult') + .lean(); + expect(row).not.toHaveProperty('backgroundToolResultBatch'); + expect(row).not.toHaveProperty('backgroundToolResult'); + }); +}); + +describe('background receipt batch contention', () => { + async function seed( + groups = 1, + capability = AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3, + ) { + const user = new mongoose.Types.ObjectId(); + const claims: Parameters[0][] = []; + for (let group = 0; group < groups; group++) { + for (let task = 0; task < 4; task++) { + const key = `group-${group}-task-${task}`; + await methods.enqueueAgentTriggerDelivery( + enqueueInput({ + deliveryKey: key, + orderingKey: key, + user, + requiredWorkerCapability: capability, + envelope: { + event: { + type: 'background-tool.completion', + source: { id: 'background-tool-completion', type: 'internal' }, + payload: { taskId: key, toolCallId: key, toolName: 'tool' }, + }, + target: { + conversationId: `conversation-${group}`, + parentMessageId: 'parent', + agentId: 'agent', + }, + }, + }), + ); + await methods.persistAgentBackgroundToolResult({ + deliveryKey: key, + sourceId: 'background-tool-completion', + result: { status: 'completed', output: key, settledAt: START }, + }); + claims.push({ + deliveryKey: key, + sourceId: 'background-tool-completion', + userId: user.toString(), + tenantId: 'tenant-1', + conversationId: `conversation-${group}`, + parentMessageId: 'parent', + agentId: 'agent', + limit: 8, + maxMetadataChars: 16000, + }); + } + } + return claims; + } + + it('makes progress when two workers cross the collecting barrier together', async () => { + const claims = await seed(); + let arrivals = 0; + let open: () => void = () => undefined; + const barrier = new Promise((resolve) => { + open = resolve; + }); + const updateMany = Delivery.updateMany.bind(Delivery); + const race = jest.spyOn(Delivery, 'updateMany').mockImplementation((...args) => { + const query = updateMany(...args); + const execute = query.exec.bind(query); + jest.spyOn(query, 'exec').mockImplementationOnce(async () => { + if (++arrivals === 2) open(); + await barrier; + return execute(); + }); + return query; + }); + const batches = await Promise.all( + claims.slice(0, 2).map((claim) => methods.claimAgentBackgroundToolResultBatch(claim)), + ); + race.mockRestore(); + const consumed: string[] = []; + for (let index = 0; index < batches.length; index++) { + const batch = batches[index]!; + if (batch.status !== 'acquired') continue; + consumed.push(...batch.results.map((result) => result.taskId)); + await confirmBatch(claims[index]!); + } + for (const claim of claims) { + const batch = await methods.claimAgentBackgroundToolResultBatch(claim); + if (batch.status === 'acquired' && !consumed.includes(claim.deliveryKey)) { + consumed.push(...batch.results.map((result) => result.taskId)); + await confirmBatch(claim); + } + } + expect(new Set(consumed).size).toBe(4); + expect(consumed).toHaveLength(4); + }); + + it('reduces 8 conversations by 4 ready receipts without duplicates or stranded results', async () => { + const baselineClaims = await seed( + 8, + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + ); + let baselineTurns = 0; + for (const claim of baselineClaims) { + const batch = await methods.claimAgentBackgroundToolResults({ + ...claim, + claimId: claim.deliveryKey, + }); + if (batch.status === 'acquired') baselineTurns++; + } + await Promise.all([Delivery.deleteMany({}), LaneSequence.deleteMany({})]); + const claims = await seed(8); + const started = Date.now(); + const consumed: string[] = []; + const latency: number[] = []; + let turns = 0; + // One ready root per conversation, as with a bounded worker pool behind + // a busy parent. A separate barrier test covers same-conversation overlap. + for (let round = 0; round < 4; round++) { + await Promise.all( + Array.from({ length: 8 }, async (_, group) => { + const claim = claims[group * 4 + round]!; + const batch = await methods.claimAgentBackgroundToolResultBatch(claim); + if (batch.status !== 'acquired') return; + turns++; + consumed.push(...batch.results.map((result) => result.taskId)); + await confirmBatch(claim); + latency.push(Date.now() - started); + }), + ); + } + const unique = new Set(consumed).size; + const report = { + uniqueResults: unique, + duplicates: consumed.length - unique, + strandedResults: 32 - unique, + baselineTurns, + turns, + admissionLatencyMs: Math.max(...latency), + }; + console.info('BACKGROUND_RECEIPT_STRESS', JSON.stringify(report)); + expect(report).toMatchObject({ + uniqueResults: 32, + duplicates: 0, + strandedResults: 0, + baselineTurns: 32, + turns: 8, + }); + }); +}); diff --git a/packages/data-schemas/src/methods/triggerDelivery.ts b/packages/data-schemas/src/methods/triggerDelivery.ts index 81d4d7140c..22cab0aded 100644 --- a/packages/data-schemas/src/methods/triggerDelivery.ts +++ b/packages/data-schemas/src/methods/triggerDelivery.ts @@ -1,4 +1,5 @@ -import { createHash } from 'node:crypto'; +import { createHash, randomUUID } from 'node:crypto'; +import { backgroundResultMetadata } from 'librechat-data-provider'; import type { FilterQuery, Model, Types } from 'mongoose'; import type { AgentBackgroundToolResultReceipt, @@ -17,8 +18,10 @@ import type { IAgentTriggerUserPurge, IAgentTriggerUserPurgeDocument, } from '~/types/triggerDelivery'; +import type { MessageMethods } from './message'; import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3, AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1, @@ -38,6 +41,10 @@ const DEFAULT_PURGE_RECOVERY_LIMIT = 25; const MAX_PURGE_RECOVERY_LIMIT = 200; const HISTORY_LIMIT = 64; const MAX_BATCH_SIZE = 8; +const BACKGROUND_RECEIPT_CAPABILITIES = [ + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3, +]; const MAX_BATCH_BYTES = 512 * 1024; /** Bounds the claim's compare-and-swap retries. Each lost round means another * worker claimed the row this one read, so the queue is making progress. */ @@ -167,6 +174,48 @@ export type AgentBackgroundToolResultClaim = }>; }; +export interface AgentBackgroundToolResultBatchInput { + deliveryKey: string; + sourceId: string; + userId: string; + tenantId?: string; + conversationId: string; + parentMessageId: string; + agentId: string; + limit: number; + maxMetadataChars: number; + deliveryClaimToken?: string; +} + +export type AgentBackgroundToolResultBatchClaim = + | { status: 'legacy' } + | { status: 'not_ready'; waitingForResult?: true } + | { + status: 'claimed'; + claimId: string; + batchId: string; + ownerStatus: 'pending' | 'applied' | 'recoverable'; + } + | (Extract & { batchId: string }); + +export type AgentBackgroundToolResultBatchOwnerInput = Omit< + AgentBackgroundToolResultBatchInput, + 'limit' | 'maxMetadataChars' | 'agentId' +> & { agentId?: string }; + +export type AgentBackgroundToolResultBatchReleaseInput = { + sourceId: string; + userId: string; + conversationId: string; + parentMessageId: string; + claimId: string; + deliveryClaimToken?: string; + batchId?: string; + dispatchId?: string; + /** Retirement and generation-claim CAS proved that no admission can occur. */ + recoveryFenced?: true; +}; + export interface AgentTriggerDeliveryFence { id: string; workerId: string; @@ -414,13 +463,21 @@ export interface AgentTriggerDeliveryMethods { claimId: string; limit?: number; }) => Promise; - releaseAgentBackgroundToolResultClaims: (input: { - sourceId: string; - userId: string; - conversationId: string; - parentMessageId: string; - claimId: string; - }) => Promise; + releaseAgentBackgroundToolResultClaims: ( + input: AgentBackgroundToolResultBatchReleaseInput, + ) => Promise; + claimAgentBackgroundToolResultBatch: ( + input: AgentBackgroundToolResultBatchInput, + ) => Promise; + getAgentBackgroundToolResultBatch: ( + input: AgentBackgroundToolResultBatchOwnerInput, + ) => Promise; + beginAgentBackgroundToolResultBatchDispatch: ( + input: AgentBackgroundToolResultBatchOwnerInput & { batchId: string; dispatchId: string }, + ) => Promise; + confirmAgentBackgroundToolResultBatch: ( + input: AgentBackgroundToolResultBatchOwnerInput & { batchId: string }, + ) => Promise; settleAgentTriggerHandlingOutcome: ( input: SettleAgentTriggerHandlingOutcomeInput, ) => Promise; @@ -628,6 +685,7 @@ function requireClaim(delivery: IAgentTriggerDelivery | null): AgentTriggerDeliv export function createAgentTriggerDeliveryMethods( mongoose: typeof import('mongoose'), deps: { + releaseBatchProjections?: MessageMethods['releaseBackgroundToolResultClaims']; purgeQueuedTurnsForUser?: (user: string | Types.ObjectId) => Promise; } = {}, ): AgentTriggerDeliveryMethods { @@ -1125,6 +1183,8 @@ export function createAgentTriggerDeliveryMethods( input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 && input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 && + input.requiredWorkerCapability !== + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3 && input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 && input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1 && input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2 @@ -2182,38 +2242,70 @@ export function createAgentTriggerDeliveryMethods( }, }; })(); - const retired = await Delivery() - .findOneAndUpdate( - { - deliveryKey: input.deliveryKey, - 'envelope.event.source.type': 'internal', - 'envelope.event.source.id': input.sourceId, - ...statusFence, - }, - { - $set: { - status: 'succeeded', - result, - settledAt: input.settledAt, - expiresAt: new Date(input.settledAt.getTime() + SUCCESS_RETENTION_MS), - laneCleanupPendingAt: input.settledAt, + const retire = (retainBatch: boolean, discardEmptyPlan = false) => + Delivery() + .findOneAndUpdate( + { + deliveryKey: input.deliveryKey, + 'envelope.event.source.type': 'internal', + 'envelope.event.source.id': input.sourceId, + ...statusFence, + ...(discardEmptyPlan && { + backgroundToolResultBatch: { $exists: true }, + 'backgroundToolResultBatch.members.0': { $exists: false }, + 'backgroundToolResult.resultClaim.claimId': { $ne: input.deliveryKey }, + }), + ...(!discardEmptyPlan && + retainBatch && { + backgroundToolResultBatch: { $exists: true }, + 'backgroundToolResultBatch.proofCopiedAt': { $exists: false }, + }), + ...(!discardEmptyPlan && + !retainBatch && { + $and: [ + { + $or: [ + { backgroundToolResultBatch: { $exists: false } }, + { 'backgroundToolResultBatch.proofCopiedAt': { $exists: true } }, + ], + }, + ], + }), + ...(input.onlyIfUnclaimed === true + ? { 'backgroundToolResult.resultClaim': { $exists: false } } + : {}), }, - $unset: { - leaseBy: 1, - leaseUntil: 1, - claimToken: 1, - capabilityStatus: 1, - claimAvailableAt: 1, - capabilityLeaseBy: 1, - capabilityLeaseUntil: 1, - capabilityClaimToken: 1, - lastError: 1, + { + $set: { + status: 'succeeded', + result, + settledAt: input.settledAt, + ...(!retainBatch && { + expiresAt: new Date(input.settledAt.getTime() + SUCCESS_RETENTION_MS), + }), + laneCleanupPendingAt: input.settledAt, + }, + $unset: { + leaseBy: 1, + leaseUntil: 1, + claimToken: 1, + capabilityStatus: 1, + claimAvailableAt: 1, + capabilityLeaseBy: 1, + capabilityLeaseUntil: 1, + capabilityClaimToken: 1, + lastError: 1, + ...(retainBatch && { expiresAt: 1 }), + ...(discardEmptyPlan && { backgroundToolResultBatch: 1 }), + }, }, - }, - { new: true }, - ) - .select('_id orderingKey laneCleanupPendingAt') - .lean>(); + { new: true }, + ) + .select('_id orderingKey laneCleanupPendingAt') + .lean>(); + // A retired owner is its followers' recovery and frozen-membership record. + // Its TTL cannot start while release or receipt-proof copying is incomplete. + const retired = (await retire(false)) ?? (await retire(false, true)) ?? (await retire(true)); if (retired?._id != null) { try { await fulfillLaneCleanupRequest(retired); @@ -2268,7 +2360,7 @@ export function createAgentTriggerDeliveryMethods( requiredWorkerCapability: { $in: [ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, - AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + ...BACKGROUND_RECEIPT_CAPABILITIES, ], }, 'envelope.event.source.type': 'internal', @@ -2312,7 +2404,7 @@ export function createAgentTriggerDeliveryMethods( requiredWorkerCapability: { $in: [ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, - AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + ...BACKGROUND_RECEIPT_CAPABILITIES, ], }, 'envelope.event.source.type': 'internal', @@ -2366,7 +2458,7 @@ export function createAgentTriggerDeliveryMethods( 'envelope.target.conversationId': input.conversationId, ...(input.taskId != null && { 'envelope.event.payload.taskId': input.taskId }), /** Legacy rows keep results only on the parent message. */ - requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES }, }; const read = (filter: FilterQuery) => Delivery() @@ -2568,7 +2660,7 @@ export function createAgentTriggerDeliveryMethods( } const identity = { deliveryKey: input.deliveryKey, - requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES }, 'envelope.event.source.type': 'internal', 'envelope.event.source.id': input.sourceId, backgroundToolResultErasedAt: { $exists: false }, @@ -2612,7 +2704,7 @@ export function createAgentTriggerDeliveryMethods( const delivery = await Delivery() .findOne({ deliveryKey: input.deliveryKey, - requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES }, 'envelope.event.source.type': 'internal', 'envelope.event.source.id': input.sourceId, }) @@ -2648,7 +2740,7 @@ export function createAgentTriggerDeliveryMethods( }) { return { user: input.userId, - requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES }, 'envelope.event.source.type': 'internal', 'envelope.event.source.id': input.sourceId, 'envelope.target.conversationId': input.conversationId, @@ -2769,38 +2861,592 @@ export function createAgentTriggerDeliveryMethods( return result == null ? { status: 'not_ready' } : { status: 'acquired', results: [result] }; } - async function releaseAgentBackgroundToolResultClaims(input: { - sourceId: string; - userId: string; - conversationId: string; - parentMessageId: string; - claimId: string; - }): Promise { + async function releaseAgentBackgroundToolResultClaims( + input: AgentBackgroundToolResultBatchReleaseInput, + ): Promise { if ( - input.sourceId.length === 0 || - input.sourceId.length > 256 || - input.userId.length === 0 || - input.conversationId.length === 0 || - input.conversationId.length > 256 || - input.parentMessageId.length === 0 || - input.parentMessageId.length > 256 || - input.claimId.length === 0 || + [ + input.sourceId, + input.userId, + input.conversationId, + input.parentMessageId, + input.claimId, + ].some((value) => value.length === 0 || value.length > 256) || input.claimId.length > 128 ) { throw new TypeError('Invalid background tool result receipt claim release'); } const scope = backgroundResultIdentity(input); + const releaseId = randomUUID(); + const releasing = await Delivery() + .findOneAndUpdate( + { + ...scope, + deliveryKey: input.claimId, + backgroundToolResultBatch: { $exists: true }, + 'backgroundToolResultBatch.appliedAt': { $exists: false }, + ...(input.batchId == null ? {} : { 'backgroundToolResultBatch.batchId': input.batchId }), + ...(input.deliveryClaimToken == null + ? {} + : { capabilityClaimToken: input.deliveryClaimToken }), + ...(input.recoveryFenced === true + ? {} + : { + $or: [ + { 'backgroundToolResultBatch.releasing': true }, + { 'backgroundToolResultBatch.dispatchCount': 0 }, + ...(input.dispatchId == null + ? [] + : [ + { + 'backgroundToolResultBatch.dispatchCount': 1, + 'backgroundToolResultBatch.dispatchId': input.dispatchId, + }, + ]), + ], + }), + }, + { + $set: { + 'backgroundToolResultBatch.releasing': true, + 'backgroundToolResultBatch.releaseId': releaseId, + }, + }, + { new: true, timestamps: false }, + ) + .select('status +backgroundToolResultBatch') + .lean>(); + if ( + releasing == null && + (await Delivery().exists({ + ...scope, + deliveryKey: input.claimId, + backgroundToolResultBatch: { $exists: true }, + })) != null + ) + return false; + const batch = releasing?.backgroundToolResultBatch; + const batchId = batch?.batchId ?? input.batchId; + const receiptScope = { + ...scope, + 'backgroundToolResult.resultClaim.claimId': input.claimId, + 'backgroundToolResult.resultClaim.batchId': batchId ?? null, + }; await Delivery().updateMany( - { ...scope, 'backgroundToolResult.resultClaim.claimId': input.claimId }, + { ...receiptScope, 'backgroundToolResult.resultClaim.appliedAt': { $exists: false } }, { $unset: { 'backgroundToolResult.resultClaim': 1 } }, { timestamps: false }, ); - return ( + if ((await Delivery().exists(receiptScope)) != null) return false; + if (batch != null) { + if (deps.releaseBatchProjections == null) + throw new Error('Background batch projection cleanup is unavailable'); + if ( + !(await deps.releaseBatchProjections({ + userId: input.userId, + conversationId: input.conversationId, + messageId: input.parentMessageId, + kind: 'wakeup', + claimId: input.claimId, + batchId: batch.batchId, + allowMissingMessage: true, + })) + ) + return false; + const clear = (retired: boolean) => + Delivery().updateOne( + { + ...scope, + deliveryKey: input.claimId, + 'backgroundToolResultBatch.batchId': batch.batchId, + 'backgroundToolResultBatch.releaseId': releaseId, + 'backgroundToolResultBatch.appliedAt': { $exists: false }, + status: retired ? 'succeeded' : { $ne: 'succeeded' }, + ...(input.deliveryClaimToken == null + ? {} + : { capabilityClaimToken: input.deliveryClaimToken }), + }, + { + $unset: { backgroundToolResultBatch: 1 }, + ...(retired && { $max: { expiresAt: new Date(Date.now() + SUCCESS_RETENTION_MS) } }), + }, + { timestamps: false }, + ); + const cleared = await clear(releasing?.status === 'succeeded'); + if (cleared.matchedCount === 1) return true; + return releasing?.status !== 'succeeded' && (await clear(true)).matchedCount === 1; + } + return true; + } + + function backgroundBatchScope( + input: + | AgentBackgroundToolResultBatchInput + | Omit, + ) { + return { + ...backgroundResultIdentity(input), + tenantId: input.tenantId ?? null, + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3, + 'envelope.event.type': 'background-tool.completion', + }; + } + + /** Each result has its own CAS. Candidates survive a collecting crash; only + * confirmed owners enter the frozen member list used by every retry. */ + async function claimAgentBackgroundToolResultBatch( + input: AgentBackgroundToolResultBatchInput, + ): Promise { + if ( + [ + input.deliveryKey, + input.sourceId, + input.userId, + input.conversationId, + input.parentMessageId, + input.agentId, + ].some((value) => value.length === 0 || value.length > 256) || + input.deliveryKey.length > 128 + ) { + throw new TypeError('Invalid background result batch identity'); + } + if ( + !Number.isSafeInteger(input.limit) || + input.limit < 1 || + !Number.isSafeInteger(input.maxMetadataChars) || + input.maxMetadataChars < 1 + ) { + throw new TypeError('Invalid background result batch bounds'); + } + const scope = backgroundBatchScope(input); + const projection = + 'deliveryKey requiredWorkerCapability status capabilityStatus result +backgroundToolResult +backgroundToolResultBatch'; + type Row = Pick< + IAgentTriggerDelivery, + | 'deliveryKey' + | 'requiredWorkerCapability' + | 'status' + | 'capabilityStatus' + | 'result' + | 'backgroundToolResult' + | 'backgroundToolResultBatch' + > & { envelope?: unknown }; + const readRoot = () => + Delivery() + .findOne({ ...scope, deliveryKey: input.deliveryKey }) + .select(projection + ' envelope') + .lean(); + let root = await readRoot(); + if (root == null) { + const gated = await Delivery().exists({ + user: input.userId, + deliveryKey: input.deliveryKey, + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3, + }); + return { status: gated == null ? 'legacy' : 'not_ready' }; + } + if (root.backgroundToolResultBatch?.appliedAt != null) { + if ( + root.backgroundToolResultBatch.proofCopiedAt == null && + !(await confirmAgentBackgroundToolResultBatch({ + ...input, + batchId: root.backgroundToolResultBatch.batchId, + })) + ) + return { status: 'not_ready' }; + return { + status: 'claimed', + claimId: input.deliveryKey, + batchId: root.backgroundToolResultBatch.batchId, + ownerStatus: 'applied', + }; + } + if (root.backgroundToolResultBatch?.releasing === true) { + const released = await releaseAgentBackgroundToolResultClaims({ + ...input, + claimId: input.deliveryKey, + batchId: root.backgroundToolResultBatch.batchId, + }); + return released ? claimAgentBackgroundToolResultBatch(input) : { status: 'not_ready' }; + } + if (root.backgroundToolResult == null) return { status: 'not_ready', waitingForResult: true }; + const claim = root.backgroundToolResult.resultClaim; + if ( + claim?.claimId === input.deliveryKey && + claim.batchId !== root.backgroundToolResultBatch?.batchId + ) { + await Delivery().updateOne( + { + ...scope, + deliveryKey: input.deliveryKey, + 'backgroundToolResult.resultClaim.claimId': input.deliveryKey, + 'backgroundToolResult.resultClaim.batchId': claim.batchId ?? null, + 'backgroundToolResult.resultClaim.appliedAt': { $exists: false }, + }, + { $unset: { 'backgroundToolResult.resultClaim': 1 } }, + { timestamps: false }, + ); + return { status: 'not_ready' }; + } + if (claim != null && claim.claimId !== input.deliveryKey) { + if ( + root.backgroundToolResultBatch != null && + root.backgroundToolResultBatch.appliedAt == null + ) { + await releaseAgentBackgroundToolResultClaims({ + ...input, + claimId: input.deliveryKey, + batchId: root.backgroundToolResultBatch.batchId, + }); + } + const owner = await Delivery() + .findOne({ ...scope, deliveryKey: claim.claimId }) + .select(projection) + .lean(); + const batch = owner?.backgroundToolResultBatch; + const member = batch?.members?.includes(input.deliveryKey); + if (member && batch?.batchId === claim.batchId && batch?.appliedAt != null) { + // Retirement removed this owner from the queue. Its followers must + // finish the proof copy before they can all settle and abandon it. + if ( + batch.proofCopiedAt == null && + !(await confirmAgentBackgroundToolResultBatch({ + ...input, + deliveryKey: claim.claimId, + batchId: batch.batchId, + })) + ) + return { status: 'not_ready' }; + return { + status: 'claimed', + claimId: claim.claimId, + batchId: claim.batchId!, + ownerStatus: 'applied', + }; + } + if (claim.appliedAt != null) + return { + status: 'claimed', + claimId: claim.claimId, + batchId: claim.batchId!, + ownerStatus: 'applied', + }; + // A stale collector may finish a CAS after another collector froze a subset. + if ( + (batch?.members != null && !member) || + (batch != null && batch.batchId !== claim.batchId) + ) { + await Delivery().updateOne( + { + ...scope, + deliveryKey: input.deliveryKey, + 'backgroundToolResult.resultClaim.claimId': claim.claimId, + 'backgroundToolResult.resultClaim.batchId': claim.batchId ?? null, + 'backgroundToolResult.resultClaim.appliedAt': { $exists: false }, + }, + { $unset: { 'backgroundToolResult.resultClaim': 1 } }, + { timestamps: false }, + ); + return { status: 'not_ready' }; + } + const retired = + owner?.result != null && + typeof owner.result === 'object' && + 'backgroundToolCompletionRetired' in owner.result; + const recoverable = + owner == null || + DEAD_STATUSES.includes(owner.status) || + owner.capabilityStatus === 'dead' || + retired; + return { + status: 'claimed', + claimId: claim.claimId, + batchId: claim.batchId!, + ownerStatus: recoverable ? 'recoverable' : 'pending', + }; + } + if (root.backgroundToolResultBatch == null) { + const candidates = + input.limit === 1 + ? [] + : await Delivery() + .find({ + ...scope, + deliveryKey: { $ne: input.deliveryKey }, + status: { $in: UNDELIVERED_STATUSES }, + capabilityStatus: { $ne: 'dead' }, + backgroundToolResult: { $exists: true }, + 'backgroundToolResult.resultClaim': { $exists: false }, + }) + .select('deliveryKey envelope +backgroundToolResult') + .sort({ createdAt: 1, _id: 1 }) + .limit(Math.min(input.limit, 32) - 1) + .lean(); + const requested = projectBackgroundResult(root); + if (requested == null) return { status: 'not_ready' }; + const metadataCost = (result: NonNullable>) => + JSON.stringify(backgroundResultMetadata(result)).length + 1; + let remaining = input.maxMetadataChars - metadataCost(requested) - 1; + if (remaining < 0) throw new TypeError('Background result metadata exceeds its input budget'); + const batchId = randomUUID(); + const keys = [input.deliveryKey]; + for (const candidate of candidates) { + const result = projectBackgroundResult(candidate); + if (result == null) continue; + const cost = metadataCost(result); + if (cost > remaining) continue; + remaining -= cost; + keys.push(candidate.deliveryKey); + } + const planned = await Delivery().updateOne( + { + ...scope, + deliveryKey: input.deliveryKey, + backgroundToolResultBatch: { $exists: false }, + status: { $in: UNDELIVERED_STATUSES }, + capabilityStatus: { $ne: 'dead' }, + ...(input.deliveryClaimToken == null + ? {} + : { capabilityClaimToken: input.deliveryClaimToken }), + $or: [ + { 'backgroundToolResult.resultClaim': { $exists: false } }, + { 'backgroundToolResult.resultClaim.claimId': input.deliveryKey }, + ], + }, + { $set: { backgroundToolResultBatch: { batchId, dispatchCount: 0, candidates: keys } } }, + { timestamps: false }, + ); + root = + planned.modifiedCount === 1 + ? { ...root, backgroundToolResultBatch: { batchId, dispatchCount: 0, candidates: keys } } + : await readRoot(); + if (root == null) return { status: 'not_ready' }; + } + let batch = root.backgroundToolResultBatch; + if (batch == null) return { status: 'not_ready' }; + if (batch.members == null) { + // Retirement of a collecting-only plan contends with this root CAS. + // No sibling can depend on the owner until its own receipt is claimed. + if (root.backgroundToolResult?.resultClaim?.claimId !== input.deliveryKey) { + const claimedRoot = await Delivery().updateOne( + { + ...scope, + deliveryKey: input.deliveryKey, + status: { $in: UNDELIVERED_STATUSES }, + capabilityStatus: { $ne: 'dead' }, + 'backgroundToolResultBatch.batchId': batch.batchId, + 'backgroundToolResultBatch.releasing': { $ne: true }, + 'backgroundToolResult.resultClaim': { $exists: false }, + ...(input.deliveryClaimToken == null + ? {} + : { capabilityClaimToken: input.deliveryClaimToken }), + }, + { + $set: { + 'backgroundToolResult.resultClaim': { + kind: 'wakeup', + claimId: input.deliveryKey, + claimedAt: new Date(), + batchId: batch.batchId, + }, + }, + }, + { timestamps: false }, + ); + if (claimedRoot.matchedCount !== 1) return { status: 'not_ready' }; + } + await Delivery().updateMany( + { + ...scope, + deliveryKey: { $in: batch.candidates }, + status: { $in: UNDELIVERED_STATUSES }, + capabilityStatus: { $ne: 'dead' }, + backgroundToolResult: { $exists: true }, + 'backgroundToolResult.resultClaim': { $exists: false }, + }, + { + $set: { + 'backgroundToolResult.resultClaim': { + kind: 'wakeup', + claimId: input.deliveryKey, + claimedAt: new Date(), + batchId: batch.batchId, + }, + }, + }, + { timestamps: false }, + ); + const owned = await Delivery() + .find({ + ...scope, + deliveryKey: { $in: batch.candidates }, + 'backgroundToolResult.resultClaim.claimId': input.deliveryKey, + 'backgroundToolResult.resultClaim.batchId': batch.batchId, + }) + .select('deliveryKey') + .lean[]>(); + const owners = new Set(owned.map((row) => row.deliveryKey)); + const members = batch.candidates.filter((key) => owners.has(key)); + if (!owners.has(input.deliveryKey)) { + await releaseAgentBackgroundToolResultClaims({ + ...input, + claimId: input.deliveryKey, + batchId: batch.batchId, + }); + return { status: 'not_ready' }; + } + const frozen = await Delivery().updateOne( + { + ...scope, + deliveryKey: input.deliveryKey, + 'backgroundToolResult.resultClaim.claimId': input.deliveryKey, + 'backgroundToolResultBatch.batchId': batch.batchId, + 'backgroundToolResultBatch.releasing': { $ne: true }, + ...(input.deliveryClaimToken == null + ? {} + : { capabilityClaimToken: input.deliveryClaimToken }), + 'backgroundToolResultBatch.members': { $exists: false }, + }, + { $set: { 'backgroundToolResultBatch.members': members } }, + { timestamps: false }, + ); + if (frozen.modifiedCount === 1) { + batch = { ...batch, members }; + } else { + root = await readRoot(); + batch = root?.backgroundToolResultBatch; + } + } + if (batch?.members == null || !batch.members.includes(input.deliveryKey)) + return { status: 'not_ready' }; + const rows = await Delivery() + .find({ + ...scope, + deliveryKey: { $in: batch.members }, + 'backgroundToolResult.resultClaim.claimId': input.deliveryKey, + 'backgroundToolResult.resultClaim.batchId': batch.batchId, + }) + .select('deliveryKey envelope +backgroundToolResult') + .lean(); + const byKey = new Map(rows.map((row) => [row.deliveryKey, projectBackgroundResult(row)])); + const results = batch.members.map((key) => byKey.get(key)); + if (results.some((result) => result == null)) return { status: 'not_ready' }; + if ( + input.deliveryClaimToken != null && (await Delivery().exists({ ...scope, - 'backgroundToolResult.resultClaim.claimId': input.claimId, + deliveryKey: input.deliveryKey, + capabilityClaimToken: input.deliveryClaimToken, + 'backgroundToolResultBatch.batchId': batch.batchId, + 'backgroundToolResultBatch.releasing': { $ne: true }, })) == null + ) { + return { status: 'not_ready' }; + } + return { + status: 'acquired', + batchId: batch.batchId, + results: results.filter((result): result is NonNullable => result != null), + }; + } + + async function getAgentBackgroundToolResultBatch( + input: AgentBackgroundToolResultBatchOwnerInput, + ): Promise { + const row = await Delivery() + .findOne({ ...backgroundResultIdentity(input), deliveryKey: input.deliveryKey }) + .select('+backgroundToolResultBatch') + .lean>(); + return row?.backgroundToolResultBatch ?? null; + } + + async function beginAgentBackgroundToolResultBatchDispatch( + input: AgentBackgroundToolResultBatchOwnerInput & { batchId: string; dispatchId: string }, + ): Promise { + const marked = await Delivery().updateOne( + { + ...backgroundResultIdentity(input), + deliveryKey: input.deliveryKey, + status: { $in: UNDELIVERED_STATUSES }, + capabilityStatus: { $ne: 'dead' }, + 'backgroundToolResultBatch.batchId': input.batchId, + 'backgroundToolResultBatch.members.0': { $exists: true }, + 'backgroundToolResultBatch.releasing': { $ne: true }, + 'backgroundToolResultBatch.appliedAt': { $exists: false }, + ...(input.deliveryClaimToken == null + ? {} + : { capabilityClaimToken: input.deliveryClaimToken }), + }, + { + $inc: { 'backgroundToolResultBatch.dispatchCount': 1 }, + $set: { 'backgroundToolResultBatch.dispatchId': input.dispatchId }, + }, + { timestamps: false }, ); + return marked.matchedCount === 1; + } + + /** Admission proof precedes follower settlement, including lost HTTP replies. */ + async function confirmAgentBackgroundToolResultBatch( + input: AgentBackgroundToolResultBatchOwnerInput & { batchId: string }, + ): Promise { + const scope = backgroundResultIdentity(input); + const root = await Delivery() + .findOneAndUpdate( + { + ...scope, + deliveryKey: input.deliveryKey, + 'backgroundToolResultBatch.members.0': { $exists: true }, + 'backgroundToolResultBatch.batchId': input.batchId, + 'backgroundToolResultBatch.releasing': { $ne: true }, + }, + { $min: { 'backgroundToolResultBatch.appliedAt': new Date() } }, + { new: true, timestamps: false }, + ) + .select('status +backgroundToolResultBatch') + .lean>(); + if (root?.backgroundToolResultBatch?.appliedAt == null) return false; + await Delivery().updateMany( + { + ...scope, + deliveryKey: { $in: root.backgroundToolResultBatch.members }, + 'backgroundToolResult.resultClaim.claimId': input.deliveryKey, + 'backgroundToolResult.resultClaim.batchId': input.batchId, + }, + { + $set: { + 'backgroundToolResult.resultClaim.appliedAt': root.backgroundToolResultBatch.appliedAt, + }, + }, + { timestamps: false }, + ); + if ( + (await Delivery().exists({ + ...scope, + 'backgroundToolResult.resultClaim.claimId': input.deliveryKey, + 'backgroundToolResult.resultClaim.batchId': input.batchId, + 'backgroundToolResult.resultClaim.appliedAt': { $exists: false }, + })) != null + ) + return false; + const finish = (retired: boolean) => + Delivery().updateOne( + { + ...scope, + deliveryKey: input.deliveryKey, + 'backgroundToolResultBatch.batchId': input.batchId, + 'backgroundToolResultBatch.appliedAt': { $exists: true }, + status: retired ? 'succeeded' : { $ne: 'succeeded' }, + }, + { + $min: { 'backgroundToolResultBatch.proofCopiedAt': new Date() }, + ...(retired && { $max: { expiresAt: new Date(Date.now() + SUCCESS_RETENTION_MS) } }), + }, + { timestamps: false }, + ); + const finished = await finish(root.status === 'succeeded'); + if (finished.matchedCount === 1) return true; + return root.status !== 'succeeded' && (await finish(true)).matchedCount === 1; } async function settleAgentTriggerHandlingOutcome( @@ -3715,8 +4361,7 @@ export function createAgentTriggerDeliveryMethods( const error = normalizeFailure(input.error); const receiptFence = recoverReceipt ? { - requiredWorkerCapability: - AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES }, backgroundToolResult: { $exists: true }, } : {}; @@ -4273,7 +4918,11 @@ export function createAgentTriggerDeliveryMethods( }, { $set: { backgroundToolResultErasedAt: erasedAt }, - $unset: { backgroundToolResult: 1, backgroundToolResultDeletionPendingAt: 1 }, + $unset: { + backgroundToolResult: 1, + backgroundToolResultBatch: 1, + backgroundToolResultDeletionPendingAt: 1, + }, }, { timestamps: false }, ); @@ -4287,7 +4936,7 @@ export function createAgentTriggerDeliveryMethods( await Delivery().updateMany( { user, - requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES }, 'envelope.target.conversationId': { $in: conversationIds }, backgroundToolResultErasedAt: { $exists: false }, }, @@ -4316,6 +4965,10 @@ export function createAgentTriggerDeliveryMethods( getAgentBackgroundToolResult, getAgentBackgroundToolResultClaim, claimAgentBackgroundToolResults, + claimAgentBackgroundToolResultBatch, + confirmAgentBackgroundToolResultBatch, + beginAgentBackgroundToolResultBatchDispatch, + getAgentBackgroundToolResultBatch, releaseAgentBackgroundToolResultClaims, settleAgentTriggerHandlingOutcome, admitAgentEventActorAction, diff --git a/packages/data-schemas/src/schema/triggerDelivery.ts b/packages/data-schemas/src/schema/triggerDelivery.ts index 927e421b3d..4db14e4d96 100644 --- a/packages/data-schemas/src/schema/triggerDelivery.ts +++ b/packages/data-schemas/src/schema/triggerDelivery.ts @@ -168,6 +168,8 @@ const triggerDeliverySchema: Schema = new Schema( kind: { type: String, enum: ['wakeup'], required: true }, claimId: { type: String, required: true, maxlength: 128 }, claimedAt: { type: Date, required: true }, + appliedAt: { type: Date }, + batchId: { type: String, maxlength: 128 }, }, { _id: false }, ), @@ -178,6 +180,23 @@ const triggerDeliverySchema: Schema = new Schema( ), select: false, }, + backgroundToolResultBatch: { + type: new Schema( + { + batchId: { type: String, required: true, maxlength: 128 }, + dispatchCount: { type: Number, required: true, min: 0 }, + dispatchId: { type: String, maxlength: 128 }, + releaseId: { type: String, maxlength: 128 }, + proofCopiedAt: { type: Date }, + candidates: { type: [String], required: true }, + releasing: { type: Boolean }, + members: { type: [String], default: undefined }, + appliedAt: { type: Date }, + }, + { _id: false }, + ), + select: false, + }, backgroundToolResultErasedAt: { type: Date, select: false }, backgroundToolResultDeletionPendingAt: { type: Date, select: false }, attempts: { type: Number, required: true, default: 0, min: 0 }, diff --git a/packages/data-schemas/src/types/triggerDelivery.ts b/packages/data-schemas/src/types/triggerDelivery.ts index 73ede79450..694dfc4991 100644 --- a/packages/data-schemas/src/types/triggerDelivery.ts +++ b/packages/data-schemas/src/types/triggerDelivery.ts @@ -17,6 +17,8 @@ export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 = 'background_tool_completion_v1'; export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 = 'background_tool_completion_receipt_v2'; +export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3 = + 'background_tool_completion_batch_v3'; export const AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1 = 'agent_queued_turn_v1'; export const AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2 = 'agent_queued_turn_v2'; export const AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS: number = 64 * 1024; @@ -92,6 +94,8 @@ export interface AgentBackgroundToolResultReceipt { kind: 'wakeup'; claimId: string; claimedAt: Date; + appliedAt?: Date; + batchId?: string; }; } @@ -128,6 +132,18 @@ export interface IAgentTriggerDelivery { producerLeaseUntil?: Date; /** Durable source of truth for a background completion. */ backgroundToolResult?: AgentBackgroundToolResultReceipt; + /** Receipt batching is separate from event-envelope coalescing. */ + backgroundToolResultBatch?: { + batchId: string; + dispatchCount: number; + dispatchId?: string; + releaseId?: string; + proofCopiedAt?: Date; + candidates: string[]; + members?: string[]; + appliedAt?: Date; + releasing?: true; + }; /** Conversation-deletion fence preventing a late producer from restoring private output. */ backgroundToolResultErasedAt?: Date; backgroundToolResultDeletionPendingAt?: Date;