diff --git a/api/server/controllers/agents/__tests__/openai.spec.js b/api/server/controllers/agents/__tests__/openai.spec.js index 6fac81d976..b02664babc 100644 --- a/api/server/controllers/agents/__tests__/openai.spec.js +++ b/api/server/controllers/agents/__tests__/openai.spec.js @@ -1355,7 +1355,10 @@ describe('OpenAIChatCompletionController', () => { const { loadAgentTools, loadToolsForExecution } = require('~/server/services/ToolService'); const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions'); - req.config.endpoints.agents.backgroundTasks = { ordinaryToolCancellation: true }; + req.config.endpoints.agents.backgroundTasks = { + ordinaryToolCancellation: true, + completionResultMaxChars: 4096, + }; await OpenAIChatCompletionController(req, res); const [initializeParams, dbMethods] = initializeAgent.mock.calls.at(-1); @@ -1384,6 +1387,7 @@ describe('OpenAIChatCompletionController', () => { const toolExecuteOptions = createToolExecuteHandler.mock.calls.at(-1)[0]; expect(toolExecuteOptions.ordinaryToolCancellation).toBe(true); + expect(toolExecuteOptions.backgroundCompletionResultMaxChars).toBe(4096); expect(toolExecuteOptions.runSignal).toBe(mockExecution.signal); expect(toolExecuteOptions.foregroundRunId).toBe(initializeParams.requestBody.messageId); const effectiveSignal = new AbortController().signal; diff --git a/api/server/controllers/agents/__tests__/responses.unit.spec.js b/api/server/controllers/agents/__tests__/responses.unit.spec.js index 45d00b5684..70f0e93754 100644 --- a/api/server/controllers/agents/__tests__/responses.unit.spec.js +++ b/api/server/controllers/agents/__tests__/responses.unit.spec.js @@ -2065,7 +2065,10 @@ describe('createResponse controller', () => { const { loadAgentTools, loadToolsForExecution } = require('~/server/services/ToolService'); const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions'); - req.config.endpoints.agents.backgroundTasks = { ordinaryToolCancellation: true }; + req.config.endpoints.agents.backgroundTasks = { + ordinaryToolCancellation: true, + completionResultMaxChars: 4096, + }; req.body.stream = stream; await createResponse(req, res); @@ -2095,6 +2098,7 @@ describe('createResponse controller', () => { const toolExecuteOptions = createToolExecuteHandler.mock.calls.at(-1)[0]; expect(toolExecuteOptions.ordinaryToolCancellation).toBe(true); + expect(toolExecuteOptions.backgroundCompletionResultMaxChars).toBe(4096); expect(toolExecuteOptions.runSignal).toBe(mockExecution.signal); expect(toolExecuteOptions.foregroundRunId).toBe(initializeParams.requestBody.messageId); const effectiveSignal = new AbortController().signal; diff --git a/api/server/controllers/agents/openai.js b/api/server/controllers/agents/openai.js index 7bcbe4b9fa..65e2230594 100644 --- a/api/server/controllers/agents/openai.js +++ b/api/server/controllers/agents/openai.js @@ -490,6 +490,8 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { const allowedProviders = new Set(agentsEConfig?.allowedProviders); const ordinaryToolCancellationEnabled = agentsEConfig?.backgroundTasks?.ordinaryToolCancellation === true; + const backgroundCompletionResultMaxChars = + agentsEConfig?.backgroundTasks?.completionResultMaxChars; // Create tool loader const loadTools = createToolLoader({ req, res, signal: execution.signal }); @@ -840,6 +842,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { runSignal: execution.signal, foregroundRunId: responseId, ordinaryToolCancellation: ordinaryToolCancellationEnabled, + backgroundCompletionResultMaxChars, provisionFiles: createProvisionFilesCallback({ req, agentToolContexts, diff --git a/api/server/controllers/agents/responses.js b/api/server/controllers/agents/responses.js index 24e668576b..1b4772ffd0 100644 --- a/api/server/controllers/agents/responses.js +++ b/api/server/controllers/agents/responses.js @@ -732,6 +732,8 @@ const executeResponse = async (envelope, { req, res }) => { const agentsEConfig = appConfig?.endpoints?.[EModelEndpoint.agents]; const ordinaryToolCancellationEnabled = agentsEConfig?.backgroundTasks?.ordinaryToolCancellation === true; + const backgroundCompletionResultMaxChars = + agentsEConfig?.backgroundTasks?.completionResultMaxChars; const previousMessages = request.previous_response_id ? await loadPreviousMessages(request.previous_response_id, principal.userId) : []; @@ -1194,6 +1196,7 @@ const executeResponse = async (envelope, { req, res }) => { runSignal: execution.signal, foregroundRunId: responseId, ordinaryToolCancellation: ordinaryToolCancellationEnabled, + backgroundCompletionResultMaxChars, provisionFiles: createProvisionFilesCallback({ req, agentToolContexts, @@ -1424,6 +1427,7 @@ const executeResponse = async (envelope, { req, res }) => { runSignal: execution.signal, foregroundRunId: responseId, ordinaryToolCancellation: ordinaryToolCancellationEnabled, + backgroundCompletionResultMaxChars, provisionFiles: createProvisionFilesCallback({ req, agentToolContexts, diff --git a/api/server/services/Agents/triggers.js b/api/server/services/Agents/triggers.js index 66926c2991..1854c70e84 100644 --- a/api/server/services/Agents/triggers.js +++ b/api/server/services/Agents/triggers.js @@ -86,6 +86,9 @@ module.exports = { requeueAgentTrigger: service.requeue, retireAgentTrigger: service.retire, renewAgentTriggerProducerLease: service.renewProducerLease, + persistAgentBackgroundToolResult: service.persistBackgroundToolResult, + getAgentBackgroundToolResultClaim: service.getBackgroundToolResultClaim, + releaseAgentBackgroundToolResultClaims: service.releaseBackgroundToolResultClaims, drainAgentTriggerDeliveriesForUser: service.drainUser, prepareAgentTriggerUserPurge: service.prepareUserPurge, cancelAgentTriggerUserPurge: service.cancelUserPurge, diff --git a/api/server/services/Endpoints/agents/backgroundCompletion.js b/api/server/services/Endpoints/agents/backgroundCompletion.js index 6fc0996e62..8ca1cae6ef 100644 --- a/api/server/services/Endpoints/agents/backgroundCompletion.js +++ b/api/server/services/Endpoints/agents/backgroundCompletion.js @@ -2,9 +2,13 @@ const { createBackgroundToolCompletionWakeupHandler, createBackgroundToolDeadClaimRecovery, createBackgroundToolResultHandler, + claimBackgroundToolResult: claimResult, } = require('@librechat/api'); const { enqueueAgentTrigger, + persistAgentBackgroundToolResult, + getAgentBackgroundToolResultClaim, + releaseAgentBackgroundToolResultClaims, renewAgentTriggerProducerLease, retireAgentTrigger, } = require('../../Agents/triggers'); @@ -13,12 +17,17 @@ const preregisterBackgroundToolCompletion = createBackgroundToolCompletionWakeup enqueueAgentTrigger, retireAgentTrigger, renewAgentTriggerProducerLease, + (deliveryKey, sourceId, result) => + persistAgentBackgroundToolResult({ deliveryKey, sourceId, result }), ); function createBackgroundToolResultPersistence({ req, updateToolCallResult }) { return createBackgroundToolResultHandler({ req, updateToolCallResult }); } +const claimBackgroundToolResult = (db, input) => + claimResult(db, getAgentBackgroundToolResultClaim, input); + function createDeadBackgroundToolClaimRecovery( releaseBackgroundToolResultClaims, getGenerationJob, @@ -29,11 +38,13 @@ function createDeadBackgroundToolClaimRecovery( releaseBackgroundToolResultClaims, getGenerationJob, fenceGenerationClaim, + releaseAgentBackgroundToolResultClaims, ); } module.exports = { preregisterBackgroundToolCompletion, createBackgroundToolResultPersistence, + claimBackgroundToolResult, createDeadBackgroundToolClaimRecovery, }; diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js index ee8ebb207d..875d085951 100644 --- a/api/server/services/Endpoints/agents/initialize.js +++ b/api/server/services/Endpoints/agents/initialize.js @@ -95,6 +95,7 @@ const subagentThreadTaskStore = require('./subagentThreadStore'); const { preregisterBackgroundToolCompletion, createBackgroundToolResultPersistence, + claimBackgroundToolResult, createDeadBackgroundToolClaimRecovery, } = require('./backgroundCompletion'); const { logViolation } = require('~/cache'); @@ -215,6 +216,8 @@ const initializeClientWithProvider = async ({ const ordinaryToolCancellationEnabled = appConfig?.endpoints?.[EModelEndpoint.agents]?.backgroundTasks?.ordinaryToolCancellation === true; + const backgroundCompletionResultMaxChars = + appConfig?.endpoints?.[EModelEndpoint.agents]?.backgroundTasks?.completionResultMaxChars; /** The normal controller resolves this once for timestamp anchoring. Reuse * that trusted document for child-thread execution policy; resume and direct * callers fall back to the same owner-scoped lookup. */ @@ -440,6 +443,7 @@ const initializeClientWithProvider = async ({ runSignal: signal, foregroundRunId, ordinaryToolCancellation: ordinaryToolCancellationEnabled, + backgroundCompletionResultMaxChars, loadTools: async ( toolNames, agentId, @@ -515,7 +519,7 @@ const initializeClientWithProvider = async ({ req, updateToolCallResult: db.updateToolCallResult, }), - claim: db.claimBackgroundToolResults, + claim: (input) => claimBackgroundToolResult(db, input), recoverDeadClaim: createDeadBackgroundToolClaimRecovery( db.releaseBackgroundToolResultClaims, (conversationId) => GenerationJobManager.getJob(conversationId), diff --git a/api/server/services/Endpoints/agents/initialize.spec.js b/api/server/services/Endpoints/agents/initialize.spec.js index b01af98760..fe65c2493a 100644 --- a/api/server/services/Endpoints/agents/initialize.spec.js +++ b/api/server/services/Endpoints/agents/initialize.spec.js @@ -408,10 +408,11 @@ describe('initializeClient — processAgent ACL gate', () => { endpointOption: makeEndpointOption(), }); expect(capturedToolExecuteOptions.ordinaryToolCancellation).toBe(false); + expect(capturedToolExecuteOptions.backgroundCompletionResultMaxChars).toBeUndefined(); const enabledReq = makeReq(); enabledReq.config.endpoints.agents = { - backgroundTasks: { ordinaryToolCancellation: true }, + backgroundTasks: { ordinaryToolCancellation: true, completionResultMaxChars: 4096 }, }; await initializeClient({ req: enabledReq, @@ -420,6 +421,7 @@ describe('initializeClient — processAgent ACL gate', () => { endpointOption: makeEndpointOption(), }); expect(capturedToolExecuteOptions.ordinaryToolCancellation).toBe(true); + expect(capturedToolExecuteOptions.backgroundCompletionResultMaxChars).toBe(4096); }); it('propagates an expected-MCP-tools failure from the runtime tool loader', async () => { diff --git a/packages/api/src/agents/backgroundClaims.spec.ts b/packages/api/src/agents/backgroundClaims.spec.ts new file mode 100644 index 0000000000..f0bf556e90 --- /dev/null +++ b/packages/api/src/agents/backgroundClaims.spec.ts @@ -0,0 +1,82 @@ +import type { AgentTriggerDeliveryMethods, MessageMethods } from '@librechat/data-schemas'; +import { claimBackgroundToolResult } from './backgroundClaims'; + +describe('manual background receipt arbitration', () => { + const input = { + userId: 'user', + conversationId: 'conversation', + taskId: 'task', + kind: 'manual' as const, + claimId: 'poll', + }; + const receipt = { kind: 'wakeup' as const, claimId: 'delivery', claimedAt: new Date() }; + function fixture() { + const methods = { + claimBackgroundToolResults: jest + .fn< + ReturnType, + Parameters + >() + .mockResolvedValue({ + status: 'acquired', + messageId: 'recovered-parent', + results: [ + { + taskId: 'task', + toolCallId: 'call', + toolName: 'tool', + status: 'completed', + output: 'done', + }, + ], + }), + releaseBackgroundToolResultClaims: jest.fn(async () => true), + }; + const lookup = jest + .fn< + ReturnType, + Parameters + >() + .mockResolvedValue(null); + return { methods, lookup }; + } + it('uses the recovered parent identity when the process-local registry is gone', async () => { + const { methods, lookup } = fixture(); + lookup.mockResolvedValue(receipt); + await expect(claimBackgroundToolResult(methods, lookup, input)).resolves.toEqual({ + status: 'claimed', + claim: receipt, + messageId: 'recovered-parent', + }); + expect(lookup).toHaveBeenCalledWith( + expect.objectContaining({ parentMessageId: 'recovered-parent' }), + ); + expect(methods.releaseBackgroundToolResultClaims).toHaveBeenCalledWith( + expect.objectContaining({ messageId: 'recovered-parent', taskIds: ['task'] }), + ); + }); + it('releases a speculative message claim if receipt lookup fails', async () => { + const { methods, lookup } = fixture(); + lookup.mockRejectedValue(new Error('database unavailable')); + await expect(claimBackgroundToolResult(methods, lookup, input)).rejects.toThrow( + 'database unavailable', + ); + expect(methods.releaseBackgroundToolResultClaims).toHaveBeenCalledTimes(1); + }); + it('does not report arbitration as complete if rollback fails', async () => { + const { methods, lookup } = fixture(); + lookup.mockResolvedValue(receipt); + methods.releaseBackgroundToolResultClaims.mockResolvedValue(false); + await expect(claimBackgroundToolResult(methods, lookup, input)).rejects.toThrow( + 'could not be released', + ); + }); + it('does not claim the projection when its known receipt already has an owner', async () => { + const { methods, lookup } = fixture(); + lookup.mockResolvedValue(receipt); + await expect( + claimBackgroundToolResult(methods, lookup, { ...input, messageId: 'parent' }), + ).resolves.toMatchObject({ status: 'claimed', messageId: 'parent' }); + expect(methods.claimBackgroundToolResults).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/api/src/agents/backgroundClaims.ts b/packages/api/src/agents/backgroundClaims.ts new file mode 100644 index 0000000000..6f92bca763 --- /dev/null +++ b/packages/api/src/agents/backgroundClaims.ts @@ -0,0 +1,58 @@ +import type { AgentTriggerDeliveryMethods, MessageMethods } from '@librechat/data-schemas'; +import { BACKGROUND_TOOL_COMPLETION_SOURCE } from './backgroundCompletionWakeup'; + +/** Reconciles manual polling with the independent automatic-delivery receipt, + * including polls reconstructed after the process-local registry was lost. */ +export async function claimBackgroundToolResult( + methods: Pick, + getReceiptClaim: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'], + input: Parameters[0], +): ReturnType { + const lookup = (messageId: string) => + getReceiptClaim({ + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + userId: input.userId, + conversationId: input.conversationId, + parentMessageId: messageId, + taskId: input.taskId, + }); + if (input.messageId != null) { + const receiptClaim = await lookup(input.messageId); + if (receiptClaim != null && receiptClaim.claimId !== input.claimId) { + return { status: 'claimed', claim: receiptClaim, messageId: input.messageId }; + } + } + const messageClaim = await methods.claimBackgroundToolResults(input); + if (messageClaim.status !== 'acquired') { + return messageClaim; + } + const messageId = input.messageId ?? messageClaim.messageId; + if (messageId == null) { + throw new Error('The background result claim has no parent message identity'); + } + const release = async () => { + const released = await methods.releaseBackgroundToolResultClaims({ + userId: input.userId, + conversationId: input.conversationId, + messageId, + taskIds: messageClaim.results.map((result) => result.taskId), + kind: input.kind, + claimId: input.claimId, + }); + if (!released) { + throw new Error('The background result claim could not be released'); + } + }; + let receiptClaim: Awaited>; + try { + receiptClaim = await lookup(messageId); + } catch (error) { + await release(); + throw error; + } + if (receiptClaim == null || receiptClaim.claimId === input.claimId) { + return messageClaim; + } + await release(); + return { status: 'claimed', claim: receiptClaim, messageId }; +} diff --git a/packages/api/src/agents/backgroundCompletion.ts b/packages/api/src/agents/backgroundCompletion.ts index 08b84b0b80..8ef1d74cca 100644 --- a/packages/api/src/agents/backgroundCompletion.ts +++ b/packages/api/src/agents/backgroundCompletion.ts @@ -31,6 +31,13 @@ export interface BackgroundToolWakeupRetireOptions { export interface BackgroundToolWakeupAdmission { /** Renews durable proof that the process-local executor still owns work. */ renew: () => Promise; + /** Persists terminal output on the pre-admitted delivery before the parent + * message projection exists. */ + persistResult?: (result: { + status: 'completed' | 'error' | 'cancelled'; + output: string; + settledAt: Date; + }) => Promise; /** Retires a delivery whose terminal result can no longer be made durable. * Manual polling requires an atomic unclaimed-only transition: once a * resolver owns a lease, its prepared continuation cannot be cancelled. */ diff --git a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts index 19a4a9560d..90f226f634 100644 --- a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts +++ b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts @@ -1,4 +1,4 @@ -import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 } from '@librechat/data-schemas'; +import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 } from '@librechat/data-schemas'; import type { AgentTriggerProducerLeaseStatus } from '@librechat/data-schemas'; import type { EnqueueBackgroundToolCompletion } from './backgroundCompletionWakeup'; import { @@ -91,6 +91,10 @@ function resolverMethods() { leaseUntil: new Date(NOW + 30_000), }), ), + getAgentBackgroundToolResult: jest.fn(async () => null), + getAgentBackgroundToolResultClaim: jest.fn(async () => null), + claimAgentBackgroundToolResults: jest.fn(async () => ({ status: 'not_ready' as const })), + releaseAgentBackgroundToolResultClaims: jest.fn(async () => true), }, }; } @@ -111,7 +115,13 @@ describe('background tool completion wakeups', () => { >(async () => ({ deliveryKey: 'delivery-key-1' })); const retire = jest.fn(async () => true); const renew = jest.fn(async () => true); - const notify = createBackgroundToolCompletionWakeupHandler(enqueue, retire, renew); + const persistResult = jest.fn(async () => true); + const notify = createBackgroundToolCompletionWakeupHandler( + enqueue, + retire, + renew, + persistResult, + ); const admission = await notify(registration()); expect(admission).not.toBe(false); @@ -134,7 +144,7 @@ describe('background tool completion wakeups', () => { expect(options).toEqual({ orderingKey: 'background-tool-completion:conversation-1:task-1', availableAt: new Date(NOW + 250), - requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, producerLeaseUntil: new Date(NOW + 30_000), }); if (admission !== false) { @@ -145,6 +155,20 @@ describe('background tool completion wakeups', () => { 'background-tool-completion', new Date(NOW + 30_000), ); + if (admission !== false) { + await expect( + admission.persistResult?.({ + status: 'completed', + output: 'done', + settledAt: new Date(NOW), + }), + ).resolves.toBe(true); + } + expect(persistResult).toHaveBeenCalledWith('delivery-key-1', 'background-tool-completion', { + status: 'completed', + output: 'done', + settledAt: new Date(NOW), + }); if (admission !== false) { await expect(admission.retire('result unavailable')).resolves.toBe(true); } @@ -180,6 +204,7 @@ describe('background tool completion wakeups', () => { it('retires the batch-root delivery before releasing all of its sibling claims', async () => { const retire = jest.fn(async () => true); const release = jest.fn(async () => true); + const releaseReceipts = jest.fn(async () => true); const getGenerationJob = jest.fn(async () => ({ status: 'complete' })); const fenceGenerationClaim = jest.fn(async () => 'fenced' as const); const recover = createBackgroundToolDeadClaimRecovery( @@ -187,6 +212,7 @@ describe('background tool completion wakeups', () => { release, getGenerationJob, fenceGenerationClaim, + releaseReceipts, ); await expect( @@ -218,6 +244,13 @@ describe('background tool completion wakeups', () => { kind: 'wakeup', claimId: 'batch-root-delivery', }); + expect(releaseReceipts).toHaveBeenCalledWith({ + sourceId: 'background-tool-completion', + userId: 'user-1', + conversationId: 'conversation-1', + parentMessageId: 'response-1', + claimId: 'batch-root-delivery', + }); }); it('keeps a manual result claim while its owning generation is active', async () => { @@ -428,6 +461,120 @@ describe('background tool completion wakeups', () => { ); }); + it('continues from the independent delivery receipt before the parent projection lands', async () => { + const { methods } = resolverMethods(); + methods.claimBackgroundToolResults.mockResolvedValueOnce({ status: 'not_ready', results: [] }); + methods.claimAgentBackgroundToolResults.mockResolvedValueOnce({ + status: 'acquired', + results: [ + { + taskId: 'task-1', + toolCallId: 'call-1', + toolName: 'slow_tool', + status: 'completed', + output: 'independently durable', + }, + ], + } as never); + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods: methods as never, + getGenerationJob: async () => null, + }); + + const prepared = await resolve(await envelope(), { idempotencyKey: 'delivery-1' }); + + expect(prepared).toMatchObject({ status: 'ready', parentMessageId: 'response-2' }); + expect(prepared?.status === 'ready' && prepared.input).toContain('independently durable'); + expect(methods.claimAgentBackgroundToolResults).toHaveBeenCalledWith({ + deliveryKey: 'delivery-1', + sourceId: 'background-tool-completion', + userId: 'user-1', + conversationId: 'conversation-1', + parentMessageId: 'response-1', + agentId: 'agent_parent_1', + claimId: 'delivery-1', + limit: 1, + }); + expect(methods.claimBackgroundToolResults).toHaveBeenCalledWith( + expect.objectContaining({ taskId: 'task-1', kind: 'wakeup', claimId: 'delivery-1' }), + ); + }); + + it('honors a manual message claim before an independent receipt', async () => { + const { methods } = resolverMethods(); + methods.claimBackgroundToolResults.mockResolvedValueOnce({ status: 'claimed', results: [] }); + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods: methods as never, + getGenerationJob: async () => null, + }); + + await expect(resolve(await envelope(), { idempotencyKey: 'delivery-1' })).resolves.toEqual({ + status: 'settled', + }); + expect(methods.claimAgentBackgroundToolResults).not.toHaveBeenCalled(); + expect(methods.getAgentBackgroundToolResult).not.toHaveBeenCalled(); + }); + + it('does not settle a resumed receipt owner while a speculative manual message claim yields', async () => { + const { methods } = resolverMethods(); + methods.claimBackgroundToolResults.mockResolvedValueOnce({ status: 'claimed', results: [] }); + methods.getAgentBackgroundToolResultClaim.mockResolvedValueOnce({ + kind: 'wakeup', + claimId: 'delivery-1', + claimedAt: new Date(NOW), + } as never); + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods: methods as never, + getGenerationJob: async () => null, + }); + await expect(resolve(await envelope(), { idempotencyKey: 'delivery-1' })).rejects.toMatchObject( + { + code: 'BACKGROUND_TOOL_CLAIM_RECONCILING', + deferWithoutAttempt: true, + }, + ); + expect(methods.releaseAgentBackgroundToolResultClaims).toHaveBeenCalledWith( + expect.objectContaining({ claimId: 'delivery-1' }), + ); + expect(methods.releaseBackgroundToolResultClaims).not.toHaveBeenCalled(); + }); + + it('retries after yielding to a racing manual claim so mutual yielding cannot lose output', async () => { + const { methods } = resolverMethods(); + methods.claimBackgroundToolResults + .mockResolvedValueOnce({ status: 'not_ready' } as never) + .mockResolvedValueOnce({ + status: 'claimed', + claim: { kind: 'manual', claimId: 'poll-1' }, + } as never); + methods.claimAgentBackgroundToolResults.mockResolvedValueOnce({ + status: 'acquired', + results: [ + { + taskId: 'task-1', + toolCallId: 'call-1', + toolName: 'slow_tool', + status: 'completed', + output: 'durable', + }, + ], + } as never); + const resolve = createBackgroundToolCompletionWakeupResolver({ + methods: methods as never, + getGenerationJob: async () => null, + }); + + await expect(resolve(await envelope(), { idempotencyKey: 'delivery-1' })).rejects.toMatchObject( + { + code: 'BACKGROUND_TOOL_CLAIM_RECONCILING', + deferWithoutAttempt: true, + }, + ); + expect(methods.releaseAgentBackgroundToolResultClaims).toHaveBeenCalledWith( + expect.objectContaining({ claimId: 'delivery-1', parentMessageId: 'response-1' }), + ); + }); + it('shares one bounded input budget across a full sibling batch', async () => { const { methods } = resolverMethods(); const results = Array.from({ length: 8 }, (_, index) => ({ @@ -468,7 +615,10 @@ describe('background tool completion wakeups', () => { await prepared.releaseOnDefiniteFailure?.(); } expect(releaseBackgroundToolResultClaims).toHaveBeenCalledWith( - expect.objectContaining({ taskIds: ['task-1', 'task-2'], claimId: 'delivery-1' }), + expect.objectContaining({ claimId: 'delivery-1', kind: 'wakeup' }), + ); + expect(releaseBackgroundToolResultClaims).toHaveBeenCalledWith( + expect.not.objectContaining({ taskIds: expect.anything() }), ); }); diff --git a/packages/api/src/agents/backgroundCompletionWakeup.ts b/packages/api/src/agents/backgroundCompletionWakeup.ts index bdb2686940..ee4c8e992d 100644 --- a/packages/api/src/agents/backgroundCompletionWakeup.ts +++ b/packages/api/src/agents/backgroundCompletionWakeup.ts @@ -1,8 +1,9 @@ import { randomUUID } from 'node:crypto'; import { isEphemeralAgentId } from 'librechat-data-provider'; -import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 } from '@librechat/data-schemas'; +import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 } from '@librechat/data-schemas'; import type { AgentTriggerProducerLeaseStatus, + AgentTriggerDeliveryMethods, BackgroundToolResultClaim, ConversationMethods, IMessage, @@ -51,6 +52,16 @@ export type RenewBackgroundToolCompletionProducerLease = ( leaseUntil: Date, ) => Promise; +export type PersistBackgroundToolCompletionResult = ( + deliveryKey: string, + sourceId: string, + result: { + status: 'completed' | 'error' | 'cancelled'; + output: string; + settledAt: Date; + }, +) => Promise; + type WakeupMethods = Pick & Pick & { claimBackgroundToolResults(params: { @@ -76,6 +87,14 @@ type WakeupMethods = Pick & sourceId: string; now: Date; }): Promise; + getAgentBackgroundToolResult?(params: { deliveryKey: string; sourceId: string }): Promise<{ + status: 'completed' | 'error' | 'cancelled'; + output: string; + settledAt: Date; + } | null>; + claimAgentBackgroundToolResults?: AgentTriggerDeliveryMethods['claimAgentBackgroundToolResults']; + getAgentBackgroundToolResultClaim?: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']; + releaseAgentBackgroundToolResultClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims']; }; interface GenerationState { @@ -326,8 +345,33 @@ export function createBackgroundToolCompletionWakeupResolver({ agentId: envelope.target.agentId, kind: 'wakeup', claimId: context.idempotencyKey, + limit: 1, }); if (claim.status === 'claimed') { + const receiptOwner = await methods.getAgentBackgroundToolResultClaim?.({ + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + userId, + conversationId: envelope.target.conversationId, + parentMessageId: envelope.target.parentMessageId, + taskId: registration.taskId, + }); + if (receiptOwner?.claimId === context.idempotencyKey) { + const released = await methods.releaseAgentBackgroundToolResultClaims?.({ + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + userId, + conversationId: envelope.target.conversationId, + parentMessageId: envelope.target.parentMessageId, + claimId: context.idempotencyKey, + }); + if (released === false) + throw new Error('Background receipt claim release was not confirmed'); + throw executionError('Background result ownership is being reconciled.', { + code: 'BACKGROUND_TOOL_CLAIM_RECONCILING', + retryable: true, + deferWithoutAttempt: true, + retryAfter: '1', + }); + } return { status: 'settled' }; } if (claim.status === 'outcome_unknown') { @@ -336,47 +380,17 @@ export function createBackgroundToolCompletionWakeupResolver({ retryable: false, }); } - if (claim.status !== 'acquired') { - let producerLease: AgentTriggerProducerLeaseStatus; - try { - producerLease = await methods.getAgentTriggerDeliveryProducerLease({ - deliveryKey: context.idempotencyKey, - sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, - now: new Date(), - }); - } catch (error) { - throw executionError( - `Background tool producer liveness is temporarily unavailable: ${ - error instanceof Error ? error.message : String(error) - }`, - { code: 'BACKGROUND_TOOL_PRODUCER_STATE_UNAVAILABLE', retryable: true }, - ); - } - if (producerLease.status === 'expired') { - throw executionError('The process-local background tool executor was lost.', { - code: 'BACKGROUND_TOOL_PRODUCER_LOST', - retryable: false, - }); - } - /** A live lease proves the invocation or its durable persistence retry - * still has an owner. Missing remains defer-only for compatibility with - * completion rows admitted before producer leases existed. */ - throw executionError('The background tool result is not durable yet.', { - code: 'BACKGROUND_TOOL_RESULT_NOT_READY', - retryable: true, - status: 409, - retryAfter: '1', - deferWithoutAttempt: true, + if (claim.status === 'acquired') { + const taskIds = claim.results.map((result) => result.taskId); + const receiptOwner = await methods.getAgentBackgroundToolResultClaim?.({ + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + userId, + conversationId: envelope.target.conversationId, + parentMessageId: envelope.target.parentMessageId, + taskId: registration.taskId, }); - } - const taskIds = claim.results.map((result) => result.taskId); - const input = buildWakeupInput(claim.results); - return { - status: 'ready', - parentMessageId, - input, - releaseOnDefiniteFailure: async () => { - await methods.releaseBackgroundToolResultClaims({ + if (receiptOwner != null && receiptOwner.claimId !== context.idempotencyKey) { + const released = await methods.releaseBackgroundToolResultClaims({ userId, conversationId: envelope.target.conversationId, messageId: envelope.target.parentMessageId, @@ -384,8 +398,153 @@ export function createBackgroundToolCompletionWakeupResolver({ kind: 'wakeup', claimId: context.idempotencyKey, }); - }, - }; + if (!released) throw new Error('Background projection claim release was not confirmed'); + return { status: 'settled' }; + } + const input = buildWakeupInput(claim.results); + return { + status: 'ready', + parentMessageId, + input, + releaseOnDefiniteFailure: async () => { + const released = await methods.releaseBackgroundToolResultClaims({ + userId, + conversationId: envelope.target.conversationId, + messageId: envelope.target.parentMessageId, + kind: 'wakeup', + claimId: context.idempotencyKey, + }); + if (!released) throw new Error('Background projection claim release was not confirmed'); + const receiptReleased = await methods.releaseAgentBackgroundToolResultClaims?.({ + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + userId, + conversationId: envelope.target.conversationId, + parentMessageId: envelope.target.parentMessageId, + claimId: context.idempotencyKey, + }); + if (receiptReleased === false) + throw new Error('Background receipt claim release was not confirmed'); + }, + }; + } + const receiptClaim = await methods.claimAgentBackgroundToolResults?.({ + deliveryKey: context.idempotencyKey, + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + userId, + conversationId: envelope.target.conversationId, + parentMessageId: envelope.target.parentMessageId, + agentId: envelope.target.agentId, + claimId: context.idempotencyKey, + limit: 1, + }); + if (receiptClaim?.status === 'claimed') { + return { status: 'settled' }; + } + if (receiptClaim?.status === 'acquired') { + /** Reconcile with a parent projection that may have appeared while the + * receipt CAS was in flight. A manual poll that already owns the message + * wins; otherwise stamp this same automatic owner before dispatch. */ + const projectedClaim = await methods.claimBackgroundToolResults({ + userId, + conversationId: envelope.target.conversationId, + messageId: envelope.target.parentMessageId, + taskId: registration.taskId, + agentId: envelope.target.agentId, + kind: 'wakeup', + claimId: context.idempotencyKey, + limit: 1, + }); + if (projectedClaim.status === 'claimed') { + const released = await methods.releaseAgentBackgroundToolResultClaims?.({ + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + userId, + conversationId: envelope.target.conversationId, + parentMessageId: envelope.target.parentMessageId, + claimId: context.idempotencyKey, + }); + if (released === false) + throw new Error('Background receipt claim release was not confirmed'); + throw executionError('Background result ownership is being reconciled.', { + code: 'BACKGROUND_TOOL_CLAIM_RECONCILING', + retryable: true, + deferWithoutAttempt: true, + retryAfter: '1', + }); + } + return { + status: 'ready', + parentMessageId, + input: buildWakeupInput(receiptClaim.results), + releaseOnDefiniteFailure: async () => { + const projectionReleased = await methods.releaseBackgroundToolResultClaims({ + userId, + conversationId: envelope.target.conversationId, + messageId: envelope.target.parentMessageId, + kind: 'wakeup', + claimId: context.idempotencyKey, + }); + if (projectedClaim.status === 'acquired' && !projectionReleased) { + throw new Error('Background projection claim release was not confirmed'); + } + const receiptReleased = await methods.releaseAgentBackgroundToolResultClaims?.({ + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + userId, + conversationId: envelope.target.conversationId, + parentMessageId: envelope.target.parentMessageId, + claimId: context.idempotencyKey, + }); + if (receiptReleased === false) + throw new Error('Background receipt claim release was not confirmed'); + }, + }; + } + const receipt = + methods.claimAgentBackgroundToolResults == null + ? await methods.getAgentBackgroundToolResult?.({ + deliveryKey: context.idempotencyKey, + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + }) + : null; + if (receipt != null) { + return { + status: 'ready', + parentMessageId, + input: buildWakeupInput([ + { ...registration, status: receipt.status, output: receipt.output }, + ]), + }; + } + let producerLease: AgentTriggerProducerLeaseStatus; + try { + producerLease = await methods.getAgentTriggerDeliveryProducerLease({ + deliveryKey: context.idempotencyKey, + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + now: new Date(), + }); + } catch (error) { + throw executionError( + `Background tool producer liveness is temporarily unavailable: ${ + error instanceof Error ? error.message : String(error) + }`, + { code: 'BACKGROUND_TOOL_PRODUCER_STATE_UNAVAILABLE', retryable: true }, + ); + } + if (producerLease.status === 'expired') { + throw executionError('The process-local background tool executor was lost.', { + code: 'BACKGROUND_TOOL_PRODUCER_LOST', + retryable: false, + }); + } + /** A live lease proves the invocation or its durable persistence retry + * still has an owner. Missing remains defer-only for compatibility with + * completion rows admitted before producer leases existed. */ + throw executionError('The background tool result is not durable yet.', { + code: 'BACKGROUND_TOOL_RESULT_NOT_READY', + retryable: true, + status: 409, + retryAfter: '1', + deferWithoutAttempt: true, + }); }; } @@ -394,6 +553,7 @@ export function createBackgroundToolCompletionWakeupHandler( enqueue: EnqueueBackgroundToolCompletion, retire: RetireBackgroundToolCompletion, renewProducerLease: RenewBackgroundToolCompletionProducerLease, + persistResult?: PersistBackgroundToolCompletionResult, ): ( registration: BackgroundToolWakeupRegistration, ) => Promise { @@ -437,7 +597,7 @@ export function createBackgroundToolCompletionWakeupHandler( availableAt: new Date( Math.max(Date.now(), registration.createdAt) + WAKEUP_ADMISSION_DELAY_MS, ), - requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, producerLeaseUntil: new Date(Date.now() + BACKGROUND_TOOL_PRODUCER_LEASE_MS), }); return { @@ -447,6 +607,12 @@ export function createBackgroundToolCompletionWakeupHandler( BACKGROUND_TOOL_COMPLETION_SOURCE, new Date(Date.now() + BACKGROUND_TOOL_PRODUCER_LEASE_MS), ), + ...(persistResult == null + ? {} + : { + persistResult: (result) => + persistResult(admitted.deliveryKey, BACKGROUND_TOOL_COMPLETION_SOURCE, result), + }), retire: (reason, options) => options == null ? retire(admitted.deliveryKey, BACKGROUND_TOOL_COMPLETION_SOURCE, reason) @@ -469,6 +635,7 @@ export function createBackgroundToolDeadClaimRecovery( conversationId: string; claimId: string; }) => Promise<'fenced' | 'started' | 'unavailable'>, + releaseReceiptClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'], ): BackgroundToolDeadClaimRecovery { return async ({ userId, conversationId, messageId, claimId, kind, generationId }) => { if (kind === 'manual') { @@ -518,12 +685,24 @@ export function createBackgroundToolDeadClaimRecovery( if (await claimGenerationIsActive()) { return false; } - return releaseClaims({ + const released = await releaseClaims({ userId, conversationId, messageId, kind: 'wakeup', claimId, }); + if (!released) { + return false; + } + return ( + (await releaseReceiptClaims?.({ + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + userId, + conversationId, + parentMessageId: messageId, + claimId, + })) ?? true + ); }; } diff --git a/packages/api/src/agents/handlers.background.spec.ts b/packages/api/src/agents/handlers.background.spec.ts index ef1202bf4a..645a473cf9 100644 --- a/packages/api/src/agents/handlers.background.spec.ts +++ b/packages/api/src/agents/handlers.background.spec.ts @@ -3,6 +3,7 @@ import { logger } from '@librechat/data-schemas'; import { tool as structuredTool } from '@librechat/agents/langchain/tools'; import type { StructuredToolInterface } from '@librechat/agents/langchain/tools'; import type { FiltersConfig } from 'librechat-data-provider'; +import type { BackgroundToolWakeupAdmission } from './backgroundCompletion'; import type { ToolExecuteOptions } from './handlers'; import { BACKGROUND_TASK_ABORT_GRACE_MS, @@ -147,9 +148,13 @@ describe('createToolExecuteHandler — background tool calls', () => { it('pre-registers an ordinary completion before invoke and persists its terminal receipt', async () => { const events: string[] = []; const retire = jest.fn(async () => true); + const persistResult = jest.fn(async () => { + events.push('receipt'); + return true; + }); const preregister = jest.fn(async () => { events.push('preregister'); - return { renew: jest.fn(async () => true), retire }; + return { renew: jest.fn(async () => true), persistResult, retire }; }); const persist = jest.fn(async () => { events.push('persist'); @@ -198,7 +203,7 @@ describe('createToolExecuteHandler — background tool calls', () => { ); expect(JSON.parse(dispatch.content).message).toContain('host will resume you'); await flushMicrotasks(); - expect(events).toEqual(['preregister', 'invoke', 'persist']); + expect(events).toEqual(['preregister', 'invoke', 'receipt', 'persist']); expect(preregister).toHaveBeenCalledWith( expect.objectContaining({ toolCallId: 'call-wakeup', @@ -217,6 +222,9 @@ describe('createToolExecuteHandler — background tool calls', () => { }), }), ); + expect(persistResult).toHaveBeenCalledWith( + expect.objectContaining({ status: 'completed', output: 'durable result' }), + ); expect(retire).not.toHaveBeenCalled(); }); @@ -264,6 +272,55 @@ describe('createToolExecuteHandler — background tool calls', () => { ); }); + it.each(['acknowledged', 'ambiguous'])( + 'keeps delivery live for an %s receipt without a projection', + async (ack) => { + const retire = jest.fn(async () => true); + const persistResult = jest.fn(async () => { + if (ack === 'ambiguous') throw new Error('receipt committed but acknowledgement lost'); + return true; + }); + const tool = { + name: 'search_mcp_docs', + description: 'search docs', + schema: z.object({ q: z.string() }), + invoke: jest.fn(async () => ({ content: 'available before parent row' })), + } as unknown as StructuredToolInterface; + const handler = createToolExecuteHandler({ + loadTools: async () => ({ loadedTools: [tool] }), + backgroundToolCompletion: { + preregister: jest.fn(async () => ({ + renew: jest.fn(async () => true), + persistResult, + retire, + })), + persist: jest.fn(async () => false), + claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })), + }, + }); + + await runBatch(handler, { + toolCalls: [ + { + id: `call-independent-receipt-${ack}`, + name: tool.name, + args: { q: 'receipt', run_in_background: true }, + stepId: 'step-independent-receipt', + }, + ], + agentId: 'agent_parent_1', + configurable: buildConfig([tool.name]), + metadata: { thread_id: 'exec_convo', run_id: `response-independent-receipt-${ack}` }, + }); + await flushMicrotasks(); + + expect(persistResult).toHaveBeenCalledWith( + expect.objectContaining({ status: 'completed', output: 'available before parent row' }), + ); + expect(retire).not.toHaveBeenCalled(); + }, + ); + it('keeps polling guidance when the completion adapter skips registration', async () => { const tool = makeSearchTool({ calls: 0 }); const preregister = jest.fn(async () => false as const); @@ -2570,6 +2627,84 @@ describe('createToolExecuteHandler — backgrounded code execution', () => { expect(poll[0].artifact).toEqual(CODE_ARTIFACT); }); + it('does not publish an output-only receipt when generated files miss their message anchor', async () => { + const state: CodeToolState = { calls: 0 }; + const retire = jest.fn(async () => true); + const persistResult = jest.fn(async () => true); + const handler = createToolExecuteHandler({ + loadTools: async () => ({ loadedTools: [makeCodeTool(state)] }), + persistBackgroundCodeResult: async () => ({ + deliveryReady: false, + attachments: [{ filename: 'plot.png' }], + }), + backgroundToolCompletion: { + preregister: jest.fn(async () => ({ + renew: jest.fn(async () => true), + persistResult, + retire, + })), + persist: jest.fn(async () => true), + claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })), + }, + }); + + await runBatch(handler, { + toolCalls: [codeCall({ id: 'call_code_missing_anchor' })], + agentId: 'a', + configurable: buildConfig(['execute_code']), + metadata: { thread_id: 'exec_convo_code_missing_anchor', run_id: 'msg-missing-anchor' }, + }); + await flushMicrotasks(); + await flushMicrotasks(); + + expect(persistResult).not.toHaveBeenCalled(); + expect(retire).toHaveBeenCalledWith('background code result was not persisted', undefined); + }); + + it('uses the configured durable completion result size', async () => { + const persistResult: jest.MockedFunction< + NonNullable + > = jest.fn( + async (_result: Parameters>[0]) => + true, + ); + const tool = { + name: 'execute_code', + description: 'run code', + schema: z.object({ lang: z.string(), code: z.string() }), + invoke: jest.fn(async () => ({ content: '0123456789'.repeat(10), artifact: CODE_ARTIFACT })), + } as unknown as StructuredToolInterface; + const handler = createToolExecuteHandler({ + loadTools: async () => ({ loadedTools: [tool] }), + backgroundCompletionResultMaxChars: 32, + persistBackgroundCodeResult: async () => ({ deliveryReady: true, attachments: [] }), + backgroundToolCompletion: { + preregister: jest.fn(async () => ({ + renew: jest.fn(async () => true), + persistResult, + retire: jest.fn(async () => true), + })), + persist: jest.fn(async () => true), + claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })), + }, + }); + + await runBatch(handler, { + toolCalls: [codeCall({ id: 'call-sized-receipt' })], + agentId: 'agent_parent_1', + configurable: buildConfig([tool.name]), + metadata: { thread_id: 'exec_convo', run_id: 'response-sized-receipt' }, + }); + await flushMicrotasks(); + await flushMicrotasks(); + await flushMicrotasks(); + + expect(persistResult).toHaveBeenCalledWith( + expect.objectContaining({ status: 'completed', output: expect.any(String) }), + ); + expect(persistResult.mock.calls[0][0].output).toHaveLength(32); + }); + it('makes a completion-time generated-file policy rejection terminal across polls', async () => { const protectedValue = 'PROTECTED-GENERATED-FILE-BYTES'; const blockedArtifact = { @@ -2596,12 +2731,17 @@ describe('createToolExecuteHandler — backgrounded code execution', () => { }); const toolEndCallback = jest.fn(); const retire = jest.fn(async () => true); + const persistResult = jest.fn(async () => true); const handler = createToolExecuteHandler({ loadTools: async () => ({ loadedTools: [codeTool] }), toolEndCallback, persistBackgroundCodeResult, backgroundToolCompletion: { - preregister: jest.fn(async () => ({ renew: jest.fn(async () => true), retire })), + preregister: jest.fn(async () => ({ + renew: jest.fn(async () => true), + retire, + persistResult, + })), persist: jest.fn(async () => true), claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })), }, @@ -2621,7 +2761,10 @@ describe('createToolExecuteHandler — backgrounded code execution', () => { const taskId = JSON.parse(dispatch[0].content).background_task_id; await flushMicrotasks(); await flushMicrotasks(); - expect(retire).toHaveBeenCalledWith('background code result persistence failed', undefined); + expect(persistResult).toHaveBeenCalledWith( + expect.objectContaining({ status: 'error', output: MODEL_BOUND_FILE_CONTENT_BLOCK }), + ); + expect(retire).not.toHaveBeenCalled(); await flushMicrotasks(); for (const pollId of ['call_policy_poll_1', 'call_policy_poll_2']) { diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts index 155f20bd18..44d8f50bc8 100644 --- a/packages/api/src/agents/handlers.ts +++ b/packages/api/src/agents/handlers.ts @@ -3,6 +3,7 @@ import { Types } from 'mongoose'; import { GraphEvents, Constants, ToolEndHandler } from '@librechat/agents'; import { logger, normalizeSkillFrontmatterKeys } from '@librechat/data-schemas'; import { + AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT, hasActivePiiFields, hasActivePiiPatterns, hasToolCallErrorPrefix, @@ -271,6 +272,8 @@ export interface ToolExecuteOptions { runFiles?: Pick; /** Trusted deployment gate for cooperative ordinary-tool cancellation. */ ordinaryToolCancellation?: boolean; + /** Deployment-configured cap for terminal output copied into durable receipts. */ + backgroundCompletionResultMaxChars?: number; /** Callback to process tool artifacts (code output files, file citations, etc.) */ toolEndCallback?: ToolEndCallback; /** Durable internal-completion adapter, present only for an Event Actor invocation. */ @@ -5317,6 +5320,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand subagentTasks, runFiles, ordinaryToolCancellation = false, + backgroundCompletionResultMaxChars = AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT, provisionFiles, } = options; @@ -5774,6 +5778,9 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand ...(params.status === 'cancelled' ? { cancelled: true } : {}), settledAt: new Date(current?.updatedAt ?? Date.now()), ...(completionPreregistered ? { completionWakeup: true } : {}), + ...(completionAdmission?.persistResult != null + ? { completionReceipt: true } + : {}), ...(current?.resultClaim != null ? { resultClaim: { @@ -5794,10 +5801,40 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand task.id, ); const backgroundTask = resolveBackgroundTask(); + let durableReceiptReady = false; + let durableReceiptAmbiguous = false; + const persistDurableReceipt = async (receipt: { + status: 'completed' | 'error' | 'cancelled'; + output?: string; + }): Promise => { + if (completionAdmission?.persistResult == null) { + return false; + } + try { + return await completionAdmission.persistResult({ + status: receipt.status, + output: truncateMiddle( + receipt.output ?? '', + backgroundCompletionResultMaxChars, + ), + settledAt: backgroundTask.settledAt, + }); + } catch (receiptError) { + durableReceiptAmbiguous = true; + logger.warn( + `[background] Failed to persist independent result receipt for task ${task.id}:`, + receiptError, + ); + return false; + } + }; const retireFailedPersistence = async ( reason: string, certainty: 'definite' | 'ambiguous', ): Promise => { + if (durableReceiptAmbiguous) { + return; + } if (completionAdmission == null) { backgroundTaskRegistry.markCompletionPersistenceFailed( backgroundUserId, @@ -5852,6 +5889,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand } }; if (!harvestEnabled || !persistBackgroundCodeResult) { + durableReceiptReady = await persistDurableReceipt({ + status: params.status, + output: params.output ?? localTask?.result, + }); if ( backgroundToolCompletion == null || detachedReservation?.status === 'reserved' @@ -5870,17 +5911,19 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand backgroundTask, resolveBackgroundTask, }); - if (!deliveryReady) { + if (!deliveryReady && !durableReceiptReady) { await retireFailedPersistence( 'background tool result was not persisted', 'definite', ); } } catch (persistError) { - await retireFailedPersistence( - 'background tool result persistence failed', - isContentFilterError(persistError) ? 'definite' : 'ambiguous', - ); + if (!durableReceiptReady) { + await retireFailedPersistence( + 'background tool result persistence failed', + isContentFilterError(persistError) ? 'definite' : 'ambiguous', + ); + } logger.warn( `[background] Failed to persist result for task ${task.id}:`, persistError, @@ -5922,7 +5965,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand task.id, params.artifact, ); - if (completionPreregistered) { + if (completionPreregistered && !durableReceiptReady) { await retireFailedPersistence( 'background code result had no durable message anchor', 'definite', @@ -5930,6 +5973,12 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand } return; } + if (persisted.deliveryReady !== false) { + durableReceiptReady = await persistDurableReceipt({ + status: params.status, + output: params.output ?? localTask?.result, + }); + } if (persisted.deliveryReady === false) { await retireFailedPersistence( 'background code result was not persisted', @@ -5943,26 +5992,38 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand persisted.attachments, ); } catch (persistError) { - if (completionPreregistered) { - await retireFailedPersistence( - 'background code result persistence failed', - isContentFilterError(persistError) ? 'definite' : 'ambiguous', - ); - } if (isContentFilterError(persistError)) { + const blockedMessage = + persistError instanceof ContentFilterError + ? modelBoundContentFilterErrorMessage(persistError.body) + : persistError.body.message; backgroundTaskRegistry.blockArtifact( backgroundUserId, backgroundConversationId, task.id, - persistError instanceof ContentFilterError - ? modelBoundContentFilterErrorMessage(persistError.body) - : persistError.body.message, + blockedMessage, ); + durableReceiptReady = await persistDurableReceipt({ + status: 'error', + output: blockedMessage, + }); + if (completionPreregistered && !durableReceiptReady) { + await retireFailedPersistence( + 'background code result was blocked by content policy', + 'definite', + ); + } logger.warn( `[background] Generated code output for task ${task.id} was blocked by content policy.`, ); return; } + if (completionPreregistered && !durableReceiptReady) { + await retireFailedPersistence( + 'background code result persistence failed', + 'ambiguous', + ); + } logger.warn( `[background] Failed to persist code result for task ${task.id}:`, persistError, diff --git a/packages/api/src/agents/harvest.ts b/packages/api/src/agents/harvest.ts index 55eb828bb2..84738597ad 100644 --- a/packages/api/src/agents/harvest.ts +++ b/packages/api/src/agents/harvest.ts @@ -40,6 +40,8 @@ export interface BackgroundToolResultState { settledAt: Date; /** This exact task owns a pre-registered automatic continuation delivery. */ completionWakeup?: true; + /** Independent receipts retain task-local delivery ownership. */ + completionReceipt?: true; resultClaim?: { kind: 'manual' | 'wakeup'; claimId: string; diff --git a/packages/api/src/agents/index.ts b/packages/api/src/agents/index.ts index 56090134f1..cfca8b0cfc 100644 --- a/packages/api/src/agents/index.ts +++ b/packages/api/src/agents/index.ts @@ -25,6 +25,7 @@ export * from './handlers'; export * from './guard'; export * from './harvest'; export * from './backgroundCompletion'; +export * from './backgroundClaims'; export * from './backgroundCompletionWakeup'; export * from './initialize'; export * from './legacy'; diff --git a/packages/api/src/agents/triggers/engine.spec.ts b/packages/api/src/agents/triggers/engine.spec.ts index 91f24aa8f5..e00a600654 100644 --- a/packages/api/src/agents/triggers/engine.spec.ts +++ b/packages/api/src/agents/triggers/engine.spec.ts @@ -589,6 +589,32 @@ describe('createAgentTriggerDeliveryEngine', () => { expect(store.dead).toHaveBeenCalledWith(expect.objectContaining({ claimToken: 'claim-1' })); }); + it.each([false, true])( + 'offers configured receipt recovery when already exhausted is %s', + async (exhausted) => { + const store = storeWith({ + claimNext: jest.fn(async () => delivery({ attempts: exhausted ? 1 : 0 })), + }); + const engine = createAgentTriggerDeliveryEngine( + { + store, + now: () => START, + dispatch: async () => { + throw new Error('database unavailable'); + }, + }, + { concurrency: 1, maxAttempts: 1, retryCapMs: 123_000 }, + ); + await engine.runTick(); + expect(store.dead).toHaveBeenCalledWith( + expect.objectContaining({ + receiptRetryAt: new Date(START.getTime() + 123_000), + error: expect.objectContaining({ retryable: true }), + }), + ); + }, + ); + it('bounds a persisted last failure before exhausting its source', async () => { const oversized = 'x'.repeat(3_000); const store = storeWith({ diff --git a/packages/api/src/agents/triggers/engine.ts b/packages/api/src/agents/triggers/engine.ts index 164854a0e7..8fd6592a3b 100644 --- a/packages/api/src/agents/triggers/engine.ts +++ b/packages/api/src/agents/triggers/engine.ts @@ -183,6 +183,7 @@ export interface AgentTriggerDeliveryStore { attempt: number; error: AgentTriggerDeliveryFailure; settledAt: Date; + receiptRetryAt?: Date; }) => Promise; } @@ -409,6 +410,9 @@ export function createAgentTriggerDeliveryEngine( attempt: delivery.attempts, error: recorded, settledAt: now(), + ...(recorded.retryable && { + receiptRetryAt: new Date(now().getTime() + retryCapMs), + }), }); if (deadLettered) { logger.error('[agent-triggers] delivery dead-lettered after exhausting retries', { @@ -532,6 +536,9 @@ export function createAgentTriggerDeliveryEngine( attempt, error: recorded, settledAt: attemptedAt, + ...(recorded.retryable && { + receiptRetryAt: new Date(attemptedAt.getTime() + retryCapMs), + }), }); if (deadLettered) { logger.error('[agent-triggers] delivery dead-lettered', { diff --git a/packages/api/src/agents/triggers/service.delivery.spec.ts b/packages/api/src/agents/triggers/service.delivery.spec.ts index bd3977598b..f3b6b4c103 100644 --- a/packages/api/src/agents/triggers/service.delivery.spec.ts +++ b/packages/api/src/agents/triggers/service.delivery.spec.ts @@ -1,4 +1,5 @@ import { + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1, @@ -180,6 +181,7 @@ describe('durable agent trigger service', () => { expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalledWith( expect.objectContaining({ workerCapabilities: [ + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1, AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1, @@ -218,6 +220,7 @@ describe('durable agent trigger service', () => { expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalledWith( expect.objectContaining({ workerCapabilities: [ + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1, ], diff --git a/packages/api/src/agents/triggers/service.ts b/packages/api/src/agents/triggers/service.ts index 029312caee..9775bf5661 100644 --- a/packages/api/src/agents/triggers/service.ts +++ b/packages/api/src/agents/triggers/service.ts @@ -1,4 +1,5 @@ import { + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1, @@ -112,6 +113,9 @@ export interface AgentTriggerDeliveryPersistence { completeAgentTriggerDelivery: AgentTriggerDeliveryStore['complete']; retireAgentTriggerDelivery: AgentTriggerDeliveryMethods['retireAgentTriggerDelivery']; renewAgentTriggerDeliveryProducerLease: AgentTriggerDeliveryMethods['renewAgentTriggerDeliveryProducerLease']; + persistAgentBackgroundToolResult?: AgentTriggerDeliveryMethods['persistAgentBackgroundToolResult']; + getAgentBackgroundToolResultClaim?: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']; + releaseAgentBackgroundToolResultClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims']; retryAgentTriggerDelivery: AgentTriggerDeliveryStore['retry']; deadLetterAgentTriggerDelivery: AgentTriggerDeliveryStore['dead']; getAgentTriggerDelivery: (deliveryKey: string) => Promise; @@ -168,6 +172,19 @@ export interface AgentTriggerService { options?: { onlyIfUnclaimed?: boolean; onlyIfDead?: boolean }, ) => Promise; renewProducerLease: (deliveryKey: string, sourceId: string, leaseUntil: Date) => Promise; + persistBackgroundToolResult: (input: { + deliveryKey: string; + sourceId: string; + result: { + status: 'completed' | 'error' | 'cancelled'; + output: string; + settledAt: Date; + }; + }) => Promise; + getBackgroundToolResultClaim: ( + input: Parameters[0], + ) => ReturnType; + releaseBackgroundToolResultClaims: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims']; drainUser: (userId: string) => Promise; prepareUserPurge: (userId: string, fenceStartedAt: Date, tenantId?: string) => Promise; cancelUserPurge: (userId: string, fenceStartedAt: Date) => Promise; @@ -183,6 +200,7 @@ function createDeliveryStore( methods.claimNextAgentTriggerDelivery({ ...input, workerCapabilities: [ + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1, ...(supportsDetachedActionCompletion() @@ -600,6 +618,21 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A leaseUntil, }), ), + persistBackgroundToolResult: (input) => + runAsSystem(async () => { + const persist = requireMethods().persistAgentBackgroundToolResult; + return persist == null ? false : persist(input); + }), + getBackgroundToolResultClaim: (input) => + runAsSystem(async () => { + const getClaim = requireMethods().getAgentBackgroundToolResultClaim; + return getClaim == null ? null : getClaim(input); + }), + releaseBackgroundToolResultClaims: (input) => + runAsSystem(async () => { + const release = requireMethods().releaseAgentBackgroundToolResultClaims; + return release == null ? false : release(input); + }), drainUser, prepareUserPurge: (userId, fenceStartedAt, tenantId) => runAsSystem(async () => diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts index b582ba9a6d..c20a24d8f3 100644 --- a/packages/data-provider/src/config.spec.ts +++ b/packages/data-provider/src/config.spec.ts @@ -577,6 +577,7 @@ describe('agent background task config', () => { } expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({ completionWakeups: true, + completionResultMaxChars: 24 * 1024, ordinaryToolCancellation: false, }); }); @@ -593,6 +594,7 @@ describe('agent background task config', () => { } expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({ completionWakeups: false, + completionResultMaxChars: 24 * 1024, ordinaryToolCancellation: false, }); }); @@ -609,9 +611,31 @@ describe('agent background task config', () => { } expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({ completionWakeups: true, + completionResultMaxChars: 24 * 1024, ordinaryToolCancellation: true, }); }); + + it('accepts a bounded durable completion result limit', () => { + const result = configSchema.safeParse({ + version: '1.0', + endpoints: { agents: { backgroundTasks: { completionResultMaxChars: 4096 } } }, + }); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.endpoints?.agents?.backgroundTasks?.completionResultMaxChars).toBe(4096); + } + }); + + it.each([0, 64 * 1024 + 1])('rejects an unsafe completion result limit: %s', (limit) => { + expect( + configSchema.safeParse({ + version: '1.0', + endpoints: { agents: { backgroundTasks: { completionResultMaxChars: limit } } }, + }).success, + ).toBe(false); + }); }); describe('speechTab schema', () => { diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 0b0ec4f21e..c39109a7a6 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -22,6 +22,9 @@ import { MIN_BALANCE_RESERVATION_TTL_MS, DEFAULT_BALANCE_RESERVATION_TTL_MS, } from './balance'; + +export const AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT = 24 * 1024; +export const AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX = 64 * 1024; import { MAX_SUBAGENTS, MAX_SUBAGENTS_CEILING, @@ -1465,6 +1468,15 @@ export const agentsEndpointSchema = baseEndpointSchema backgroundTasks: z .object({ completionWakeups: z.boolean().optional().default(true), + /** Maximum terminal output copied into the private completion receipt. + * Generated files remain governed by their separate attachment policy. */ + completionResultMaxChars: z + .number() + .int() + .min(1) + .max(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX) + .optional() + .default(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT), /** Cooperative cancellation for process-local ordinary tools. Off * by default so existing deployments opt into the new control. */ ordinaryToolCancellation: z.boolean().optional().default(false), diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.ts index 0b463796ad..4a2a7f5a23 100644 --- a/packages/data-schemas/src/index.ts +++ b/packages/data-schemas/src/index.ts @@ -67,6 +67,8 @@ export { isCompactionSemanticIndexProjection, } from './types/compaction'; export { + AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS, + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1, diff --git a/packages/data-schemas/src/methods/conversation.spec.ts b/packages/data-schemas/src/methods/conversation.spec.ts index 86c370d375..92ff017399 100644 --- a/packages/data-schemas/src/methods/conversation.spec.ts +++ b/packages/data-schemas/src/methods/conversation.spec.ts @@ -2230,10 +2230,14 @@ describe('Conversation Operations', () => { const deleteAgentQueuedTurns = jest.fn(async () => { transitions.push('queue-retired'); }); + const eraseAgentTriggerDeliveryConversationResults = jest.fn(async () => { + transitions.push('trigger-results-erased'); + }); const scopedMethods = createConversationMethods(mongoose, { getMessages, deleteMessages, deleteAgentQueuedTurns, + eraseAgentTriggerDeliveryConversationResults, }); await scopedMethods.deleteConvos( @@ -2249,7 +2253,99 @@ describe('Conversation Operations', () => { expect(deleteAgentQueuedTurns).toHaveBeenCalledWith('user123', [ { conversationId, tenantId: 'tenant-1' }, ]); - expect(transitions).toEqual(['queue-retired', 'generation-drained']); + expect(eraseAgentTriggerDeliveryConversationResults).toHaveBeenCalledWith('user123', [ + conversationId, + ]); + expect(transitions).toEqual([ + 'queue-retired', + 'generation-drained', + 'trigger-results-erased', + ]); + }); + + it('preserves background receipts when a pre-delete drain fails', async () => { + const conversationId = uuidv4(); + await Conversation.create({ + conversationId, + user: 'user123', + endpoint: EModelEndpoint.agents, + }); + const eraseAgentTriggerDeliveryConversationResults = jest.fn(async () => undefined); + const scopedMethods = createConversationMethods(mongoose, { + getMessages, + deleteMessages, + eraseAgentTriggerDeliveryConversationResults, + }); + + await expect( + scopedMethods.deleteConvos( + 'user123', + { conversationId }, + { beforeDelete: async () => Promise.reject(new Error('drain unavailable')) }, + ), + ).rejects.toThrow('drain unavailable'); + + expect(await Conversation.findOne({ conversationId })).not.toBeNull(); + expect(eraseAgentTriggerDeliveryConversationResults).not.toHaveBeenCalled(); + }); + + it('finishes descendant bookkeeping and message cleanup when receipt erasure fails', async () => { + const parentId = uuidv4(); + const childId = uuidv4(); + const grandchildId = uuidv4(); + const ids = [parentId, childId, grandchildId]; + const project = await ChatProject.create({ user: 'user123', name: 'Receipt cleanup' }); + await ConversationTag.create({ user: 'user123', tag: 'work', count: 3, position: 0 }); + await Conversation.create( + ids.map((conversationId, depth) => ({ + conversationId, + user: 'user123', + endpoint: EModelEndpoint.agents, + tags: ['work'], + chatProjectId: project._id.toString(), + ...(depth === 0 + ? {} + : { + subagentThread: { + rootConversationId: parentId, + parentConversationId: ids[depth - 1], + parentMessageId: `message-${depth}`, + parentToolCallId: `call-${depth}`, + subagentType: 'agent-child', + subagentKind: 'agent', + depth, + }, + }), + })), + ); + const prepare = jest.fn(async () => undefined); + const erase = jest.fn(async () => { + throw new Error('receipt storage unavailable'); + }); + const scopedMethods = createConversationMethods(mongoose, { + getMessages, + deleteMessages, + prepareAgentTriggerConversationResultErasure: prepare, + eraseAgentTriggerDeliveryConversationResults: erase, + }); + + const result = await scopedMethods.deleteConvos('user123', { conversationId: parentId }); + expect(result).toMatchObject({ deletedCount: 3, conversationIds: ids }); + expect(prepare.mock.calls).toHaveLength(3); + expect(erase.mock.calls).toHaveLength(3); + expect(await Conversation.countDocuments({ user: 'user123' })).toBe(0); + expect((await ConversationTag.findOne({ user: 'user123', tag: 'work' }).lean())?.count).toBe( + 0, + ); + expect((await ChatProject.findById(project._id).lean())?.conversationCount).toBe(0); + expect(deleteMessages).toHaveBeenCalledWith({ + user: 'user123', + conversationId: { $in: ids }, + }); + + await expect( + scopedMethods.deleteConvos('user123', { conversationId: parentId }, { allowEmpty: true }), + ).resolves.toMatchObject({ deletedCount: 0, conversationIds: [parentId] }); }); it('fails closed before deleting a conversation when queued-turn retirement fails', async () => { @@ -2468,13 +2564,22 @@ describe('Conversation Operations', () => { }); it('allows a single-conversation cleanup retry after topology and messages are gone', async () => { - const result = await deleteConvos( + const eraseAgentTriggerDeliveryConversationResults = jest.fn(async () => undefined); + const scopedMethods = createConversationMethods(mongoose, { + getMessages, + deleteMessages, + eraseAgentTriggerDeliveryConversationResults, + }); + const result = await scopedMethods.deleteConvos( 'user123', { conversationId: 'already-absent' }, { allowEmpty: true }, ); expect(result.deletedCount).toBe(0); expect(result.conversationIds).toEqual(['already-absent']); + expect(eraseAgentTriggerDeliveryConversationResults).toHaveBeenCalledWith('user123', [ + 'already-absent', + ]); }); it('supports an idempotent empty recovery sweep without hiding storage failures', async () => { diff --git a/packages/data-schemas/src/methods/conversation.ts b/packages/data-schemas/src/methods/conversation.ts index ec719499cb..3d44b0e641 100644 --- a/packages/data-schemas/src/methods/conversation.ts +++ b/packages/data-schemas/src/methods/conversation.ts @@ -512,6 +512,14 @@ export interface ConversationMethodDeps user: string, conversations: Array<{ conversationId: string; tenantId?: string; allTenants?: true }>, ) => Promise; + eraseAgentTriggerDeliveryConversationResults?: ( + user: string, + conversationIds: string[], + ) => Promise; + prepareAgentTriggerConversationResultErasure?: ( + user: string, + conversationIds: string[], + ) => Promise; } export function createConversationMethods( @@ -3235,7 +3243,18 @@ export function createConversationMethods( })), ); await options?.beforeDelete?.(waveIds); + await deps?.prepareAgentTriggerConversationResultErasure?.(user, waveIds); const result = await Conversation.deleteMany({ user, conversationId: { $in: waveIds } }); + if (result.deletedCount > 0) { + /** Result erasure is irreversible. Keep receipts intact when a + * pre-delete hook or the conversation delete itself fails, so a + * retained conversation cannot lose a receipt-only completion. */ + try { + await deps?.eraseAgentTriggerDeliveryConversationResults?.(user, waveIds); + } catch (error) { + logger.error('[deleteConvos] Receipt erasure deferred to durable cleanup', error); + } + } acknowledged &&= result.acknowledged; deletedCount += result.deletedCount; await reconcileDeletedWave(wave, result.deletedCount); @@ -3266,6 +3285,11 @@ export function createConversationMethods( allTenants: true, })), ); + try { + await deps?.eraseAgentTriggerDeliveryConversationResults?.(user, recoveryConversationIds); + } catch (error) { + logger.error('[deleteConvos] Receipt erasure deferred to durable cleanup', error); + } } const deleteConvoResult: DeleteResult = { acknowledged, deletedCount }; diff --git a/packages/data-schemas/src/methods/index.ts b/packages/data-schemas/src/methods/index.ts index f84fd5f1e2..3ddf1be4b4 100644 --- a/packages/data-schemas/src/methods/index.ts +++ b/packages/data-schemas/src/methods/index.ts @@ -325,6 +325,10 @@ export function createMethods( getMessages: messageMethods.getMessages, deleteMessages: messageMethods.deleteMessages, searchMessages: messageMethods.searchMessages, + eraseAgentTriggerDeliveryConversationResults: + agentTriggerDeliveryMethods.eraseAgentTriggerDeliveryConversationResults, + prepareAgentTriggerConversationResultErasure: + agentTriggerDeliveryMethods.prepareAgentTriggerConversationResultErasure, deleteAgentQueuedTurns: async (user, conversations) => { /** Queued-turn ownership is ObjectId-backed. Conversation methods also * support synthetic/non-ObjectId owners in embedded integrations and diff --git a/packages/data-schemas/src/methods/message.ts b/packages/data-schemas/src/methods/message.ts index c8aae0d623..68b23bb1fc 100644 --- a/packages/data-schemas/src/methods/message.ts +++ b/packages/data-schemas/src/methods/message.ts @@ -726,6 +726,7 @@ export interface MessageMethods { cancelled?: true; settledAt: Date; completionWakeup?: true; + completionReceipt?: true; resultClaim?: { kind: 'manual' | 'wakeup'; claimId: string; @@ -1224,6 +1225,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa cancelled?: true; settledAt: Date; completionWakeup?: true; + completionReceipt?: true; resultClaim?: { kind: 'manual' | 'wakeup'; claimId: string; @@ -1274,6 +1276,9 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa partPatch['content.$[part].tool_call.backgroundTask.cancelled'] = backgroundTask.cancelled === true; partPatch['content.$[part].tool_call.backgroundTask.settledAt'] = backgroundTask.settledAt; + if (backgroundTask.completionReceipt === true) { + partPatch['content.$[part].tool_call.backgroundTask.completionReceipt'] = true; + } if (backgroundTask.completionWakeup === true) { partPatch['content.$[part].tool_call.backgroundTask.completionWakeup'] = true; } else { diff --git a/packages/data-schemas/src/methods/triggerDelivery.spec.ts b/packages/data-schemas/src/methods/triggerDelivery.spec.ts index 6ca5b42499..1f0fcc74f6 100644 --- a/packages/data-schemas/src/methods/triggerDelivery.spec.ts +++ b/packages/data-schemas/src/methods/triggerDelivery.spec.ts @@ -18,12 +18,14 @@ import { type AgentTriggerDeliveryMethods, } from './triggerDelivery'; import { + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1, } from '~/types/triggerDelivery'; import { createAgentTriggerLaneSequenceModel } from '../models/triggerLaneSequence'; import { createAgentTriggerUserPurgeModel } from '../models/triggerUserPurge'; import { createAgentTriggerDeliveryModel } from '../models/triggerDelivery'; +import { createConversationModel } from '../models/convo'; import { createUserModel } from '../models/user'; jest.mock('~/config/winston', () => ({ @@ -32,6 +34,7 @@ jest.mock('~/config/winston', () => ({ info: jest.fn(), debug: jest.fn(), })); +jest.mock('~/models/plugins/mongoMeili', () => jest.fn()); const DB_SETUP_TIMEOUT_MS = 60_000; const START = new Date('2026-08-17T12:00:00.000Z'); @@ -50,6 +53,7 @@ beforeAll(async () => { LaneSequence = createAgentTriggerLaneSequenceModel(mongoose); UserPurge = createAgentTriggerUserPurgeModel(mongoose); User = createUserModel(mongoose); + createConversationModel(mongoose); await Promise.all([Delivery.init(), LaneSequence.init(), UserPurge.init(), User.init()]); methods = createAgentTriggerDeliveryMethods(mongoose); }, DB_SETUP_TIMEOUT_MS); @@ -384,7 +388,7 @@ describe('agent trigger delivery methods', () => { it('shields background completion work from workers that cannot resolve it', async () => { const queued = await methods.enqueueAgentTriggerDelivery( enqueueInput({ - requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, }), ); const claimInput = { @@ -400,7 +404,7 @@ describe('agent trigger delivery methods', () => { ...claimInput, workerId: 'background-capable-worker', claimToken: 'background-capable-claim', - workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1], + workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2], }), ).resolves.toMatchObject({ id: queued.delivery.id }); }); @@ -412,7 +416,7 @@ describe('agent trigger delivery methods', () => { enqueueInput({ deliveryKey: 'background-completion-producer-lease', envelope: { event: { source } }, - requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, producerLeaseUntil: initialLease, }), ); @@ -449,6 +453,323 @@ describe('agent trigger delivery methods', () => { ).resolves.toEqual({ status: 'live', leaseUntil: renewedUntil }); }); + it('persists one private background result receipt independently of message rows', async () => { + const source = { id: 'background-tool-completion', type: 'internal' }; + const queued = await methods.enqueueAgentTriggerDelivery( + enqueueInput({ + deliveryKey: 'background-completion-result-receipt', + envelope: { event: { source } }, + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + }), + ); + const input = { + deliveryKey: queued.delivery.deliveryKey, + sourceId: source.id, + result: { status: 'completed' as const, output: 'durable output', settledAt: START }, + }; + + await expect(methods.persistAgentBackgroundToolResult(input)).resolves.toBe(true); + await expect(methods.persistAgentBackgroundToolResult(input)).resolves.toBe(true); + await expect( + methods.getAgentBackgroundToolResult({ + deliveryKey: queued.delivery.deliveryKey, + sourceId: source.id, + }), + ).resolves.toEqual(input.result); + await expect( + methods.persistAgentBackgroundToolResult({ + ...input, + result: { ...input.result, output: 'conflicting output' }, + }), + ).resolves.toBe(false); + const ordinaryRead = await Delivery.findById(queued.delivery.id).lean(); + expect(ordinaryRead).not.toHaveProperty('backgroundToolResult'); + }); + + it('erases private result receipts when their conversation is deleted', async () => { + const source = { id: 'background-tool-completion', type: 'internal' }; + const user = new mongoose.Types.ObjectId(); + const queued = await methods.enqueueAgentTriggerDelivery( + enqueueInput({ + deliveryKey: 'background-completion-result-to-erase', + user, + envelope: { + event: { source }, + target: { conversationId: 'conversation-to-delete' }, + }, + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + }), + ); + await methods.persistAgentBackgroundToolResult({ + deliveryKey: queued.delivery.deliveryKey, + sourceId: source.id, + result: { status: 'completed', output: 'private output', settledAt: START }, + }); + + await methods.eraseAgentTriggerDeliveryConversationResults(user, ['conversation-to-delete']); + + await expect( + methods.getAgentBackgroundToolResult({ + deliveryKey: queued.delivery.deliveryKey, + sourceId: source.id, + }), + ).resolves.toBeNull(); + await expect(Delivery.exists({ _id: queued.delivery.id })).resolves.not.toBeNull(); + await expect( + methods.persistAgentBackgroundToolResult({ + deliveryKey: queued.delivery.deliveryKey, + sourceId: source.id, + result: { status: 'completed', output: 'late private output', settledAt: START }, + }), + ).resolves.toBe(false); + }); + + it('does not make legacy string-owned conversation cleanup depend on an ObjectId cast', async () => { + await expect( + methods.prepareAgentTriggerConversationResultErasure('legacy-user', ['conversation']), + ).resolves.toBeUndefined(); + await expect( + methods.eraseAgentTriggerDeliveryConversationResults('legacy-user', ['conversation']), + ).resolves.toBeUndefined(); + }); + + it('recovers receipt erasure after a descendant row was already deleted', async () => { + const user = new mongoose.Types.ObjectId(); + const source = { id: 'background-tool-completion', type: 'internal' }; + const queued = await methods.enqueueAgentTriggerDelivery( + enqueueInput({ + user, + envelope: { event: { source }, target: { conversationId: 'deleted-child' } }, + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + }), + ); + const lookup = { deliveryKey: queued.delivery.deliveryKey, sourceId: source.id }; + await methods.persistAgentBackgroundToolResult({ + ...lookup, + result: { status: 'completed', output: 'private child output', settledAt: START }, + }); + await methods.prepareAgentTriggerConversationResultErasure(user, ['deleted-child']); + await mongoose.models.Conversation.collection.insertOne({ + user: user.toString(), + conversationId: 'deleted-child', + }); + await methods.recoverAgentTriggerUserPurges(); + await expect(methods.getAgentBackgroundToolResult(lookup)).resolves.not.toBeNull(); + await mongoose.models.Conversation.deleteOne({ user, conversationId: 'deleted-child' }); + /** The conversation deletion committed, but the follow-up receipt erasure did not. */ + await methods.recoverAgentTriggerUserPurges(); + await expect(methods.getAgentBackgroundToolResult(lookup)).resolves.toBeNull(); + await expect( + Delivery.exists({ + _id: queued.delivery.id, + backgroundToolResultDeletionPendingAt: { $exists: true }, + }), + ).resolves.toBeNull(); + await expect( + methods.persistAgentBackgroundToolResult({ + ...lookup, + result: { status: 'completed', output: 'late write', settledAt: START }, + }), + ).resolves.toBe(false); + }); + + it.each([ + ['receipt', 'BACKGROUND_TOOL_PRODUCER_LOST'], + ['death', 'BACKGROUND_TOOL_PRODUCER_LOST'], + ['receipt', 'PARENT_STATE_UNAVAILABLE'], + ['death', 'PARENT_STATE_UNAVAILABLE'], + ])('atomically fences result publication when %s wins before %s', async (winner, code) => { + const source = { id: 'background-tool-completion', type: 'internal' }; + const queued = await methods.enqueueAgentTriggerDelivery( + enqueueInput({ + envelope: { event: { source } }, + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + }), + ); + const owner = { workerId: 'worker', claimToken: 'claim', now: START }; + const claimed = await methods.claimNextAgentTriggerDelivery({ + ...owner, + leaseUntil: new Date(START.getTime() + 60_000), + workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2], + }); + expect(claimed).not.toBeNull(); + const attempt = await methods.beginAgentTriggerDeliveryAttempt({ ...owner, id: claimed!.id }); + const write = () => + methods.persistAgentBackgroundToolResult({ + deliveryKey: queued.delivery.deliveryKey, + sourceId: source.id, + result: { status: 'completed', output: 'late output', settledAt: START }, + }); + const dead = () => + methods.deadLetterAgentTriggerDelivery({ + ...owner, + id: claimed!.id, + attempt: attempt!, + settledAt: START, + error: transientFailure({ code, retryable: code !== 'BACKGROUND_TOOL_PRODUCER_LOST' }), + }); + await expect(winner === 'receipt' ? write() : dead()).resolves.toBe(true); + await expect(winner === 'receipt' ? dead() : write()).resolves.toBe(false); + }); + + it.each(['shield', 'legacy'] as const)( + 'recovers a committed receipt after generic retry exhaustion through the %s fence', + async (format) => { + const source = { id: 'background-tool-completion', type: 'internal' }; + const queued = await methods.enqueueAgentTriggerDelivery( + enqueueInput({ + envelope: { event: { source } }, + requiredWorkerCapability: + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + }), + ); + const owner = { workerId: 'worker', claimToken: 'claim', now: START }; + const claimed = await methods.claimNextAgentTriggerDelivery({ + ...owner, + leaseUntil: new Date(START.getTime() + 60_000), + workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2], + }); + const id = claimed!.id; + const attempt = await methods.beginAgentTriggerDeliveryAttempt({ ...owner, id }); + if (format !== 'shield') { + await Delivery.updateOne( + { _id: id }, + { + $set: { + status: 'capability_leased', + leaseBy: owner.workerId, + claimToken: owner.claimToken, + leaseUntil: new Date(START.getTime() + 60_000), + }, + $unset: { + capabilityStatus: 1, + capabilityLeaseBy: 1, + capabilityLeaseUntil: 1, + capabilityClaimToken: 1, + }, + }, + ); + } + const receipt = { deliveryKey: queued.delivery.deliveryKey, sourceId: source.id }; + await methods.persistAgentBackgroundToolResult({ + ...receipt, + result: { status: 'completed', output: 'saved output', settledAt: START }, + }); + const receiptRetryAt = new Date(START.getTime() + 300_000); + const terminal = { + ...owner, + id, + attempt: attempt!, + settledAt: START, + receiptRetryAt, + error: transientFailure({ code: 'PARENT_STATE_UNAVAILABLE' }), + }; + await expect( + methods.deadLetterAgentTriggerDelivery({ ...terminal, claimToken: 'stale' }), + ).resolves.toBe(false); + expect((await Delivery.findById(id).lean())?.attempts).toBe(attempt); + await expect(methods.deadLetterAgentTriggerDelivery(terminal)).resolves.toBe(false); + expect(await Delivery.findById(id).lean()).toMatchObject({ + attempts: 0, + claimAvailableAt: receiptRetryAt, + ...(format === 'shield' + ? { capabilityStatus: 'pending' } + : { status: 'capability_pending' }), + }); + await expect(methods.getAgentBackgroundToolResult(receipt)).resolves.toMatchObject({ + output: 'saved output', + }); + const recovered = await methods.claimNextAgentTriggerDelivery({ + ...owner, + claimToken: 'recovered', + now: receiptRetryAt, + leaseUntil: new Date(receiptRetryAt.getTime() + 60_000), + workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2], + }); + expect(recovered?.id).toBe(id); + expect( + await methods.beginAgentTriggerDeliveryAttempt({ + ...owner, + id, + claimToken: 'recovered', + now: receiptRetryAt, + }), + ).toBe(1); + await expect( + methods.deadLetterAgentTriggerDelivery({ + ...terminal, + claimToken: 'recovered', + settledAt: receiptRetryAt, + error: transientFailure({ code: 'PARENT_NOT_FOUND', retryable: false }), + }), + ).resolves.toBe(true); + }, + ); + + it('claims an independent result receipt before returning its output', async () => { + const source = { id: 'background-tool-completion', type: 'internal' }; + const user = new mongoose.Types.ObjectId(); + const queued = await methods.enqueueAgentTriggerDelivery( + enqueueInput({ + deliveryKey: 'background-completion-result-to-claim', + user, + envelope: { + event: { + source, + payload: { taskId: 'task-1', toolCallId: 'call-1', toolName: 'slow_tool' }, + }, + target: { + agentId: 'agent-1', + conversationId: 'conversation-1', + parentMessageId: 'response-1', + }, + }, + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + }), + ); + await methods.persistAgentBackgroundToolResult({ + deliveryKey: queued.delivery.deliveryKey, + sourceId: source.id, + result: { status: 'completed', output: 'private output', settledAt: START }, + }); + const claim = { + deliveryKey: queued.delivery.deliveryKey, + sourceId: source.id, + userId: user.toString(), + conversationId: 'conversation-1', + parentMessageId: 'response-1', + agentId: 'agent-1', + claimId: 'delivery-claim-1', + limit: 1, + }; + + await expect(methods.claimAgentBackgroundToolResults(claim)).resolves.toMatchObject({ + status: 'acquired', + results: [{ taskId: 'task-1', output: 'private output' }], + }); + await expect( + methods.claimAgentBackgroundToolResults({ ...claim, claimId: 'delivery-claim-2' }), + ).resolves.toEqual({ status: 'claimed', claimId: 'delivery-claim-1' }); + await expect( + methods.getAgentBackgroundToolResultClaim({ + sourceId: source.id, + userId: user.toString(), + conversationId: 'conversation-1', + parentMessageId: 'response-1', + taskId: 'task-1', + }), + ).resolves.toMatchObject({ kind: 'wakeup', claimId: 'delivery-claim-1' }); + await expect( + methods.releaseAgentBackgroundToolResultClaims({ + sourceId: source.id, + userId: user.toString(), + conversationId: 'conversation-1', + parentMessageId: 'response-1', + claimId: 'delivery-claim-1', + }), + ).resolves.toBe(true); + }); + it('keeps capability-fenced work limited to capable workers through lease recovery', async () => { const queued = await methods.enqueueAgentTriggerDelivery( enqueueInput({ diff --git a/packages/data-schemas/src/methods/triggerDelivery.ts b/packages/data-schemas/src/methods/triggerDelivery.ts index a4569db5fd..f2d12193b0 100644 --- a/packages/data-schemas/src/methods/triggerDelivery.ts +++ b/packages/data-schemas/src/methods/triggerDelivery.ts @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto'; import type { FilterQuery, Model, Types } from 'mongoose'; import type { + AgentBackgroundToolResultReceipt, AgentEventActorDetachedAction, AgentTriggerDeliveryClaim, AgentEventActorReceipt, @@ -17,6 +18,8 @@ import type { IAgentTriggerUserPurgeDocument, } from '~/types/triggerDelivery'; import { + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1, AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1, @@ -129,6 +132,26 @@ export type AgentTriggerProducerLeaseStatus = | { status: 'expired'; leaseUntil: Date } | { status: 'missing' }; +export interface PersistAgentBackgroundToolResultInput { + deliveryKey: string; + sourceId: string; + result: AgentBackgroundToolResultReceipt; +} + +export type AgentBackgroundToolResultClaim = + | { status: 'not_ready' } + | { status: 'claimed'; claimId: string } + | { + status: 'acquired'; + results: Array<{ + taskId: string; + toolCallId: string; + toolName: string; + status: AgentBackgroundToolResultReceipt['status']; + output: string; + }>; + }; + export interface AgentTriggerDeliveryFence { id: string; workerId: string; @@ -277,6 +300,37 @@ export interface AgentTriggerDeliveryMethods { sourceId: string; now: Date; }) => Promise; + persistAgentBackgroundToolResult: ( + input: PersistAgentBackgroundToolResultInput, + ) => Promise; + getAgentBackgroundToolResult: (input: { + deliveryKey: string; + sourceId: string; + }) => Promise; + getAgentBackgroundToolResultClaim: (input: { + sourceId: string; + userId: string; + conversationId: string; + parentMessageId: string; + taskId: string; + }) => Promise; + claimAgentBackgroundToolResults: (input: { + deliveryKey: string; + sourceId: string; + userId: string; + conversationId: string; + parentMessageId: string; + agentId?: string; + claimId: string; + limit?: number; + }) => Promise; + releaseAgentBackgroundToolResultClaims: (input: { + sourceId: string; + userId: string; + conversationId: string; + parentMessageId: string; + claimId: string; + }) => Promise; settleAgentTriggerHandlingOutcome: ( input: SettleAgentTriggerHandlingOutcomeInput, ) => Promise; @@ -326,6 +380,7 @@ export interface AgentTriggerDeliveryMethods { attempt: number; error: AgentTriggerDeliveryFailure; settledAt: Date; + receiptRetryAt?: Date; }, ) => Promise; getAgentTriggerDelivery: (deliveryKey: string) => Promise; @@ -358,6 +413,14 @@ export interface AgentTriggerDeliveryMethods { ) => Promise; recoverAgentTriggerUserPurges: (limit?: number) => Promise; deleteAgentTriggerDeliveriesByUser: (user: string | Types.ObjectId) => Promise; + eraseAgentTriggerDeliveryConversationResults: ( + user: string | Types.ObjectId, + conversationIds: string[], + ) => Promise; + prepareAgentTriggerConversationResultErasure: ( + user: string | Types.ObjectId, + conversationIds: string[], + ) => Promise; } function normalizeFailure(error: AgentTriggerDeliveryFailure): AgentTriggerDeliveryFailure { @@ -960,6 +1023,8 @@ export function createAgentTriggerDeliveryMethods( if ( input.requiredWorkerCapability != null && input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 && + input.requiredWorkerCapability !== + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 && input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 && input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1 ) { @@ -2122,7 +2187,12 @@ export function createAgentTriggerDeliveryMethods( const renewed = await Delivery().updateOne( { deliveryKey: input.deliveryKey, - requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, + requiredWorkerCapability: { + $in: [ + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + ], + }, 'envelope.event.source.type': 'internal', 'envelope.event.source.id': input.sourceId, status: { @@ -2161,7 +2231,12 @@ export function createAgentTriggerDeliveryMethods( const delivery = await Delivery() .findOne({ deliveryKey: input.deliveryKey, - requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, + requiredWorkerCapability: { + $in: [ + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1, + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + ], + }, 'envelope.event.source.type': 'internal', 'envelope.event.source.id': input.sourceId, }) @@ -2175,6 +2250,262 @@ export function createAgentTriggerDeliveryMethods( : { status: 'expired', leaseUntil: delivery.producerLeaseUntil }; } + /** Stores terminal output on the pre-admitted delivery before attempting the + * parent-message projection. The first terminal receipt wins; exact retries + * are idempotent and conflicting rewrites fail closed. */ + async function persistAgentBackgroundToolResult( + input: PersistAgentBackgroundToolResultInput, + ): Promise { + const { result } = input; + if ( + input.deliveryKey.length === 0 || + input.deliveryKey.length > 256 || + input.sourceId.length === 0 || + input.sourceId.length > 256 || + !['completed', 'error', 'cancelled'].includes(result.status) || + result.output.length > AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS || + !(result.settledAt instanceof Date) || + !Number.isFinite(result.settledAt.getTime()) + ) { + throw new TypeError('Invalid background tool result receipt'); + } + const identity = { + deliveryKey: input.deliveryKey, + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + 'envelope.event.source.type': 'internal', + 'envelope.event.source.id': input.sourceId, + backgroundToolResultErasedAt: { $exists: false }, + }; + const updated = await Delivery().updateOne( + { + ...identity, + backgroundToolResult: { $exists: false }, + status: { $nin: ['dead', 'capability_dead', 'succeeded'] }, + capabilityStatus: { $ne: 'dead' }, + }, + { $set: { backgroundToolResult: result } }, + { timestamps: false }, + ); + if (updated.matchedCount === 1) { + return true; + } + const existing = await Delivery() + .findOne(identity) + .select('+backgroundToolResult') + .lean>(); + return ( + existing?.backgroundToolResult?.status === result.status && + existing.backgroundToolResult.output === result.output && + existing.backgroundToolResult.settledAt.getTime() === result.settledAt.getTime() + ); + } + + async function getAgentBackgroundToolResult(input: { + deliveryKey: string; + sourceId: string; + }): Promise { + if ( + input.deliveryKey.length === 0 || + input.deliveryKey.length > 256 || + input.sourceId.length === 0 || + input.sourceId.length > 256 + ) { + throw new TypeError('Invalid background tool result receipt lookup'); + } + const delivery = await Delivery() + .findOne({ + deliveryKey: input.deliveryKey, + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + 'envelope.event.source.type': 'internal', + 'envelope.event.source.id': input.sourceId, + }) + .select('+backgroundToolResult') + .lean>(); + return delivery?.backgroundToolResult ?? null; + } + + async function getAgentBackgroundToolResultClaim(input: { + sourceId: string; + userId: string; + conversationId: string; + parentMessageId: string; + taskId: string; + }): Promise { + const delivery = await Delivery() + .findOne({ + ...backgroundResultIdentity(input), + 'envelope.event.payload.taskId': input.taskId, + 'backgroundToolResult.resultClaim': { $exists: true }, + }) + .select('+backgroundToolResult') + .lean>(); + return delivery?.backgroundToolResult?.resultClaim ?? null; + } + + function backgroundResultIdentity(input: { + sourceId: string; + userId: string; + conversationId: string; + parentMessageId: string; + agentId?: string; + }) { + return { + user: input.userId, + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + 'envelope.event.source.type': 'internal', + 'envelope.event.source.id': input.sourceId, + 'envelope.target.conversationId': input.conversationId, + 'envelope.target.parentMessageId': input.parentMessageId, + ...(input.agentId == null ? {} : { 'envelope.target.agentId': input.agentId }), + }; + } + + function projectBackgroundResult(delivery: { + envelope?: unknown; + backgroundToolResult?: AgentBackgroundToolResultReceipt; + }) { + const envelope = delivery.envelope; + if (envelope == null || typeof envelope !== 'object' || Array.isArray(envelope)) { + return null; + } + const event = (envelope as { event?: unknown }).event; + if (event == null || typeof event !== 'object' || Array.isArray(event)) { + return null; + } + const payload = (event as { payload?: unknown }).payload; + if (payload == null || typeof payload !== 'object' || Array.isArray(payload)) { + return null; + } + const { taskId, toolCallId, toolName } = payload as Record; + const receipt = delivery.backgroundToolResult; + if ( + receipt == null || + typeof taskId !== 'string' || + typeof toolCallId !== 'string' || + typeof toolName !== 'string' + ) { + return null; + } + return { + taskId, + toolCallId, + toolName, + status: receipt.status, + output: receipt.output, + }; + } + + /** Elects exactly one automatic continuation for an independently persisted + * result. A later stack layer may coalesce siblings after this ownership + * boundary, but the durable receipt itself is never dispatched unclaimed. */ + async function claimAgentBackgroundToolResults(input: { + deliveryKey: string; + sourceId: string; + userId: string; + conversationId: string; + parentMessageId: string; + agentId?: string; + claimId: string; + limit?: number; + }): Promise { + if ( + input.deliveryKey.length === 0 || + input.deliveryKey.length > 256 || + input.sourceId.length === 0 || + input.sourceId.length > 256 || + input.userId.length === 0 || + input.conversationId.length === 0 || + input.conversationId.length > 256 || + input.parentMessageId.length === 0 || + input.parentMessageId.length > 256 || + input.claimId.length === 0 || + input.claimId.length > 128 + ) { + throw new TypeError('Invalid background tool result receipt claim'); + } + const scope = backgroundResultIdentity(input); + const projection = 'envelope +backgroundToolResult'; + let requested = await Delivery() + .findOne({ ...scope, deliveryKey: input.deliveryKey }) + .select(projection) + .lean>(); + if (requested?.backgroundToolResult == null) { + return { status: 'not_ready' }; + } + const existingClaim = requested.backgroundToolResult.resultClaim; + if (existingClaim != null && existingClaim.claimId !== input.claimId) { + return { status: 'claimed', claimId: existingClaim.claimId }; + } + if (existingClaim == null) { + const claimed = await Delivery().updateOne( + { + ...scope, + deliveryKey: input.deliveryKey, + backgroundToolResult: { $exists: true }, + 'backgroundToolResult.resultClaim': { $exists: false }, + }, + { + $set: { + 'backgroundToolResult.resultClaim': { + kind: 'wakeup', + claimId: input.claimId, + claimedAt: new Date(), + }, + }, + }, + { timestamps: false }, + ); + if (claimed.modifiedCount !== 1) { + requested = await Delivery() + .findOne({ ...scope, deliveryKey: input.deliveryKey }) + .select(projection) + .lean>(); + const winner = requested?.backgroundToolResult?.resultClaim; + if (winner?.claimId !== input.claimId) { + return winner == null + ? { status: 'not_ready' } + : { status: 'claimed', claimId: winner.claimId }; + } + } + } + const result = requested == null ? null : projectBackgroundResult(requested); + return result == null ? { status: 'not_ready' } : { status: 'acquired', results: [result] }; + } + + async function releaseAgentBackgroundToolResultClaims(input: { + sourceId: string; + userId: string; + conversationId: string; + parentMessageId: string; + claimId: string; + }): Promise { + if ( + input.sourceId.length === 0 || + input.sourceId.length > 256 || + input.userId.length === 0 || + input.conversationId.length === 0 || + input.conversationId.length > 256 || + input.parentMessageId.length === 0 || + input.parentMessageId.length > 256 || + input.claimId.length === 0 || + input.claimId.length > 128 + ) { + throw new TypeError('Invalid background tool result receipt claim release'); + } + const scope = backgroundResultIdentity(input); + await Delivery().updateMany( + { ...scope, 'backgroundToolResult.resultClaim.claimId': input.claimId }, + { $unset: { 'backgroundToolResult.resultClaim': 1 } }, + { timestamps: false }, + ); + return ( + (await Delivery().exists({ + ...scope, + 'backgroundToolResult.resultClaim.claimId': input.claimId, + })) == null + ); + } + async function settleAgentTriggerHandlingOutcome( input: SettleAgentTriggerHandlingOutcomeInput, ): Promise { @@ -3076,10 +3407,26 @@ export function createAgentTriggerDeliveryMethods( error: AgentTriggerDeliveryFailure; availableAt: Date; }, + ): Promise { + return retryDelivery(input); + } + + async function retryDelivery( + input: Parameters[0], + recoverReceipt = false, ): Promise { const error = normalizeFailure(input.error); + const receiptFence = recoverReceipt + ? { + requiredWorkerCapability: + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + backgroundToolResult: { $exists: true }, + } + : {}; + const recoveredAttempt = recoverReceipt ? { attempts: 0 } : {}; const retryUpdate = (status: 'pending' | 'capability_pending') => ({ $set: { + ...recoveredAttempt, status, availableAt: input.availableAt, claimAvailableAt: input.availableAt, @@ -3108,9 +3455,10 @@ export function createAgentTriggerDeliveryMethods( }, }); const shieldResult = await Delivery().updateOne( - { _id: input.id, ...shieldCapabilityFence(input) }, + { _id: input.id, ...shieldCapabilityFence(input), ...receiptFence }, { $set: { + ...recoveredAttempt, status: 'leased', availableAt: input.availableAt, capabilityStatus: 'pending', @@ -3134,14 +3482,14 @@ export function createAgentTriggerDeliveryMethods( return true; } const capabilityResult = await Delivery().updateOne( - { _id: input.id, ...legacyCapabilityFence(input) }, + { _id: input.id, ...legacyCapabilityFence(input), ...receiptFence }, retryUpdate('capability_pending'), ); if (capabilityResult.modifiedCount === 1) { return true; } const result = await Delivery().updateOne( - { _id: input.id, ...ordinaryFence(input) }, + { _id: input.id, ...ordinaryFence(input), ...receiptFence }, retryUpdate('pending'), ); return result.modifiedCount === 1; @@ -3152,9 +3500,23 @@ export function createAgentTriggerDeliveryMethods( attempt: number; error: AgentTriggerDeliveryFailure; settledAt: Date; + receiptRetryAt?: Date; }, ): Promise { const error = normalizeFailure(input.error); + if ( + error.retryable && + input.receiptRetryAt != null && + (await retryDelivery({ ...input, availableAt: input.receiptRetryAt }, true)) + ) { + return false; + } + /** Publication must contend with every retryable terminal transition, + * including a receipt committed between recovery and the dead-letter CAS. */ + const receiptFence = + error.retryable || error.code === 'BACKGROUND_TOOL_PRODUCER_LOST' + ? { backgroundToolResult: { $exists: false } } + : {}; const deadUpdate = (status: 'dead' | 'capability_dead') => ({ $set: { status, settledAt: input.settledAt, lastError: error }, $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1, expiresAt: 1 }, @@ -3174,7 +3536,7 @@ export function createAgentTriggerDeliveryMethods( }, }); const shieldDead = await Delivery().updateOne( - { _id: input.id, ...shieldCapabilityFence(input) }, + { _id: input.id, ...shieldCapabilityFence(input), ...receiptFence }, { $set: { // The pre-capability runtime already treats `capability_dead` as a @@ -3201,16 +3563,20 @@ export function createAgentTriggerDeliveryMethods( return true; } const capabilityDead = await Delivery().updateOne( - { _id: input.id, ...legacyCapabilityFence(input) }, + { _id: input.id, ...legacyCapabilityFence(input), ...receiptFence }, deadUpdate('capability_dead'), ); if (capabilityDead.modifiedCount === 1) { return true; } const dead = await Delivery() - .findOneAndUpdate({ _id: input.id, ...ordinaryFence(input) }, deadUpdate('dead'), { - new: true, - }) + .findOneAndUpdate( + { _id: input.id, ...ordinaryFence(input), ...receiptFence }, + deadUpdate('dead'), + { + new: true, + }, + ) .select('_id orderingKey batchMemberIds batchMembersSettledAt requeueCount') .lean< Pick< @@ -3512,6 +3878,31 @@ export function createAgentTriggerDeliveryMethods( if (!Number.isSafeInteger(limit) || limit <= 0) { throw new TypeError('Agent trigger purge recovery limit must be a positive integer'); } + const pending = await Delivery() + .find({ + backgroundToolResultDeletionPendingAt: { $exists: true }, + }) + .select('user envelope') + .sort({ backgroundToolResultDeletionPendingAt: 1, _id: 1 }) + .limit(Math.min(limit, MAX_PURGE_RECOVERY_LIMIT)) + .lean(); + for (const row of pending) { + const conversationId = (row.envelope as { target?: { conversationId?: string } }).target + ?.conversationId; + if (conversationId == null) { + continue; + } + const exists = await mongoose.models.Conversation.exists({ user: row.user, conversationId }); + if (exists == null) { + await eraseAgentTriggerDeliveryConversationResults(row.user, [conversationId]); + } else { + await Delivery().updateOne( + { _id: row._id, backgroundToolResultDeletionPendingAt: { $exists: true } }, + { $currentDate: { backgroundToolResultDeletionPendingAt: true } }, + { timestamps: false }, + ); + } + } const markers = await UserPurge() .find({}) .sort({ updatedAt: 1, _id: 1 }) @@ -3565,6 +3956,44 @@ export function createAgentTriggerDeliveryMethods( await UserPurge().deleteOne({ _id: user }); } + async function eraseAgentTriggerDeliveryConversationResults( + user: string | Types.ObjectId, + conversationIds: string[], + ): Promise { + if (conversationIds.length === 0 || !mongoose.isObjectIdOrHexString(user)) { + return; + } + const erasedAt = new Date(); + await Delivery().updateMany( + { + user, + 'envelope.target.conversationId': { $in: conversationIds }, + }, + { + $set: { backgroundToolResultErasedAt: erasedAt }, + $unset: { backgroundToolResult: 1, backgroundToolResultDeletionPendingAt: 1 }, + }, + { timestamps: false }, + ); + } + + async function prepareAgentTriggerConversationResultErasure( + user: string | Types.ObjectId, + conversationIds: string[], + ): Promise { + if (conversationIds.length === 0 || !mongoose.isObjectIdOrHexString(user)) return; + await Delivery().updateMany( + { + user, + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + 'envelope.target.conversationId': { $in: conversationIds }, + backgroundToolResultErasedAt: { $exists: false }, + }, + { $set: { backgroundToolResultDeletionPendingAt: new Date() } }, + { timestamps: false }, + ); + } + return { ensureAgentTriggerDeliveryIndexes, enqueueAgentTriggerDelivery, @@ -3578,6 +4007,11 @@ export function createAgentTriggerDeliveryMethods( retireAgentTriggerDelivery, renewAgentTriggerDeliveryProducerLease, getAgentTriggerDeliveryProducerLease, + persistAgentBackgroundToolResult, + getAgentBackgroundToolResult, + getAgentBackgroundToolResultClaim, + claimAgentBackgroundToolResults, + releaseAgentBackgroundToolResultClaims, settleAgentTriggerHandlingOutcome, admitAgentEventActorAction, releaseAgentEventActorAction, @@ -3606,5 +4040,7 @@ export function createAgentTriggerDeliveryMethods( cancelAgentTriggerUserPurge, recoverAgentTriggerUserPurges, deleteAgentTriggerDeliveriesByUser, + eraseAgentTriggerDeliveryConversationResults, + prepareAgentTriggerConversationResultErasure, }; } diff --git a/packages/data-schemas/src/schema/triggerDelivery.ts b/packages/data-schemas/src/schema/triggerDelivery.ts index 34cb661587..1de097809d 100644 --- a/packages/data-schemas/src/schema/triggerDelivery.ts +++ b/packages/data-schemas/src/schema/triggerDelivery.ts @@ -1,5 +1,6 @@ import { Schema } from 'mongoose'; import type { IAgentTriggerDeliveryDocument } from '~/types/triggerDelivery'; +import { AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS } from '~/types/triggerDelivery'; const failureSchema = new Schema( { @@ -147,6 +148,38 @@ const triggerDeliverySchema: Schema = new Schema( capabilityClaimToken: { type: String }, /** Private process-owner heartbeat; never projected to legacy consumers. */ producerLeaseUntil: { type: Date, select: false }, + backgroundToolResult: { + type: new Schema( + { + status: { + type: String, + enum: ['completed', 'error', 'cancelled'], + required: true, + }, + output: { + type: String, + required: true, + maxlength: AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS, + }, + settledAt: { type: Date, required: true }, + resultClaim: { + type: new Schema( + { + kind: { type: String, enum: ['wakeup'], required: true }, + claimId: { type: String, required: true, maxlength: 128 }, + claimedAt: { type: Date, required: true }, + }, + { _id: false }, + ), + required: false, + }, + }, + { _id: false }, + ), + select: false, + }, + backgroundToolResultErasedAt: { type: Date, select: false }, + backgroundToolResultDeletionPendingAt: { type: Date, select: false }, attempts: { type: Number, required: true, default: 0, min: 0 }, availableAt: { type: Date, required: true }, envelopeBytes: { type: Number, min: 0 }, @@ -224,6 +257,7 @@ triggerDeliverySchema.index({ 'actorReceipt.resolution': 1 }, { sparse: true }); triggerDeliverySchema.index({ user: 1, actorActionAdmittedAt: 1 }, { sparse: true }); triggerDeliverySchema.index({ stagingRecoveryAt: 1 }, { sparse: true }); triggerDeliverySchema.index({ laneCleanupPendingAt: 1 }, { sparse: true }); +triggerDeliverySchema.index({ backgroundToolResultDeletionPendingAt: 1 }, { sparse: true }); // Only successful rows receive expiresAt. Dead letters remain available until // an operator explicitly requeues or removes them. triggerDeliverySchema.index({ expiresAt: 1 }, { expireAfterSeconds: 0 }); diff --git a/packages/data-schemas/src/types/triggerDelivery.ts b/packages/data-schemas/src/types/triggerDelivery.ts index 7e91c341fc..be019d65a2 100644 --- a/packages/data-schemas/src/types/triggerDelivery.ts +++ b/packages/data-schemas/src/types/triggerDelivery.ts @@ -15,7 +15,10 @@ export type AgentTriggerDeliveryStatus = export const AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 = 'event_actor_detached_action_v1'; export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 = 'background_tool_completion_v1'; +export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 = + 'background_tool_completion_receipt_v2'; export const AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1 = 'agent_queued_turn_v1'; +export const AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS: number = 64 * 1024; export type AgentTriggerDeliveryOutcome = 'succeeded' | 'retry' | 'dead'; export interface AgentTriggerHandlingState { @@ -77,6 +80,20 @@ export interface AgentTriggerDeliveryFailure { status?: number; } +/** Private terminal result produced before a background-completion delivery is + * resolved. It is independent of the parent message projection because a fast + * task can settle before that response row exists. */ +export interface AgentBackgroundToolResultReceipt { + status: 'completed' | 'error' | 'cancelled'; + output: string; + settledAt: Date; + resultClaim?: { + kind: 'wakeup'; + claimId: string; + claimedAt: Date; + }; +} + export interface AgentTriggerDeliveryHistoryEntry { attempt: number; outcome: AgentTriggerDeliveryOutcome; @@ -108,6 +125,11 @@ export interface IAgentTriggerDelivery { capabilityClaimToken?: string; /** Durable liveness evidence for process-owned capability work. */ producerLeaseUntil?: Date; + /** Durable source of truth for a background completion. */ + backgroundToolResult?: AgentBackgroundToolResultReceipt; + /** Conversation-deletion fence preventing a late producer from restoring private output. */ + backgroundToolResultErasedAt?: Date; + backgroundToolResultDeletionPendingAt?: Date; attempts: number; availableAt: Date; envelopeBytes?: number;