mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-21 07:23:11 +00:00
📬 fix: Persist Background Results Before Parent Rows (#16101)
* 📬 fix: Persist Background Results Before Parent Rows
* fix: Harden durable background result delivery
* fix: Fence durable background completion ownership
* fix: reconcile durable background receipt ownership
* test: align background receipt mocks with contracts
* fix: Fence durable result ownership and recover interrupted cleanup
* fix: Mark independent receipt projections for task-local ownership
* fix: Carry receipt provenance through message patch inputs
* fix: Preserve resumed receipt delivery during mutual claim yielding
* fix: Preserve conversation cleanup for non-receipt owner identities
* fix: Preserve receipt recovery across cleanup and retry exhaustion
This commit is contained in:
parent
5d2fa85d1f
commit
8a3ab43965
32 changed files with 1863 additions and 87 deletions
|
|
@ -1355,7 +1355,10 @@ describe('OpenAIChatCompletionController', () => {
|
|||
const { loadAgentTools, loadToolsForExecution } = require('~/server/services/ToolService');
|
||||
const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions');
|
||||
|
||||
req.config.endpoints.agents.backgroundTasks = { ordinaryToolCancellation: true };
|
||||
req.config.endpoints.agents.backgroundTasks = {
|
||||
ordinaryToolCancellation: true,
|
||||
completionResultMaxChars: 4096,
|
||||
};
|
||||
await OpenAIChatCompletionController(req, res);
|
||||
|
||||
const [initializeParams, dbMethods] = initializeAgent.mock.calls.at(-1);
|
||||
|
|
@ -1384,6 +1387,7 @@ describe('OpenAIChatCompletionController', () => {
|
|||
|
||||
const toolExecuteOptions = createToolExecuteHandler.mock.calls.at(-1)[0];
|
||||
expect(toolExecuteOptions.ordinaryToolCancellation).toBe(true);
|
||||
expect(toolExecuteOptions.backgroundCompletionResultMaxChars).toBe(4096);
|
||||
expect(toolExecuteOptions.runSignal).toBe(mockExecution.signal);
|
||||
expect(toolExecuteOptions.foregroundRunId).toBe(initializeParams.requestBody.messageId);
|
||||
const effectiveSignal = new AbortController().signal;
|
||||
|
|
|
|||
|
|
@ -2065,7 +2065,10 @@ describe('createResponse controller', () => {
|
|||
const { loadAgentTools, loadToolsForExecution } = require('~/server/services/ToolService');
|
||||
const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions');
|
||||
|
||||
req.config.endpoints.agents.backgroundTasks = { ordinaryToolCancellation: true };
|
||||
req.config.endpoints.agents.backgroundTasks = {
|
||||
ordinaryToolCancellation: true,
|
||||
completionResultMaxChars: 4096,
|
||||
};
|
||||
req.body.stream = stream;
|
||||
await createResponse(req, res);
|
||||
|
||||
|
|
@ -2095,6 +2098,7 @@ describe('createResponse controller', () => {
|
|||
|
||||
const toolExecuteOptions = createToolExecuteHandler.mock.calls.at(-1)[0];
|
||||
expect(toolExecuteOptions.ordinaryToolCancellation).toBe(true);
|
||||
expect(toolExecuteOptions.backgroundCompletionResultMaxChars).toBe(4096);
|
||||
expect(toolExecuteOptions.runSignal).toBe(mockExecution.signal);
|
||||
expect(toolExecuteOptions.foregroundRunId).toBe(initializeParams.requestBody.messageId);
|
||||
const effectiveSignal = new AbortController().signal;
|
||||
|
|
|
|||
|
|
@ -490,6 +490,8 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
const allowedProviders = new Set(agentsEConfig?.allowedProviders);
|
||||
const ordinaryToolCancellationEnabled =
|
||||
agentsEConfig?.backgroundTasks?.ordinaryToolCancellation === true;
|
||||
const backgroundCompletionResultMaxChars =
|
||||
agentsEConfig?.backgroundTasks?.completionResultMaxChars;
|
||||
|
||||
// Create tool loader
|
||||
const loadTools = createToolLoader({ req, res, signal: execution.signal });
|
||||
|
|
@ -840,6 +842,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
runSignal: execution.signal,
|
||||
foregroundRunId: responseId,
|
||||
ordinaryToolCancellation: ordinaryToolCancellationEnabled,
|
||||
backgroundCompletionResultMaxChars,
|
||||
provisionFiles: createProvisionFilesCallback({
|
||||
req,
|
||||
agentToolContexts,
|
||||
|
|
|
|||
|
|
@ -732,6 +732,8 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
const agentsEConfig = appConfig?.endpoints?.[EModelEndpoint.agents];
|
||||
const ordinaryToolCancellationEnabled =
|
||||
agentsEConfig?.backgroundTasks?.ordinaryToolCancellation === true;
|
||||
const backgroundCompletionResultMaxChars =
|
||||
agentsEConfig?.backgroundTasks?.completionResultMaxChars;
|
||||
const previousMessages = request.previous_response_id
|
||||
? await loadPreviousMessages(request.previous_response_id, principal.userId)
|
||||
: [];
|
||||
|
|
@ -1194,6 +1196,7 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
runSignal: execution.signal,
|
||||
foregroundRunId: responseId,
|
||||
ordinaryToolCancellation: ordinaryToolCancellationEnabled,
|
||||
backgroundCompletionResultMaxChars,
|
||||
provisionFiles: createProvisionFilesCallback({
|
||||
req,
|
||||
agentToolContexts,
|
||||
|
|
@ -1424,6 +1427,7 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
runSignal: execution.signal,
|
||||
foregroundRunId: responseId,
|
||||
ordinaryToolCancellation: ordinaryToolCancellationEnabled,
|
||||
backgroundCompletionResultMaxChars,
|
||||
provisionFiles: createProvisionFilesCallback({
|
||||
req,
|
||||
agentToolContexts,
|
||||
|
|
|
|||
|
|
@ -86,6 +86,9 @@ module.exports = {
|
|||
requeueAgentTrigger: service.requeue,
|
||||
retireAgentTrigger: service.retire,
|
||||
renewAgentTriggerProducerLease: service.renewProducerLease,
|
||||
persistAgentBackgroundToolResult: service.persistBackgroundToolResult,
|
||||
getAgentBackgroundToolResultClaim: service.getBackgroundToolResultClaim,
|
||||
releaseAgentBackgroundToolResultClaims: service.releaseBackgroundToolResultClaims,
|
||||
drainAgentTriggerDeliveriesForUser: service.drainUser,
|
||||
prepareAgentTriggerUserPurge: service.prepareUserPurge,
|
||||
cancelAgentTriggerUserPurge: service.cancelUserPurge,
|
||||
|
|
|
|||
|
|
@ -2,9 +2,13 @@ const {
|
|||
createBackgroundToolCompletionWakeupHandler,
|
||||
createBackgroundToolDeadClaimRecovery,
|
||||
createBackgroundToolResultHandler,
|
||||
claimBackgroundToolResult: claimResult,
|
||||
} = require('@librechat/api');
|
||||
const {
|
||||
enqueueAgentTrigger,
|
||||
persistAgentBackgroundToolResult,
|
||||
getAgentBackgroundToolResultClaim,
|
||||
releaseAgentBackgroundToolResultClaims,
|
||||
renewAgentTriggerProducerLease,
|
||||
retireAgentTrigger,
|
||||
} = require('../../Agents/triggers');
|
||||
|
|
@ -13,12 +17,17 @@ const preregisterBackgroundToolCompletion = createBackgroundToolCompletionWakeup
|
|||
enqueueAgentTrigger,
|
||||
retireAgentTrigger,
|
||||
renewAgentTriggerProducerLease,
|
||||
(deliveryKey, sourceId, result) =>
|
||||
persistAgentBackgroundToolResult({ deliveryKey, sourceId, result }),
|
||||
);
|
||||
|
||||
function createBackgroundToolResultPersistence({ req, updateToolCallResult }) {
|
||||
return createBackgroundToolResultHandler({ req, updateToolCallResult });
|
||||
}
|
||||
|
||||
const claimBackgroundToolResult = (db, input) =>
|
||||
claimResult(db, getAgentBackgroundToolResultClaim, input);
|
||||
|
||||
function createDeadBackgroundToolClaimRecovery(
|
||||
releaseBackgroundToolResultClaims,
|
||||
getGenerationJob,
|
||||
|
|
@ -29,11 +38,13 @@ function createDeadBackgroundToolClaimRecovery(
|
|||
releaseBackgroundToolResultClaims,
|
||||
getGenerationJob,
|
||||
fenceGenerationClaim,
|
||||
releaseAgentBackgroundToolResultClaims,
|
||||
);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
preregisterBackgroundToolCompletion,
|
||||
createBackgroundToolResultPersistence,
|
||||
claimBackgroundToolResult,
|
||||
createDeadBackgroundToolClaimRecovery,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -95,6 +95,7 @@ const subagentThreadTaskStore = require('./subagentThreadStore');
|
|||
const {
|
||||
preregisterBackgroundToolCompletion,
|
||||
createBackgroundToolResultPersistence,
|
||||
claimBackgroundToolResult,
|
||||
createDeadBackgroundToolClaimRecovery,
|
||||
} = require('./backgroundCompletion');
|
||||
const { logViolation } = require('~/cache');
|
||||
|
|
@ -215,6 +216,8 @@ const initializeClientWithProvider = async ({
|
|||
const ordinaryToolCancellationEnabled =
|
||||
appConfig?.endpoints?.[EModelEndpoint.agents]?.backgroundTasks?.ordinaryToolCancellation ===
|
||||
true;
|
||||
const backgroundCompletionResultMaxChars =
|
||||
appConfig?.endpoints?.[EModelEndpoint.agents]?.backgroundTasks?.completionResultMaxChars;
|
||||
/** The normal controller resolves this once for timestamp anchoring. Reuse
|
||||
* that trusted document for child-thread execution policy; resume and direct
|
||||
* callers fall back to the same owner-scoped lookup. */
|
||||
|
|
@ -440,6 +443,7 @@ const initializeClientWithProvider = async ({
|
|||
runSignal: signal,
|
||||
foregroundRunId,
|
||||
ordinaryToolCancellation: ordinaryToolCancellationEnabled,
|
||||
backgroundCompletionResultMaxChars,
|
||||
loadTools: async (
|
||||
toolNames,
|
||||
agentId,
|
||||
|
|
@ -515,7 +519,7 @@ const initializeClientWithProvider = async ({
|
|||
req,
|
||||
updateToolCallResult: db.updateToolCallResult,
|
||||
}),
|
||||
claim: db.claimBackgroundToolResults,
|
||||
claim: (input) => claimBackgroundToolResult(db, input),
|
||||
recoverDeadClaim: createDeadBackgroundToolClaimRecovery(
|
||||
db.releaseBackgroundToolResultClaims,
|
||||
(conversationId) => GenerationJobManager.getJob(conversationId),
|
||||
|
|
|
|||
|
|
@ -408,10 +408,11 @@ describe('initializeClient — processAgent ACL gate', () => {
|
|||
endpointOption: makeEndpointOption(),
|
||||
});
|
||||
expect(capturedToolExecuteOptions.ordinaryToolCancellation).toBe(false);
|
||||
expect(capturedToolExecuteOptions.backgroundCompletionResultMaxChars).toBeUndefined();
|
||||
|
||||
const enabledReq = makeReq();
|
||||
enabledReq.config.endpoints.agents = {
|
||||
backgroundTasks: { ordinaryToolCancellation: true },
|
||||
backgroundTasks: { ordinaryToolCancellation: true, completionResultMaxChars: 4096 },
|
||||
};
|
||||
await initializeClient({
|
||||
req: enabledReq,
|
||||
|
|
@ -420,6 +421,7 @@ describe('initializeClient — processAgent ACL gate', () => {
|
|||
endpointOption: makeEndpointOption(),
|
||||
});
|
||||
expect(capturedToolExecuteOptions.ordinaryToolCancellation).toBe(true);
|
||||
expect(capturedToolExecuteOptions.backgroundCompletionResultMaxChars).toBe(4096);
|
||||
});
|
||||
|
||||
it('propagates an expected-MCP-tools failure from the runtime tool loader', async () => {
|
||||
|
|
|
|||
82
packages/api/src/agents/backgroundClaims.spec.ts
Normal file
82
packages/api/src/agents/backgroundClaims.spec.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
import type { AgentTriggerDeliveryMethods, MessageMethods } from '@librechat/data-schemas';
|
||||
import { claimBackgroundToolResult } from './backgroundClaims';
|
||||
|
||||
describe('manual background receipt arbitration', () => {
|
||||
const input = {
|
||||
userId: 'user',
|
||||
conversationId: 'conversation',
|
||||
taskId: 'task',
|
||||
kind: 'manual' as const,
|
||||
claimId: 'poll',
|
||||
};
|
||||
const receipt = { kind: 'wakeup' as const, claimId: 'delivery', claimedAt: new Date() };
|
||||
function fixture() {
|
||||
const methods = {
|
||||
claimBackgroundToolResults: jest
|
||||
.fn<
|
||||
ReturnType<MessageMethods['claimBackgroundToolResults']>,
|
||||
Parameters<MessageMethods['claimBackgroundToolResults']>
|
||||
>()
|
||||
.mockResolvedValue({
|
||||
status: 'acquired',
|
||||
messageId: 'recovered-parent',
|
||||
results: [
|
||||
{
|
||||
taskId: 'task',
|
||||
toolCallId: 'call',
|
||||
toolName: 'tool',
|
||||
status: 'completed',
|
||||
output: 'done',
|
||||
},
|
||||
],
|
||||
}),
|
||||
releaseBackgroundToolResultClaims: jest.fn(async () => true),
|
||||
};
|
||||
const lookup = jest
|
||||
.fn<
|
||||
ReturnType<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>,
|
||||
Parameters<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>
|
||||
>()
|
||||
.mockResolvedValue(null);
|
||||
return { methods, lookup };
|
||||
}
|
||||
it('uses the recovered parent identity when the process-local registry is gone', async () => {
|
||||
const { methods, lookup } = fixture();
|
||||
lookup.mockResolvedValue(receipt);
|
||||
await expect(claimBackgroundToolResult(methods, lookup, input)).resolves.toEqual({
|
||||
status: 'claimed',
|
||||
claim: receipt,
|
||||
messageId: 'recovered-parent',
|
||||
});
|
||||
expect(lookup).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ parentMessageId: 'recovered-parent' }),
|
||||
);
|
||||
expect(methods.releaseBackgroundToolResultClaims).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ messageId: 'recovered-parent', taskIds: ['task'] }),
|
||||
);
|
||||
});
|
||||
it('releases a speculative message claim if receipt lookup fails', async () => {
|
||||
const { methods, lookup } = fixture();
|
||||
lookup.mockRejectedValue(new Error('database unavailable'));
|
||||
await expect(claimBackgroundToolResult(methods, lookup, input)).rejects.toThrow(
|
||||
'database unavailable',
|
||||
);
|
||||
expect(methods.releaseBackgroundToolResultClaims).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
it('does not report arbitration as complete if rollback fails', async () => {
|
||||
const { methods, lookup } = fixture();
|
||||
lookup.mockResolvedValue(receipt);
|
||||
methods.releaseBackgroundToolResultClaims.mockResolvedValue(false);
|
||||
await expect(claimBackgroundToolResult(methods, lookup, input)).rejects.toThrow(
|
||||
'could not be released',
|
||||
);
|
||||
});
|
||||
it('does not claim the projection when its known receipt already has an owner', async () => {
|
||||
const { methods, lookup } = fixture();
|
||||
lookup.mockResolvedValue(receipt);
|
||||
await expect(
|
||||
claimBackgroundToolResult(methods, lookup, { ...input, messageId: 'parent' }),
|
||||
).resolves.toMatchObject({ status: 'claimed', messageId: 'parent' });
|
||||
expect(methods.claimBackgroundToolResults).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
58
packages/api/src/agents/backgroundClaims.ts
Normal file
58
packages/api/src/agents/backgroundClaims.ts
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
import type { AgentTriggerDeliveryMethods, MessageMethods } from '@librechat/data-schemas';
|
||||
import { BACKGROUND_TOOL_COMPLETION_SOURCE } from './backgroundCompletionWakeup';
|
||||
|
||||
/** Reconciles manual polling with the independent automatic-delivery receipt,
|
||||
* including polls reconstructed after the process-local registry was lost. */
|
||||
export async function claimBackgroundToolResult(
|
||||
methods: Pick<MessageMethods, 'claimBackgroundToolResults' | 'releaseBackgroundToolResultClaims'>,
|
||||
getReceiptClaim: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'],
|
||||
input: Parameters<MessageMethods['claimBackgroundToolResults']>[0],
|
||||
): ReturnType<MessageMethods['claimBackgroundToolResults']> {
|
||||
const lookup = (messageId: string) =>
|
||||
getReceiptClaim({
|
||||
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
userId: input.userId,
|
||||
conversationId: input.conversationId,
|
||||
parentMessageId: messageId,
|
||||
taskId: input.taskId,
|
||||
});
|
||||
if (input.messageId != null) {
|
||||
const receiptClaim = await lookup(input.messageId);
|
||||
if (receiptClaim != null && receiptClaim.claimId !== input.claimId) {
|
||||
return { status: 'claimed', claim: receiptClaim, messageId: input.messageId };
|
||||
}
|
||||
}
|
||||
const messageClaim = await methods.claimBackgroundToolResults(input);
|
||||
if (messageClaim.status !== 'acquired') {
|
||||
return messageClaim;
|
||||
}
|
||||
const messageId = input.messageId ?? messageClaim.messageId;
|
||||
if (messageId == null) {
|
||||
throw new Error('The background result claim has no parent message identity');
|
||||
}
|
||||
const release = async () => {
|
||||
const released = await methods.releaseBackgroundToolResultClaims({
|
||||
userId: input.userId,
|
||||
conversationId: input.conversationId,
|
||||
messageId,
|
||||
taskIds: messageClaim.results.map((result) => result.taskId),
|
||||
kind: input.kind,
|
||||
claimId: input.claimId,
|
||||
});
|
||||
if (!released) {
|
||||
throw new Error('The background result claim could not be released');
|
||||
}
|
||||
};
|
||||
let receiptClaim: Awaited<ReturnType<typeof getReceiptClaim>>;
|
||||
try {
|
||||
receiptClaim = await lookup(messageId);
|
||||
} catch (error) {
|
||||
await release();
|
||||
throw error;
|
||||
}
|
||||
if (receiptClaim == null || receiptClaim.claimId === input.claimId) {
|
||||
return messageClaim;
|
||||
}
|
||||
await release();
|
||||
return { status: 'claimed', claim: receiptClaim, messageId };
|
||||
}
|
||||
|
|
@ -31,6 +31,13 @@ export interface BackgroundToolWakeupRetireOptions {
|
|||
export interface BackgroundToolWakeupAdmission {
|
||||
/** Renews durable proof that the process-local executor still owns work. */
|
||||
renew: () => Promise<boolean>;
|
||||
/** Persists terminal output on the pre-admitted delivery before the parent
|
||||
* message projection exists. */
|
||||
persistResult?: (result: {
|
||||
status: 'completed' | 'error' | 'cancelled';
|
||||
output: string;
|
||||
settledAt: Date;
|
||||
}) => Promise<boolean>;
|
||||
/** Retires a delivery whose terminal result can no longer be made durable.
|
||||
* Manual polling requires an atomic unclaimed-only transition: once a
|
||||
* resolver owns a lease, its prepared continuation cannot be cancelled. */
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 } from '@librechat/data-schemas';
|
||||
import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 } from '@librechat/data-schemas';
|
||||
import type { AgentTriggerProducerLeaseStatus } from '@librechat/data-schemas';
|
||||
import type { EnqueueBackgroundToolCompletion } from './backgroundCompletionWakeup';
|
||||
import {
|
||||
|
|
@ -91,6 +91,10 @@ function resolverMethods() {
|
|||
leaseUntil: new Date(NOW + 30_000),
|
||||
}),
|
||||
),
|
||||
getAgentBackgroundToolResult: jest.fn(async () => null),
|
||||
getAgentBackgroundToolResultClaim: jest.fn(async () => null),
|
||||
claimAgentBackgroundToolResults: jest.fn(async () => ({ status: 'not_ready' as const })),
|
||||
releaseAgentBackgroundToolResultClaims: jest.fn(async () => true),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -111,7 +115,13 @@ describe('background tool completion wakeups', () => {
|
|||
>(async () => ({ deliveryKey: 'delivery-key-1' }));
|
||||
const retire = jest.fn(async () => true);
|
||||
const renew = jest.fn(async () => true);
|
||||
const notify = createBackgroundToolCompletionWakeupHandler(enqueue, retire, renew);
|
||||
const persistResult = jest.fn(async () => true);
|
||||
const notify = createBackgroundToolCompletionWakeupHandler(
|
||||
enqueue,
|
||||
retire,
|
||||
renew,
|
||||
persistResult,
|
||||
);
|
||||
|
||||
const admission = await notify(registration());
|
||||
expect(admission).not.toBe(false);
|
||||
|
|
@ -134,7 +144,7 @@ describe('background tool completion wakeups', () => {
|
|||
expect(options).toEqual({
|
||||
orderingKey: 'background-tool-completion:conversation-1:task-1',
|
||||
availableAt: new Date(NOW + 250),
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
producerLeaseUntil: new Date(NOW + 30_000),
|
||||
});
|
||||
if (admission !== false) {
|
||||
|
|
@ -145,6 +155,20 @@ describe('background tool completion wakeups', () => {
|
|||
'background-tool-completion',
|
||||
new Date(NOW + 30_000),
|
||||
);
|
||||
if (admission !== false) {
|
||||
await expect(
|
||||
admission.persistResult?.({
|
||||
status: 'completed',
|
||||
output: 'done',
|
||||
settledAt: new Date(NOW),
|
||||
}),
|
||||
).resolves.toBe(true);
|
||||
}
|
||||
expect(persistResult).toHaveBeenCalledWith('delivery-key-1', 'background-tool-completion', {
|
||||
status: 'completed',
|
||||
output: 'done',
|
||||
settledAt: new Date(NOW),
|
||||
});
|
||||
if (admission !== false) {
|
||||
await expect(admission.retire('result unavailable')).resolves.toBe(true);
|
||||
}
|
||||
|
|
@ -180,6 +204,7 @@ describe('background tool completion wakeups', () => {
|
|||
it('retires the batch-root delivery before releasing all of its sibling claims', async () => {
|
||||
const retire = jest.fn(async () => true);
|
||||
const release = jest.fn(async () => true);
|
||||
const releaseReceipts = jest.fn(async () => true);
|
||||
const getGenerationJob = jest.fn(async () => ({ status: 'complete' }));
|
||||
const fenceGenerationClaim = jest.fn(async () => 'fenced' as const);
|
||||
const recover = createBackgroundToolDeadClaimRecovery(
|
||||
|
|
@ -187,6 +212,7 @@ describe('background tool completion wakeups', () => {
|
|||
release,
|
||||
getGenerationJob,
|
||||
fenceGenerationClaim,
|
||||
releaseReceipts,
|
||||
);
|
||||
|
||||
await expect(
|
||||
|
|
@ -218,6 +244,13 @@ describe('background tool completion wakeups', () => {
|
|||
kind: 'wakeup',
|
||||
claimId: 'batch-root-delivery',
|
||||
});
|
||||
expect(releaseReceipts).toHaveBeenCalledWith({
|
||||
sourceId: 'background-tool-completion',
|
||||
userId: 'user-1',
|
||||
conversationId: 'conversation-1',
|
||||
parentMessageId: 'response-1',
|
||||
claimId: 'batch-root-delivery',
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps a manual result claim while its owning generation is active', async () => {
|
||||
|
|
@ -428,6 +461,120 @@ describe('background tool completion wakeups', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('continues from the independent delivery receipt before the parent projection lands', async () => {
|
||||
const { methods } = resolverMethods();
|
||||
methods.claimBackgroundToolResults.mockResolvedValueOnce({ status: 'not_ready', results: [] });
|
||||
methods.claimAgentBackgroundToolResults.mockResolvedValueOnce({
|
||||
status: 'acquired',
|
||||
results: [
|
||||
{
|
||||
taskId: 'task-1',
|
||||
toolCallId: 'call-1',
|
||||
toolName: 'slow_tool',
|
||||
status: 'completed',
|
||||
output: 'independently durable',
|
||||
},
|
||||
],
|
||||
} as never);
|
||||
const resolve = createBackgroundToolCompletionWakeupResolver({
|
||||
methods: methods as never,
|
||||
getGenerationJob: async () => null,
|
||||
});
|
||||
|
||||
const prepared = await resolve(await envelope(), { idempotencyKey: 'delivery-1' });
|
||||
|
||||
expect(prepared).toMatchObject({ status: 'ready', parentMessageId: 'response-2' });
|
||||
expect(prepared?.status === 'ready' && prepared.input).toContain('independently durable');
|
||||
expect(methods.claimAgentBackgroundToolResults).toHaveBeenCalledWith({
|
||||
deliveryKey: 'delivery-1',
|
||||
sourceId: 'background-tool-completion',
|
||||
userId: 'user-1',
|
||||
conversationId: 'conversation-1',
|
||||
parentMessageId: 'response-1',
|
||||
agentId: 'agent_parent_1',
|
||||
claimId: 'delivery-1',
|
||||
limit: 1,
|
||||
});
|
||||
expect(methods.claimBackgroundToolResults).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ taskId: 'task-1', kind: 'wakeup', claimId: 'delivery-1' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('honors a manual message claim before an independent receipt', async () => {
|
||||
const { methods } = resolverMethods();
|
||||
methods.claimBackgroundToolResults.mockResolvedValueOnce({ status: 'claimed', results: [] });
|
||||
const resolve = createBackgroundToolCompletionWakeupResolver({
|
||||
methods: methods as never,
|
||||
getGenerationJob: async () => null,
|
||||
});
|
||||
|
||||
await expect(resolve(await envelope(), { idempotencyKey: 'delivery-1' })).resolves.toEqual({
|
||||
status: 'settled',
|
||||
});
|
||||
expect(methods.claimAgentBackgroundToolResults).not.toHaveBeenCalled();
|
||||
expect(methods.getAgentBackgroundToolResult).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not settle a resumed receipt owner while a speculative manual message claim yields', async () => {
|
||||
const { methods } = resolverMethods();
|
||||
methods.claimBackgroundToolResults.mockResolvedValueOnce({ status: 'claimed', results: [] });
|
||||
methods.getAgentBackgroundToolResultClaim.mockResolvedValueOnce({
|
||||
kind: 'wakeup',
|
||||
claimId: 'delivery-1',
|
||||
claimedAt: new Date(NOW),
|
||||
} as never);
|
||||
const resolve = createBackgroundToolCompletionWakeupResolver({
|
||||
methods: methods as never,
|
||||
getGenerationJob: async () => null,
|
||||
});
|
||||
await expect(resolve(await envelope(), { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(
|
||||
{
|
||||
code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
|
||||
deferWithoutAttempt: true,
|
||||
},
|
||||
);
|
||||
expect(methods.releaseAgentBackgroundToolResultClaims).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ claimId: 'delivery-1' }),
|
||||
);
|
||||
expect(methods.releaseBackgroundToolResultClaims).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('retries after yielding to a racing manual claim so mutual yielding cannot lose output', async () => {
|
||||
const { methods } = resolverMethods();
|
||||
methods.claimBackgroundToolResults
|
||||
.mockResolvedValueOnce({ status: 'not_ready' } as never)
|
||||
.mockResolvedValueOnce({
|
||||
status: 'claimed',
|
||||
claim: { kind: 'manual', claimId: 'poll-1' },
|
||||
} as never);
|
||||
methods.claimAgentBackgroundToolResults.mockResolvedValueOnce({
|
||||
status: 'acquired',
|
||||
results: [
|
||||
{
|
||||
taskId: 'task-1',
|
||||
toolCallId: 'call-1',
|
||||
toolName: 'slow_tool',
|
||||
status: 'completed',
|
||||
output: 'durable',
|
||||
},
|
||||
],
|
||||
} as never);
|
||||
const resolve = createBackgroundToolCompletionWakeupResolver({
|
||||
methods: methods as never,
|
||||
getGenerationJob: async () => null,
|
||||
});
|
||||
|
||||
await expect(resolve(await envelope(), { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(
|
||||
{
|
||||
code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
|
||||
deferWithoutAttempt: true,
|
||||
},
|
||||
);
|
||||
expect(methods.releaseAgentBackgroundToolResultClaims).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ claimId: 'delivery-1', parentMessageId: 'response-1' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('shares one bounded input budget across a full sibling batch', async () => {
|
||||
const { methods } = resolverMethods();
|
||||
const results = Array.from({ length: 8 }, (_, index) => ({
|
||||
|
|
@ -468,7 +615,10 @@ describe('background tool completion wakeups', () => {
|
|||
await prepared.releaseOnDefiniteFailure?.();
|
||||
}
|
||||
expect(releaseBackgroundToolResultClaims).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ taskIds: ['task-1', 'task-2'], claimId: 'delivery-1' }),
|
||||
expect.objectContaining({ claimId: 'delivery-1', kind: 'wakeup' }),
|
||||
);
|
||||
expect(releaseBackgroundToolResultClaims).toHaveBeenCalledWith(
|
||||
expect.not.objectContaining({ taskIds: expect.anything() }),
|
||||
);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
import { randomUUID } from 'node:crypto';
|
||||
import { isEphemeralAgentId } from 'librechat-data-provider';
|
||||
import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 } from '@librechat/data-schemas';
|
||||
import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 } from '@librechat/data-schemas';
|
||||
import type {
|
||||
AgentTriggerProducerLeaseStatus,
|
||||
AgentTriggerDeliveryMethods,
|
||||
BackgroundToolResultClaim,
|
||||
ConversationMethods,
|
||||
IMessage,
|
||||
|
|
@ -51,6 +52,16 @@ export type RenewBackgroundToolCompletionProducerLease = (
|
|||
leaseUntil: Date,
|
||||
) => Promise<boolean>;
|
||||
|
||||
export type PersistBackgroundToolCompletionResult = (
|
||||
deliveryKey: string,
|
||||
sourceId: string,
|
||||
result: {
|
||||
status: 'completed' | 'error' | 'cancelled';
|
||||
output: string;
|
||||
settledAt: Date;
|
||||
},
|
||||
) => Promise<boolean>;
|
||||
|
||||
type WakeupMethods = Pick<ConversationMethods, 'getConvo'> &
|
||||
Pick<MessageMethods, 'getMessages'> & {
|
||||
claimBackgroundToolResults(params: {
|
||||
|
|
@ -76,6 +87,14 @@ type WakeupMethods = Pick<ConversationMethods, 'getConvo'> &
|
|||
sourceId: string;
|
||||
now: Date;
|
||||
}): Promise<AgentTriggerProducerLeaseStatus>;
|
||||
getAgentBackgroundToolResult?(params: { deliveryKey: string; sourceId: string }): Promise<{
|
||||
status: 'completed' | 'error' | 'cancelled';
|
||||
output: string;
|
||||
settledAt: Date;
|
||||
} | null>;
|
||||
claimAgentBackgroundToolResults?: AgentTriggerDeliveryMethods['claimAgentBackgroundToolResults'];
|
||||
getAgentBackgroundToolResultClaim?: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'];
|
||||
releaseAgentBackgroundToolResultClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];
|
||||
};
|
||||
|
||||
interface GenerationState {
|
||||
|
|
@ -326,8 +345,33 @@ export function createBackgroundToolCompletionWakeupResolver({
|
|||
agentId: envelope.target.agentId,
|
||||
kind: 'wakeup',
|
||||
claimId: context.idempotencyKey,
|
||||
limit: 1,
|
||||
});
|
||||
if (claim.status === 'claimed') {
|
||||
const receiptOwner = await methods.getAgentBackgroundToolResultClaim?.({
|
||||
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
userId,
|
||||
conversationId: envelope.target.conversationId,
|
||||
parentMessageId: envelope.target.parentMessageId,
|
||||
taskId: registration.taskId,
|
||||
});
|
||||
if (receiptOwner?.claimId === context.idempotencyKey) {
|
||||
const released = await methods.releaseAgentBackgroundToolResultClaims?.({
|
||||
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
userId,
|
||||
conversationId: envelope.target.conversationId,
|
||||
parentMessageId: envelope.target.parentMessageId,
|
||||
claimId: context.idempotencyKey,
|
||||
});
|
||||
if (released === false)
|
||||
throw new Error('Background receipt claim release was not confirmed');
|
||||
throw executionError('Background result ownership is being reconciled.', {
|
||||
code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
|
||||
retryable: true,
|
||||
deferWithoutAttempt: true,
|
||||
retryAfter: '1',
|
||||
});
|
||||
}
|
||||
return { status: 'settled' };
|
||||
}
|
||||
if (claim.status === 'outcome_unknown') {
|
||||
|
|
@ -336,47 +380,17 @@ export function createBackgroundToolCompletionWakeupResolver({
|
|||
retryable: false,
|
||||
});
|
||||
}
|
||||
if (claim.status !== 'acquired') {
|
||||
let producerLease: AgentTriggerProducerLeaseStatus;
|
||||
try {
|
||||
producerLease = await methods.getAgentTriggerDeliveryProducerLease({
|
||||
deliveryKey: context.idempotencyKey,
|
||||
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
now: new Date(),
|
||||
});
|
||||
} catch (error) {
|
||||
throw executionError(
|
||||
`Background tool producer liveness is temporarily unavailable: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
{ code: 'BACKGROUND_TOOL_PRODUCER_STATE_UNAVAILABLE', retryable: true },
|
||||
);
|
||||
}
|
||||
if (producerLease.status === 'expired') {
|
||||
throw executionError('The process-local background tool executor was lost.', {
|
||||
code: 'BACKGROUND_TOOL_PRODUCER_LOST',
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
/** A live lease proves the invocation or its durable persistence retry
|
||||
* still has an owner. Missing remains defer-only for compatibility with
|
||||
* completion rows admitted before producer leases existed. */
|
||||
throw executionError('The background tool result is not durable yet.', {
|
||||
code: 'BACKGROUND_TOOL_RESULT_NOT_READY',
|
||||
retryable: true,
|
||||
status: 409,
|
||||
retryAfter: '1',
|
||||
deferWithoutAttempt: true,
|
||||
if (claim.status === 'acquired') {
|
||||
const taskIds = claim.results.map((result) => result.taskId);
|
||||
const receiptOwner = await methods.getAgentBackgroundToolResultClaim?.({
|
||||
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
userId,
|
||||
conversationId: envelope.target.conversationId,
|
||||
parentMessageId: envelope.target.parentMessageId,
|
||||
taskId: registration.taskId,
|
||||
});
|
||||
}
|
||||
const taskIds = claim.results.map((result) => result.taskId);
|
||||
const input = buildWakeupInput(claim.results);
|
||||
return {
|
||||
status: 'ready',
|
||||
parentMessageId,
|
||||
input,
|
||||
releaseOnDefiniteFailure: async () => {
|
||||
await methods.releaseBackgroundToolResultClaims({
|
||||
if (receiptOwner != null && receiptOwner.claimId !== context.idempotencyKey) {
|
||||
const released = await methods.releaseBackgroundToolResultClaims({
|
||||
userId,
|
||||
conversationId: envelope.target.conversationId,
|
||||
messageId: envelope.target.parentMessageId,
|
||||
|
|
@ -384,8 +398,153 @@ export function createBackgroundToolCompletionWakeupResolver({
|
|||
kind: 'wakeup',
|
||||
claimId: context.idempotencyKey,
|
||||
});
|
||||
},
|
||||
};
|
||||
if (!released) throw new Error('Background projection claim release was not confirmed');
|
||||
return { status: 'settled' };
|
||||
}
|
||||
const input = buildWakeupInput(claim.results);
|
||||
return {
|
||||
status: 'ready',
|
||||
parentMessageId,
|
||||
input,
|
||||
releaseOnDefiniteFailure: async () => {
|
||||
const released = await methods.releaseBackgroundToolResultClaims({
|
||||
userId,
|
||||
conversationId: envelope.target.conversationId,
|
||||
messageId: envelope.target.parentMessageId,
|
||||
kind: 'wakeup',
|
||||
claimId: context.idempotencyKey,
|
||||
});
|
||||
if (!released) throw new Error('Background projection claim release was not confirmed');
|
||||
const receiptReleased = await methods.releaseAgentBackgroundToolResultClaims?.({
|
||||
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
userId,
|
||||
conversationId: envelope.target.conversationId,
|
||||
parentMessageId: envelope.target.parentMessageId,
|
||||
claimId: context.idempotencyKey,
|
||||
});
|
||||
if (receiptReleased === false)
|
||||
throw new Error('Background receipt claim release was not confirmed');
|
||||
},
|
||||
};
|
||||
}
|
||||
const receiptClaim = await methods.claimAgentBackgroundToolResults?.({
|
||||
deliveryKey: context.idempotencyKey,
|
||||
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
userId,
|
||||
conversationId: envelope.target.conversationId,
|
||||
parentMessageId: envelope.target.parentMessageId,
|
||||
agentId: envelope.target.agentId,
|
||||
claimId: context.idempotencyKey,
|
||||
limit: 1,
|
||||
});
|
||||
if (receiptClaim?.status === 'claimed') {
|
||||
return { status: 'settled' };
|
||||
}
|
||||
if (receiptClaim?.status === 'acquired') {
|
||||
/** Reconcile with a parent projection that may have appeared while the
|
||||
* receipt CAS was in flight. A manual poll that already owns the message
|
||||
* wins; otherwise stamp this same automatic owner before dispatch. */
|
||||
const projectedClaim = await methods.claimBackgroundToolResults({
|
||||
userId,
|
||||
conversationId: envelope.target.conversationId,
|
||||
messageId: envelope.target.parentMessageId,
|
||||
taskId: registration.taskId,
|
||||
agentId: envelope.target.agentId,
|
||||
kind: 'wakeup',
|
||||
claimId: context.idempotencyKey,
|
||||
limit: 1,
|
||||
});
|
||||
if (projectedClaim.status === 'claimed') {
|
||||
const released = await methods.releaseAgentBackgroundToolResultClaims?.({
|
||||
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
userId,
|
||||
conversationId: envelope.target.conversationId,
|
||||
parentMessageId: envelope.target.parentMessageId,
|
||||
claimId: context.idempotencyKey,
|
||||
});
|
||||
if (released === false)
|
||||
throw new Error('Background receipt claim release was not confirmed');
|
||||
throw executionError('Background result ownership is being reconciled.', {
|
||||
code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
|
||||
retryable: true,
|
||||
deferWithoutAttempt: true,
|
||||
retryAfter: '1',
|
||||
});
|
||||
}
|
||||
return {
|
||||
status: 'ready',
|
||||
parentMessageId,
|
||||
input: buildWakeupInput(receiptClaim.results),
|
||||
releaseOnDefiniteFailure: async () => {
|
||||
const projectionReleased = await methods.releaseBackgroundToolResultClaims({
|
||||
userId,
|
||||
conversationId: envelope.target.conversationId,
|
||||
messageId: envelope.target.parentMessageId,
|
||||
kind: 'wakeup',
|
||||
claimId: context.idempotencyKey,
|
||||
});
|
||||
if (projectedClaim.status === 'acquired' && !projectionReleased) {
|
||||
throw new Error('Background projection claim release was not confirmed');
|
||||
}
|
||||
const receiptReleased = await methods.releaseAgentBackgroundToolResultClaims?.({
|
||||
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
userId,
|
||||
conversationId: envelope.target.conversationId,
|
||||
parentMessageId: envelope.target.parentMessageId,
|
||||
claimId: context.idempotencyKey,
|
||||
});
|
||||
if (receiptReleased === false)
|
||||
throw new Error('Background receipt claim release was not confirmed');
|
||||
},
|
||||
};
|
||||
}
|
||||
const receipt =
|
||||
methods.claimAgentBackgroundToolResults == null
|
||||
? await methods.getAgentBackgroundToolResult?.({
|
||||
deliveryKey: context.idempotencyKey,
|
||||
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
})
|
||||
: null;
|
||||
if (receipt != null) {
|
||||
return {
|
||||
status: 'ready',
|
||||
parentMessageId,
|
||||
input: buildWakeupInput([
|
||||
{ ...registration, status: receipt.status, output: receipt.output },
|
||||
]),
|
||||
};
|
||||
}
|
||||
let producerLease: AgentTriggerProducerLeaseStatus;
|
||||
try {
|
||||
producerLease = await methods.getAgentTriggerDeliveryProducerLease({
|
||||
deliveryKey: context.idempotencyKey,
|
||||
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
now: new Date(),
|
||||
});
|
||||
} catch (error) {
|
||||
throw executionError(
|
||||
`Background tool producer liveness is temporarily unavailable: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
{ code: 'BACKGROUND_TOOL_PRODUCER_STATE_UNAVAILABLE', retryable: true },
|
||||
);
|
||||
}
|
||||
if (producerLease.status === 'expired') {
|
||||
throw executionError('The process-local background tool executor was lost.', {
|
||||
code: 'BACKGROUND_TOOL_PRODUCER_LOST',
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
/** A live lease proves the invocation or its durable persistence retry
|
||||
* still has an owner. Missing remains defer-only for compatibility with
|
||||
* completion rows admitted before producer leases existed. */
|
||||
throw executionError('The background tool result is not durable yet.', {
|
||||
code: 'BACKGROUND_TOOL_RESULT_NOT_READY',
|
||||
retryable: true,
|
||||
status: 409,
|
||||
retryAfter: '1',
|
||||
deferWithoutAttempt: true,
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -394,6 +553,7 @@ export function createBackgroundToolCompletionWakeupHandler(
|
|||
enqueue: EnqueueBackgroundToolCompletion,
|
||||
retire: RetireBackgroundToolCompletion,
|
||||
renewProducerLease: RenewBackgroundToolCompletionProducerLease,
|
||||
persistResult?: PersistBackgroundToolCompletionResult,
|
||||
): (
|
||||
registration: BackgroundToolWakeupRegistration,
|
||||
) => Promise<BackgroundToolWakeupAdmission | false> {
|
||||
|
|
@ -437,7 +597,7 @@ export function createBackgroundToolCompletionWakeupHandler(
|
|||
availableAt: new Date(
|
||||
Math.max(Date.now(), registration.createdAt) + WAKEUP_ADMISSION_DELAY_MS,
|
||||
),
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
producerLeaseUntil: new Date(Date.now() + BACKGROUND_TOOL_PRODUCER_LEASE_MS),
|
||||
});
|
||||
return {
|
||||
|
|
@ -447,6 +607,12 @@ export function createBackgroundToolCompletionWakeupHandler(
|
|||
BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
new Date(Date.now() + BACKGROUND_TOOL_PRODUCER_LEASE_MS),
|
||||
),
|
||||
...(persistResult == null
|
||||
? {}
|
||||
: {
|
||||
persistResult: (result) =>
|
||||
persistResult(admitted.deliveryKey, BACKGROUND_TOOL_COMPLETION_SOURCE, result),
|
||||
}),
|
||||
retire: (reason, options) =>
|
||||
options == null
|
||||
? retire(admitted.deliveryKey, BACKGROUND_TOOL_COMPLETION_SOURCE, reason)
|
||||
|
|
@ -469,6 +635,7 @@ export function createBackgroundToolDeadClaimRecovery(
|
|||
conversationId: string;
|
||||
claimId: string;
|
||||
}) => Promise<'fenced' | 'started' | 'unavailable'>,
|
||||
releaseReceiptClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'],
|
||||
): BackgroundToolDeadClaimRecovery {
|
||||
return async ({ userId, conversationId, messageId, claimId, kind, generationId }) => {
|
||||
if (kind === 'manual') {
|
||||
|
|
@ -518,12 +685,24 @@ export function createBackgroundToolDeadClaimRecovery(
|
|||
if (await claimGenerationIsActive()) {
|
||||
return false;
|
||||
}
|
||||
return releaseClaims({
|
||||
const released = await releaseClaims({
|
||||
userId,
|
||||
conversationId,
|
||||
messageId,
|
||||
kind: 'wakeup',
|
||||
claimId,
|
||||
});
|
||||
if (!released) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
(await releaseReceiptClaims?.({
|
||||
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
userId,
|
||||
conversationId,
|
||||
parentMessageId: messageId,
|
||||
claimId,
|
||||
})) ?? true
|
||||
);
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { logger } from '@librechat/data-schemas';
|
|||
import { tool as structuredTool } from '@librechat/agents/langchain/tools';
|
||||
import type { StructuredToolInterface } from '@librechat/agents/langchain/tools';
|
||||
import type { FiltersConfig } from 'librechat-data-provider';
|
||||
import type { BackgroundToolWakeupAdmission } from './backgroundCompletion';
|
||||
import type { ToolExecuteOptions } from './handlers';
|
||||
import {
|
||||
BACKGROUND_TASK_ABORT_GRACE_MS,
|
||||
|
|
@ -147,9 +148,13 @@ describe('createToolExecuteHandler — background tool calls', () => {
|
|||
it('pre-registers an ordinary completion before invoke and persists its terminal receipt', async () => {
|
||||
const events: string[] = [];
|
||||
const retire = jest.fn(async () => true);
|
||||
const persistResult = jest.fn(async () => {
|
||||
events.push('receipt');
|
||||
return true;
|
||||
});
|
||||
const preregister = jest.fn(async () => {
|
||||
events.push('preregister');
|
||||
return { renew: jest.fn(async () => true), retire };
|
||||
return { renew: jest.fn(async () => true), persistResult, retire };
|
||||
});
|
||||
const persist = jest.fn(async () => {
|
||||
events.push('persist');
|
||||
|
|
@ -198,7 +203,7 @@ describe('createToolExecuteHandler — background tool calls', () => {
|
|||
);
|
||||
expect(JSON.parse(dispatch.content).message).toContain('host will resume you');
|
||||
await flushMicrotasks();
|
||||
expect(events).toEqual(['preregister', 'invoke', 'persist']);
|
||||
expect(events).toEqual(['preregister', 'invoke', 'receipt', 'persist']);
|
||||
expect(preregister).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
toolCallId: 'call-wakeup',
|
||||
|
|
@ -217,6 +222,9 @@ describe('createToolExecuteHandler — background tool calls', () => {
|
|||
}),
|
||||
}),
|
||||
);
|
||||
expect(persistResult).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ status: 'completed', output: 'durable result' }),
|
||||
);
|
||||
expect(retire).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
|
|
@ -264,6 +272,55 @@ describe('createToolExecuteHandler — background tool calls', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it.each(['acknowledged', 'ambiguous'])(
|
||||
'keeps delivery live for an %s receipt without a projection',
|
||||
async (ack) => {
|
||||
const retire = jest.fn(async () => true);
|
||||
const persistResult = jest.fn(async () => {
|
||||
if (ack === 'ambiguous') throw new Error('receipt committed but acknowledgement lost');
|
||||
return true;
|
||||
});
|
||||
const tool = {
|
||||
name: 'search_mcp_docs',
|
||||
description: 'search docs',
|
||||
schema: z.object({ q: z.string() }),
|
||||
invoke: jest.fn(async () => ({ content: 'available before parent row' })),
|
||||
} as unknown as StructuredToolInterface;
|
||||
const handler = createToolExecuteHandler({
|
||||
loadTools: async () => ({ loadedTools: [tool] }),
|
||||
backgroundToolCompletion: {
|
||||
preregister: jest.fn(async () => ({
|
||||
renew: jest.fn(async () => true),
|
||||
persistResult,
|
||||
retire,
|
||||
})),
|
||||
persist: jest.fn(async () => false),
|
||||
claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
|
||||
},
|
||||
});
|
||||
|
||||
await runBatch(handler, {
|
||||
toolCalls: [
|
||||
{
|
||||
id: `call-independent-receipt-${ack}`,
|
||||
name: tool.name,
|
||||
args: { q: 'receipt', run_in_background: true },
|
||||
stepId: 'step-independent-receipt',
|
||||
},
|
||||
],
|
||||
agentId: 'agent_parent_1',
|
||||
configurable: buildConfig([tool.name]),
|
||||
metadata: { thread_id: 'exec_convo', run_id: `response-independent-receipt-${ack}` },
|
||||
});
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(persistResult).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ status: 'completed', output: 'available before parent row' }),
|
||||
);
|
||||
expect(retire).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it('keeps polling guidance when the completion adapter skips registration', async () => {
|
||||
const tool = makeSearchTool({ calls: 0 });
|
||||
const preregister = jest.fn(async () => false as const);
|
||||
|
|
@ -2570,6 +2627,84 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
|
|||
expect(poll[0].artifact).toEqual(CODE_ARTIFACT);
|
||||
});
|
||||
|
||||
it('does not publish an output-only receipt when generated files miss their message anchor', async () => {
|
||||
const state: CodeToolState = { calls: 0 };
|
||||
const retire = jest.fn(async () => true);
|
||||
const persistResult = jest.fn(async () => true);
|
||||
const handler = createToolExecuteHandler({
|
||||
loadTools: async () => ({ loadedTools: [makeCodeTool(state)] }),
|
||||
persistBackgroundCodeResult: async () => ({
|
||||
deliveryReady: false,
|
||||
attachments: [{ filename: 'plot.png' }],
|
||||
}),
|
||||
backgroundToolCompletion: {
|
||||
preregister: jest.fn(async () => ({
|
||||
renew: jest.fn(async () => true),
|
||||
persistResult,
|
||||
retire,
|
||||
})),
|
||||
persist: jest.fn(async () => true),
|
||||
claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
|
||||
},
|
||||
});
|
||||
|
||||
await runBatch(handler, {
|
||||
toolCalls: [codeCall({ id: 'call_code_missing_anchor' })],
|
||||
agentId: 'a',
|
||||
configurable: buildConfig(['execute_code']),
|
||||
metadata: { thread_id: 'exec_convo_code_missing_anchor', run_id: 'msg-missing-anchor' },
|
||||
});
|
||||
await flushMicrotasks();
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(persistResult).not.toHaveBeenCalled();
|
||||
expect(retire).toHaveBeenCalledWith('background code result was not persisted', undefined);
|
||||
});
|
||||
|
||||
it('uses the configured durable completion result size', async () => {
|
||||
const persistResult: jest.MockedFunction<
|
||||
NonNullable<BackgroundToolWakeupAdmission['persistResult']>
|
||||
> = jest.fn(
|
||||
async (_result: Parameters<NonNullable<BackgroundToolWakeupAdmission['persistResult']>>[0]) =>
|
||||
true,
|
||||
);
|
||||
const tool = {
|
||||
name: 'execute_code',
|
||||
description: 'run code',
|
||||
schema: z.object({ lang: z.string(), code: z.string() }),
|
||||
invoke: jest.fn(async () => ({ content: '0123456789'.repeat(10), artifact: CODE_ARTIFACT })),
|
||||
} as unknown as StructuredToolInterface;
|
||||
const handler = createToolExecuteHandler({
|
||||
loadTools: async () => ({ loadedTools: [tool] }),
|
||||
backgroundCompletionResultMaxChars: 32,
|
||||
persistBackgroundCodeResult: async () => ({ deliveryReady: true, attachments: [] }),
|
||||
backgroundToolCompletion: {
|
||||
preregister: jest.fn(async () => ({
|
||||
renew: jest.fn(async () => true),
|
||||
persistResult,
|
||||
retire: jest.fn(async () => true),
|
||||
})),
|
||||
persist: jest.fn(async () => true),
|
||||
claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
|
||||
},
|
||||
});
|
||||
|
||||
await runBatch(handler, {
|
||||
toolCalls: [codeCall({ id: 'call-sized-receipt' })],
|
||||
agentId: 'agent_parent_1',
|
||||
configurable: buildConfig([tool.name]),
|
||||
metadata: { thread_id: 'exec_convo', run_id: 'response-sized-receipt' },
|
||||
});
|
||||
await flushMicrotasks();
|
||||
await flushMicrotasks();
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(persistResult).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ status: 'completed', output: expect.any(String) }),
|
||||
);
|
||||
expect(persistResult.mock.calls[0][0].output).toHaveLength(32);
|
||||
});
|
||||
|
||||
it('makes a completion-time generated-file policy rejection terminal across polls', async () => {
|
||||
const protectedValue = 'PROTECTED-GENERATED-FILE-BYTES';
|
||||
const blockedArtifact = {
|
||||
|
|
@ -2596,12 +2731,17 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
|
|||
});
|
||||
const toolEndCallback = jest.fn();
|
||||
const retire = jest.fn(async () => true);
|
||||
const persistResult = jest.fn(async () => true);
|
||||
const handler = createToolExecuteHandler({
|
||||
loadTools: async () => ({ loadedTools: [codeTool] }),
|
||||
toolEndCallback,
|
||||
persistBackgroundCodeResult,
|
||||
backgroundToolCompletion: {
|
||||
preregister: jest.fn(async () => ({ renew: jest.fn(async () => true), retire })),
|
||||
preregister: jest.fn(async () => ({
|
||||
renew: jest.fn(async () => true),
|
||||
retire,
|
||||
persistResult,
|
||||
})),
|
||||
persist: jest.fn(async () => true),
|
||||
claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
|
||||
},
|
||||
|
|
@ -2621,7 +2761,10 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
|
|||
const taskId = JSON.parse(dispatch[0].content).background_task_id;
|
||||
await flushMicrotasks();
|
||||
await flushMicrotasks();
|
||||
expect(retire).toHaveBeenCalledWith('background code result persistence failed', undefined);
|
||||
expect(persistResult).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ status: 'error', output: MODEL_BOUND_FILE_CONTENT_BLOCK }),
|
||||
);
|
||||
expect(retire).not.toHaveBeenCalled();
|
||||
await flushMicrotasks();
|
||||
|
||||
for (const pollId of ['call_policy_poll_1', 'call_policy_poll_2']) {
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { Types } from 'mongoose';
|
|||
import { GraphEvents, Constants, ToolEndHandler } from '@librechat/agents';
|
||||
import { logger, normalizeSkillFrontmatterKeys } from '@librechat/data-schemas';
|
||||
import {
|
||||
AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT,
|
||||
hasActivePiiFields,
|
||||
hasActivePiiPatterns,
|
||||
hasToolCallErrorPrefix,
|
||||
|
|
@ -271,6 +272,8 @@ export interface ToolExecuteOptions {
|
|||
runFiles?: Pick<RunFileSession, 'isActive' | 'prepareTools' | 'withCodeExecution'>;
|
||||
/** Trusted deployment gate for cooperative ordinary-tool cancellation. */
|
||||
ordinaryToolCancellation?: boolean;
|
||||
/** Deployment-configured cap for terminal output copied into durable receipts. */
|
||||
backgroundCompletionResultMaxChars?: number;
|
||||
/** Callback to process tool artifacts (code output files, file citations, etc.) */
|
||||
toolEndCallback?: ToolEndCallback;
|
||||
/** Durable internal-completion adapter, present only for an Event Actor invocation. */
|
||||
|
|
@ -5317,6 +5320,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
|
|||
subagentTasks,
|
||||
runFiles,
|
||||
ordinaryToolCancellation = false,
|
||||
backgroundCompletionResultMaxChars = AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT,
|
||||
provisionFiles,
|
||||
} = options;
|
||||
|
||||
|
|
@ -5774,6 +5778,9 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
|
|||
...(params.status === 'cancelled' ? { cancelled: true } : {}),
|
||||
settledAt: new Date(current?.updatedAt ?? Date.now()),
|
||||
...(completionPreregistered ? { completionWakeup: true } : {}),
|
||||
...(completionAdmission?.persistResult != null
|
||||
? { completionReceipt: true }
|
||||
: {}),
|
||||
...(current?.resultClaim != null
|
||||
? {
|
||||
resultClaim: {
|
||||
|
|
@ -5794,10 +5801,40 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
|
|||
task.id,
|
||||
);
|
||||
const backgroundTask = resolveBackgroundTask();
|
||||
let durableReceiptReady = false;
|
||||
let durableReceiptAmbiguous = false;
|
||||
const persistDurableReceipt = async (receipt: {
|
||||
status: 'completed' | 'error' | 'cancelled';
|
||||
output?: string;
|
||||
}): Promise<boolean> => {
|
||||
if (completionAdmission?.persistResult == null) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
return await completionAdmission.persistResult({
|
||||
status: receipt.status,
|
||||
output: truncateMiddle(
|
||||
receipt.output ?? '',
|
||||
backgroundCompletionResultMaxChars,
|
||||
),
|
||||
settledAt: backgroundTask.settledAt,
|
||||
});
|
||||
} catch (receiptError) {
|
||||
durableReceiptAmbiguous = true;
|
||||
logger.warn(
|
||||
`[background] Failed to persist independent result receipt for task ${task.id}:`,
|
||||
receiptError,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
const retireFailedPersistence = async (
|
||||
reason: string,
|
||||
certainty: 'definite' | 'ambiguous',
|
||||
): Promise<void> => {
|
||||
if (durableReceiptAmbiguous) {
|
||||
return;
|
||||
}
|
||||
if (completionAdmission == null) {
|
||||
backgroundTaskRegistry.markCompletionPersistenceFailed(
|
||||
backgroundUserId,
|
||||
|
|
@ -5852,6 +5889,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
|
|||
}
|
||||
};
|
||||
if (!harvestEnabled || !persistBackgroundCodeResult) {
|
||||
durableReceiptReady = await persistDurableReceipt({
|
||||
status: params.status,
|
||||
output: params.output ?? localTask?.result,
|
||||
});
|
||||
if (
|
||||
backgroundToolCompletion == null ||
|
||||
detachedReservation?.status === 'reserved'
|
||||
|
|
@ -5870,17 +5911,19 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
|
|||
backgroundTask,
|
||||
resolveBackgroundTask,
|
||||
});
|
||||
if (!deliveryReady) {
|
||||
if (!deliveryReady && !durableReceiptReady) {
|
||||
await retireFailedPersistence(
|
||||
'background tool result was not persisted',
|
||||
'definite',
|
||||
);
|
||||
}
|
||||
} catch (persistError) {
|
||||
await retireFailedPersistence(
|
||||
'background tool result persistence failed',
|
||||
isContentFilterError(persistError) ? 'definite' : 'ambiguous',
|
||||
);
|
||||
if (!durableReceiptReady) {
|
||||
await retireFailedPersistence(
|
||||
'background tool result persistence failed',
|
||||
isContentFilterError(persistError) ? 'definite' : 'ambiguous',
|
||||
);
|
||||
}
|
||||
logger.warn(
|
||||
`[background] Failed to persist result for task ${task.id}:`,
|
||||
persistError,
|
||||
|
|
@ -5922,7 +5965,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
|
|||
task.id,
|
||||
params.artifact,
|
||||
);
|
||||
if (completionPreregistered) {
|
||||
if (completionPreregistered && !durableReceiptReady) {
|
||||
await retireFailedPersistence(
|
||||
'background code result had no durable message anchor',
|
||||
'definite',
|
||||
|
|
@ -5930,6 +5973,12 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
|
|||
}
|
||||
return;
|
||||
}
|
||||
if (persisted.deliveryReady !== false) {
|
||||
durableReceiptReady = await persistDurableReceipt({
|
||||
status: params.status,
|
||||
output: params.output ?? localTask?.result,
|
||||
});
|
||||
}
|
||||
if (persisted.deliveryReady === false) {
|
||||
await retireFailedPersistence(
|
||||
'background code result was not persisted',
|
||||
|
|
@ -5943,26 +5992,38 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
|
|||
persisted.attachments,
|
||||
);
|
||||
} catch (persistError) {
|
||||
if (completionPreregistered) {
|
||||
await retireFailedPersistence(
|
||||
'background code result persistence failed',
|
||||
isContentFilterError(persistError) ? 'definite' : 'ambiguous',
|
||||
);
|
||||
}
|
||||
if (isContentFilterError(persistError)) {
|
||||
const blockedMessage =
|
||||
persistError instanceof ContentFilterError
|
||||
? modelBoundContentFilterErrorMessage(persistError.body)
|
||||
: persistError.body.message;
|
||||
backgroundTaskRegistry.blockArtifact(
|
||||
backgroundUserId,
|
||||
backgroundConversationId,
|
||||
task.id,
|
||||
persistError instanceof ContentFilterError
|
||||
? modelBoundContentFilterErrorMessage(persistError.body)
|
||||
: persistError.body.message,
|
||||
blockedMessage,
|
||||
);
|
||||
durableReceiptReady = await persistDurableReceipt({
|
||||
status: 'error',
|
||||
output: blockedMessage,
|
||||
});
|
||||
if (completionPreregistered && !durableReceiptReady) {
|
||||
await retireFailedPersistence(
|
||||
'background code result was blocked by content policy',
|
||||
'definite',
|
||||
);
|
||||
}
|
||||
logger.warn(
|
||||
`[background] Generated code output for task ${task.id} was blocked by content policy.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (completionPreregistered && !durableReceiptReady) {
|
||||
await retireFailedPersistence(
|
||||
'background code result persistence failed',
|
||||
'ambiguous',
|
||||
);
|
||||
}
|
||||
logger.warn(
|
||||
`[background] Failed to persist code result for task ${task.id}:`,
|
||||
persistError,
|
||||
|
|
|
|||
|
|
@ -40,6 +40,8 @@ export interface BackgroundToolResultState {
|
|||
settledAt: Date;
|
||||
/** This exact task owns a pre-registered automatic continuation delivery. */
|
||||
completionWakeup?: true;
|
||||
/** Independent receipts retain task-local delivery ownership. */
|
||||
completionReceipt?: true;
|
||||
resultClaim?: {
|
||||
kind: 'manual' | 'wakeup';
|
||||
claimId: string;
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ export * from './handlers';
|
|||
export * from './guard';
|
||||
export * from './harvest';
|
||||
export * from './backgroundCompletion';
|
||||
export * from './backgroundClaims';
|
||||
export * from './backgroundCompletionWakeup';
|
||||
export * from './initialize';
|
||||
export * from './legacy';
|
||||
|
|
|
|||
|
|
@ -589,6 +589,32 @@ describe('createAgentTriggerDeliveryEngine', () => {
|
|||
expect(store.dead).toHaveBeenCalledWith(expect.objectContaining({ claimToken: 'claim-1' }));
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
'offers configured receipt recovery when already exhausted is %s',
|
||||
async (exhausted) => {
|
||||
const store = storeWith({
|
||||
claimNext: jest.fn(async () => delivery({ attempts: exhausted ? 1 : 0 })),
|
||||
});
|
||||
const engine = createAgentTriggerDeliveryEngine(
|
||||
{
|
||||
store,
|
||||
now: () => START,
|
||||
dispatch: async () => {
|
||||
throw new Error('database unavailable');
|
||||
},
|
||||
},
|
||||
{ concurrency: 1, maxAttempts: 1, retryCapMs: 123_000 },
|
||||
);
|
||||
await engine.runTick();
|
||||
expect(store.dead).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
receiptRetryAt: new Date(START.getTime() + 123_000),
|
||||
error: expect.objectContaining({ retryable: true }),
|
||||
}),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it('bounds a persisted last failure before exhausting its source', async () => {
|
||||
const oversized = 'x'.repeat(3_000);
|
||||
const store = storeWith({
|
||||
|
|
|
|||
|
|
@ -183,6 +183,7 @@ export interface AgentTriggerDeliveryStore {
|
|||
attempt: number;
|
||||
error: AgentTriggerDeliveryFailure;
|
||||
settledAt: Date;
|
||||
receiptRetryAt?: Date;
|
||||
}) => Promise<boolean>;
|
||||
}
|
||||
|
||||
|
|
@ -409,6 +410,9 @@ export function createAgentTriggerDeliveryEngine(
|
|||
attempt: delivery.attempts,
|
||||
error: recorded,
|
||||
settledAt: now(),
|
||||
...(recorded.retryable && {
|
||||
receiptRetryAt: new Date(now().getTime() + retryCapMs),
|
||||
}),
|
||||
});
|
||||
if (deadLettered) {
|
||||
logger.error('[agent-triggers] delivery dead-lettered after exhausting retries', {
|
||||
|
|
@ -532,6 +536,9 @@ export function createAgentTriggerDeliveryEngine(
|
|||
attempt,
|
||||
error: recorded,
|
||||
settledAt: attemptedAt,
|
||||
...(recorded.retryable && {
|
||||
receiptRetryAt: new Date(attemptedAt.getTime() + retryCapMs),
|
||||
}),
|
||||
});
|
||||
if (deadLettered) {
|
||||
logger.error('[agent-triggers] delivery dead-lettered', {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import {
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
|
||||
|
|
@ -180,6 +181,7 @@ describe('durable agent trigger service', () => {
|
|||
expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
workerCapabilities: [
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
|
||||
|
|
@ -218,6 +220,7 @@ describe('durable agent trigger service', () => {
|
|||
expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
workerCapabilities: [
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
|
||||
],
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import {
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
|
||||
|
|
@ -112,6 +113,9 @@ export interface AgentTriggerDeliveryPersistence {
|
|||
completeAgentTriggerDelivery: AgentTriggerDeliveryStore['complete'];
|
||||
retireAgentTriggerDelivery: AgentTriggerDeliveryMethods['retireAgentTriggerDelivery'];
|
||||
renewAgentTriggerDeliveryProducerLease: AgentTriggerDeliveryMethods['renewAgentTriggerDeliveryProducerLease'];
|
||||
persistAgentBackgroundToolResult?: AgentTriggerDeliveryMethods['persistAgentBackgroundToolResult'];
|
||||
getAgentBackgroundToolResultClaim?: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'];
|
||||
releaseAgentBackgroundToolResultClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];
|
||||
retryAgentTriggerDelivery: AgentTriggerDeliveryStore['retry'];
|
||||
deadLetterAgentTriggerDelivery: AgentTriggerDeliveryStore['dead'];
|
||||
getAgentTriggerDelivery: (deliveryKey: string) => Promise<AgentTriggerStoredRecord | null>;
|
||||
|
|
@ -168,6 +172,19 @@ export interface AgentTriggerService {
|
|||
options?: { onlyIfUnclaimed?: boolean; onlyIfDead?: boolean },
|
||||
) => Promise<boolean>;
|
||||
renewProducerLease: (deliveryKey: string, sourceId: string, leaseUntil: Date) => Promise<boolean>;
|
||||
persistBackgroundToolResult: (input: {
|
||||
deliveryKey: string;
|
||||
sourceId: string;
|
||||
result: {
|
||||
status: 'completed' | 'error' | 'cancelled';
|
||||
output: string;
|
||||
settledAt: Date;
|
||||
};
|
||||
}) => Promise<boolean>;
|
||||
getBackgroundToolResultClaim: (
|
||||
input: Parameters<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>[0],
|
||||
) => ReturnType<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>;
|
||||
releaseBackgroundToolResultClaims: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];
|
||||
drainUser: (userId: string) => Promise<void>;
|
||||
prepareUserPurge: (userId: string, fenceStartedAt: Date, tenantId?: string) => Promise<void>;
|
||||
cancelUserPurge: (userId: string, fenceStartedAt: Date) => Promise<boolean>;
|
||||
|
|
@ -183,6 +200,7 @@ function createDeliveryStore(
|
|||
methods.claimNextAgentTriggerDelivery({
|
||||
...input,
|
||||
workerCapabilities: [
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
|
||||
...(supportsDetachedActionCompletion()
|
||||
|
|
@ -600,6 +618,21 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
|
|||
leaseUntil,
|
||||
}),
|
||||
),
|
||||
persistBackgroundToolResult: (input) =>
|
||||
runAsSystem(async () => {
|
||||
const persist = requireMethods().persistAgentBackgroundToolResult;
|
||||
return persist == null ? false : persist(input);
|
||||
}),
|
||||
getBackgroundToolResultClaim: (input) =>
|
||||
runAsSystem(async () => {
|
||||
const getClaim = requireMethods().getAgentBackgroundToolResultClaim;
|
||||
return getClaim == null ? null : getClaim(input);
|
||||
}),
|
||||
releaseBackgroundToolResultClaims: (input) =>
|
||||
runAsSystem(async () => {
|
||||
const release = requireMethods().releaseAgentBackgroundToolResultClaims;
|
||||
return release == null ? false : release(input);
|
||||
}),
|
||||
drainUser,
|
||||
prepareUserPurge: (userId, fenceStartedAt, tenantId) =>
|
||||
runAsSystem(async () =>
|
||||
|
|
|
|||
|
|
@ -577,6 +577,7 @@ describe('agent background task config', () => {
|
|||
}
|
||||
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
|
||||
completionWakeups: true,
|
||||
completionResultMaxChars: 24 * 1024,
|
||||
ordinaryToolCancellation: false,
|
||||
});
|
||||
});
|
||||
|
|
@ -593,6 +594,7 @@ describe('agent background task config', () => {
|
|||
}
|
||||
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
|
||||
completionWakeups: false,
|
||||
completionResultMaxChars: 24 * 1024,
|
||||
ordinaryToolCancellation: false,
|
||||
});
|
||||
});
|
||||
|
|
@ -609,9 +611,31 @@ describe('agent background task config', () => {
|
|||
}
|
||||
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
|
||||
completionWakeups: true,
|
||||
completionResultMaxChars: 24 * 1024,
|
||||
ordinaryToolCancellation: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('accepts a bounded durable completion result limit', () => {
|
||||
const result = configSchema.safeParse({
|
||||
version: '1.0',
|
||||
endpoints: { agents: { backgroundTasks: { completionResultMaxChars: 4096 } } },
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect(result.data.endpoints?.agents?.backgroundTasks?.completionResultMaxChars).toBe(4096);
|
||||
}
|
||||
});
|
||||
|
||||
it.each([0, 64 * 1024 + 1])('rejects an unsafe completion result limit: %s', (limit) => {
|
||||
expect(
|
||||
configSchema.safeParse({
|
||||
version: '1.0',
|
||||
endpoints: { agents: { backgroundTasks: { completionResultMaxChars: limit } } },
|
||||
}).success,
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('speechTab schema', () => {
|
||||
|
|
|
|||
|
|
@ -22,6 +22,9 @@ import {
|
|||
MIN_BALANCE_RESERVATION_TTL_MS,
|
||||
DEFAULT_BALANCE_RESERVATION_TTL_MS,
|
||||
} from './balance';
|
||||
|
||||
export const AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT = 24 * 1024;
|
||||
export const AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX = 64 * 1024;
|
||||
import {
|
||||
MAX_SUBAGENTS,
|
||||
MAX_SUBAGENTS_CEILING,
|
||||
|
|
@ -1465,6 +1468,15 @@ export const agentsEndpointSchema = baseEndpointSchema
|
|||
backgroundTasks: z
|
||||
.object({
|
||||
completionWakeups: z.boolean().optional().default(true),
|
||||
/** Maximum terminal output copied into the private completion receipt.
|
||||
* Generated files remain governed by their separate attachment policy. */
|
||||
completionResultMaxChars: z
|
||||
.number()
|
||||
.int()
|
||||
.min(1)
|
||||
.max(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX)
|
||||
.optional()
|
||||
.default(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT),
|
||||
/** Cooperative cancellation for process-local ordinary tools. Off
|
||||
* by default so existing deployments opt into the new control. */
|
||||
ordinaryToolCancellation: z.boolean().optional().default(false),
|
||||
|
|
|
|||
|
|
@ -67,6 +67,8 @@ export {
|
|||
isCompactionSemanticIndexProjection,
|
||||
} from './types/compaction';
|
||||
export {
|
||||
AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
|
||||
|
|
|
|||
|
|
@ -2230,10 +2230,14 @@ describe('Conversation Operations', () => {
|
|||
const deleteAgentQueuedTurns = jest.fn(async () => {
|
||||
transitions.push('queue-retired');
|
||||
});
|
||||
const eraseAgentTriggerDeliveryConversationResults = jest.fn(async () => {
|
||||
transitions.push('trigger-results-erased');
|
||||
});
|
||||
const scopedMethods = createConversationMethods(mongoose, {
|
||||
getMessages,
|
||||
deleteMessages,
|
||||
deleteAgentQueuedTurns,
|
||||
eraseAgentTriggerDeliveryConversationResults,
|
||||
});
|
||||
|
||||
await scopedMethods.deleteConvos(
|
||||
|
|
@ -2249,7 +2253,99 @@ describe('Conversation Operations', () => {
|
|||
expect(deleteAgentQueuedTurns).toHaveBeenCalledWith('user123', [
|
||||
{ conversationId, tenantId: 'tenant-1' },
|
||||
]);
|
||||
expect(transitions).toEqual(['queue-retired', 'generation-drained']);
|
||||
expect(eraseAgentTriggerDeliveryConversationResults).toHaveBeenCalledWith('user123', [
|
||||
conversationId,
|
||||
]);
|
||||
expect(transitions).toEqual([
|
||||
'queue-retired',
|
||||
'generation-drained',
|
||||
'trigger-results-erased',
|
||||
]);
|
||||
});
|
||||
|
||||
it('preserves background receipts when a pre-delete drain fails', async () => {
|
||||
const conversationId = uuidv4();
|
||||
await Conversation.create({
|
||||
conversationId,
|
||||
user: 'user123',
|
||||
endpoint: EModelEndpoint.agents,
|
||||
});
|
||||
const eraseAgentTriggerDeliveryConversationResults = jest.fn(async () => undefined);
|
||||
const scopedMethods = createConversationMethods(mongoose, {
|
||||
getMessages,
|
||||
deleteMessages,
|
||||
eraseAgentTriggerDeliveryConversationResults,
|
||||
});
|
||||
|
||||
await expect(
|
||||
scopedMethods.deleteConvos(
|
||||
'user123',
|
||||
{ conversationId },
|
||||
{ beforeDelete: async () => Promise.reject(new Error('drain unavailable')) },
|
||||
),
|
||||
).rejects.toThrow('drain unavailable');
|
||||
|
||||
expect(await Conversation.findOne({ conversationId })).not.toBeNull();
|
||||
expect(eraseAgentTriggerDeliveryConversationResults).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('finishes descendant bookkeeping and message cleanup when receipt erasure fails', async () => {
|
||||
const parentId = uuidv4();
|
||||
const childId = uuidv4();
|
||||
const grandchildId = uuidv4();
|
||||
const ids = [parentId, childId, grandchildId];
|
||||
const project = await ChatProject.create({ user: 'user123', name: 'Receipt cleanup' });
|
||||
await ConversationTag.create({ user: 'user123', tag: 'work', count: 3, position: 0 });
|
||||
await Conversation.create(
|
||||
ids.map((conversationId, depth) => ({
|
||||
conversationId,
|
||||
user: 'user123',
|
||||
endpoint: EModelEndpoint.agents,
|
||||
tags: ['work'],
|
||||
chatProjectId: project._id.toString(),
|
||||
...(depth === 0
|
||||
? {}
|
||||
: {
|
||||
subagentThread: {
|
||||
rootConversationId: parentId,
|
||||
parentConversationId: ids[depth - 1],
|
||||
parentMessageId: `message-${depth}`,
|
||||
parentToolCallId: `call-${depth}`,
|
||||
subagentType: 'agent-child',
|
||||
subagentKind: 'agent',
|
||||
depth,
|
||||
},
|
||||
}),
|
||||
})),
|
||||
);
|
||||
const prepare = jest.fn(async () => undefined);
|
||||
const erase = jest.fn(async () => {
|
||||
throw new Error('receipt storage unavailable');
|
||||
});
|
||||
const scopedMethods = createConversationMethods(mongoose, {
|
||||
getMessages,
|
||||
deleteMessages,
|
||||
prepareAgentTriggerConversationResultErasure: prepare,
|
||||
eraseAgentTriggerDeliveryConversationResults: erase,
|
||||
});
|
||||
|
||||
const result = await scopedMethods.deleteConvos('user123', { conversationId: parentId });
|
||||
expect(result).toMatchObject({ deletedCount: 3, conversationIds: ids });
|
||||
expect(prepare.mock.calls).toHaveLength(3);
|
||||
expect(erase.mock.calls).toHaveLength(3);
|
||||
expect(await Conversation.countDocuments({ user: 'user123' })).toBe(0);
|
||||
expect((await ConversationTag.findOne({ user: 'user123', tag: 'work' }).lean())?.count).toBe(
|
||||
0,
|
||||
);
|
||||
expect((await ChatProject.findById(project._id).lean())?.conversationCount).toBe(0);
|
||||
expect(deleteMessages).toHaveBeenCalledWith({
|
||||
user: 'user123',
|
||||
conversationId: { $in: ids },
|
||||
});
|
||||
|
||||
await expect(
|
||||
scopedMethods.deleteConvos('user123', { conversationId: parentId }, { allowEmpty: true }),
|
||||
).resolves.toMatchObject({ deletedCount: 0, conversationIds: [parentId] });
|
||||
});
|
||||
|
||||
it('fails closed before deleting a conversation when queued-turn retirement fails', async () => {
|
||||
|
|
@ -2468,13 +2564,22 @@ describe('Conversation Operations', () => {
|
|||
});
|
||||
|
||||
it('allows a single-conversation cleanup retry after topology and messages are gone', async () => {
|
||||
const result = await deleteConvos(
|
||||
const eraseAgentTriggerDeliveryConversationResults = jest.fn(async () => undefined);
|
||||
const scopedMethods = createConversationMethods(mongoose, {
|
||||
getMessages,
|
||||
deleteMessages,
|
||||
eraseAgentTriggerDeliveryConversationResults,
|
||||
});
|
||||
const result = await scopedMethods.deleteConvos(
|
||||
'user123',
|
||||
{ conversationId: 'already-absent' },
|
||||
{ allowEmpty: true },
|
||||
);
|
||||
expect(result.deletedCount).toBe(0);
|
||||
expect(result.conversationIds).toEqual(['already-absent']);
|
||||
expect(eraseAgentTriggerDeliveryConversationResults).toHaveBeenCalledWith('user123', [
|
||||
'already-absent',
|
||||
]);
|
||||
});
|
||||
|
||||
it('supports an idempotent empty recovery sweep without hiding storage failures', async () => {
|
||||
|
|
|
|||
|
|
@ -512,6 +512,14 @@ export interface ConversationMethodDeps
|
|||
user: string,
|
||||
conversations: Array<{ conversationId: string; tenantId?: string; allTenants?: true }>,
|
||||
) => Promise<void>;
|
||||
eraseAgentTriggerDeliveryConversationResults?: (
|
||||
user: string,
|
||||
conversationIds: string[],
|
||||
) => Promise<void>;
|
||||
prepareAgentTriggerConversationResultErasure?: (
|
||||
user: string,
|
||||
conversationIds: string[],
|
||||
) => Promise<void>;
|
||||
}
|
||||
|
||||
export function createConversationMethods(
|
||||
|
|
@ -3235,7 +3243,18 @@ export function createConversationMethods(
|
|||
})),
|
||||
);
|
||||
await options?.beforeDelete?.(waveIds);
|
||||
await deps?.prepareAgentTriggerConversationResultErasure?.(user, waveIds);
|
||||
const result = await Conversation.deleteMany({ user, conversationId: { $in: waveIds } });
|
||||
if (result.deletedCount > 0) {
|
||||
/** Result erasure is irreversible. Keep receipts intact when a
|
||||
* pre-delete hook or the conversation delete itself fails, so a
|
||||
* retained conversation cannot lose a receipt-only completion. */
|
||||
try {
|
||||
await deps?.eraseAgentTriggerDeliveryConversationResults?.(user, waveIds);
|
||||
} catch (error) {
|
||||
logger.error('[deleteConvos] Receipt erasure deferred to durable cleanup', error);
|
||||
}
|
||||
}
|
||||
acknowledged &&= result.acknowledged;
|
||||
deletedCount += result.deletedCount;
|
||||
await reconcileDeletedWave(wave, result.deletedCount);
|
||||
|
|
@ -3266,6 +3285,11 @@ export function createConversationMethods(
|
|||
allTenants: true,
|
||||
})),
|
||||
);
|
||||
try {
|
||||
await deps?.eraseAgentTriggerDeliveryConversationResults?.(user, recoveryConversationIds);
|
||||
} catch (error) {
|
||||
logger.error('[deleteConvos] Receipt erasure deferred to durable cleanup', error);
|
||||
}
|
||||
}
|
||||
|
||||
const deleteConvoResult: DeleteResult = { acknowledged, deletedCount };
|
||||
|
|
|
|||
|
|
@ -325,6 +325,10 @@ export function createMethods(
|
|||
getMessages: messageMethods.getMessages,
|
||||
deleteMessages: messageMethods.deleteMessages,
|
||||
searchMessages: messageMethods.searchMessages,
|
||||
eraseAgentTriggerDeliveryConversationResults:
|
||||
agentTriggerDeliveryMethods.eraseAgentTriggerDeliveryConversationResults,
|
||||
prepareAgentTriggerConversationResultErasure:
|
||||
agentTriggerDeliveryMethods.prepareAgentTriggerConversationResultErasure,
|
||||
deleteAgentQueuedTurns: async (user, conversations) => {
|
||||
/** Queued-turn ownership is ObjectId-backed. Conversation methods also
|
||||
* support synthetic/non-ObjectId owners in embedded integrations and
|
||||
|
|
|
|||
|
|
@ -726,6 +726,7 @@ export interface MessageMethods {
|
|||
cancelled?: true;
|
||||
settledAt: Date;
|
||||
completionWakeup?: true;
|
||||
completionReceipt?: true;
|
||||
resultClaim?: {
|
||||
kind: 'manual' | 'wakeup';
|
||||
claimId: string;
|
||||
|
|
@ -1224,6 +1225,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
|
|||
cancelled?: true;
|
||||
settledAt: Date;
|
||||
completionWakeup?: true;
|
||||
completionReceipt?: true;
|
||||
resultClaim?: {
|
||||
kind: 'manual' | 'wakeup';
|
||||
claimId: string;
|
||||
|
|
@ -1274,6 +1276,9 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
|
|||
partPatch['content.$[part].tool_call.backgroundTask.cancelled'] =
|
||||
backgroundTask.cancelled === true;
|
||||
partPatch['content.$[part].tool_call.backgroundTask.settledAt'] = backgroundTask.settledAt;
|
||||
if (backgroundTask.completionReceipt === true) {
|
||||
partPatch['content.$[part].tool_call.backgroundTask.completionReceipt'] = true;
|
||||
}
|
||||
if (backgroundTask.completionWakeup === true) {
|
||||
partPatch['content.$[part].tool_call.backgroundTask.completionWakeup'] = true;
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -18,12 +18,14 @@ import {
|
|||
type AgentTriggerDeliveryMethods,
|
||||
} from './triggerDelivery';
|
||||
import {
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
|
||||
} from '~/types/triggerDelivery';
|
||||
import { createAgentTriggerLaneSequenceModel } from '../models/triggerLaneSequence';
|
||||
import { createAgentTriggerUserPurgeModel } from '../models/triggerUserPurge';
|
||||
import { createAgentTriggerDeliveryModel } from '../models/triggerDelivery';
|
||||
import { createConversationModel } from '../models/convo';
|
||||
import { createUserModel } from '../models/user';
|
||||
|
||||
jest.mock('~/config/winston', () => ({
|
||||
|
|
@ -32,6 +34,7 @@ jest.mock('~/config/winston', () => ({
|
|||
info: jest.fn(),
|
||||
debug: jest.fn(),
|
||||
}));
|
||||
jest.mock('~/models/plugins/mongoMeili', () => jest.fn());
|
||||
|
||||
const DB_SETUP_TIMEOUT_MS = 60_000;
|
||||
const START = new Date('2026-08-17T12:00:00.000Z');
|
||||
|
|
@ -50,6 +53,7 @@ beforeAll(async () => {
|
|||
LaneSequence = createAgentTriggerLaneSequenceModel(mongoose);
|
||||
UserPurge = createAgentTriggerUserPurgeModel(mongoose);
|
||||
User = createUserModel(mongoose);
|
||||
createConversationModel(mongoose);
|
||||
await Promise.all([Delivery.init(), LaneSequence.init(), UserPurge.init(), User.init()]);
|
||||
methods = createAgentTriggerDeliveryMethods(mongoose);
|
||||
}, DB_SETUP_TIMEOUT_MS);
|
||||
|
|
@ -384,7 +388,7 @@ describe('agent trigger delivery methods', () => {
|
|||
it('shields background completion work from workers that cannot resolve it', async () => {
|
||||
const queued = await methods.enqueueAgentTriggerDelivery(
|
||||
enqueueInput({
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
}),
|
||||
);
|
||||
const claimInput = {
|
||||
|
|
@ -400,7 +404,7 @@ describe('agent trigger delivery methods', () => {
|
|||
...claimInput,
|
||||
workerId: 'background-capable-worker',
|
||||
claimToken: 'background-capable-claim',
|
||||
workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1],
|
||||
workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2],
|
||||
}),
|
||||
).resolves.toMatchObject({ id: queued.delivery.id });
|
||||
});
|
||||
|
|
@ -412,7 +416,7 @@ describe('agent trigger delivery methods', () => {
|
|||
enqueueInput({
|
||||
deliveryKey: 'background-completion-producer-lease',
|
||||
envelope: { event: { source } },
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
producerLeaseUntil: initialLease,
|
||||
}),
|
||||
);
|
||||
|
|
@ -449,6 +453,323 @@ describe('agent trigger delivery methods', () => {
|
|||
).resolves.toEqual({ status: 'live', leaseUntil: renewedUntil });
|
||||
});
|
||||
|
||||
it('persists one private background result receipt independently of message rows', async () => {
|
||||
const source = { id: 'background-tool-completion', type: 'internal' };
|
||||
const queued = await methods.enqueueAgentTriggerDelivery(
|
||||
enqueueInput({
|
||||
deliveryKey: 'background-completion-result-receipt',
|
||||
envelope: { event: { source } },
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
}),
|
||||
);
|
||||
const input = {
|
||||
deliveryKey: queued.delivery.deliveryKey,
|
||||
sourceId: source.id,
|
||||
result: { status: 'completed' as const, output: 'durable output', settledAt: START },
|
||||
};
|
||||
|
||||
await expect(methods.persistAgentBackgroundToolResult(input)).resolves.toBe(true);
|
||||
await expect(methods.persistAgentBackgroundToolResult(input)).resolves.toBe(true);
|
||||
await expect(
|
||||
methods.getAgentBackgroundToolResult({
|
||||
deliveryKey: queued.delivery.deliveryKey,
|
||||
sourceId: source.id,
|
||||
}),
|
||||
).resolves.toEqual(input.result);
|
||||
await expect(
|
||||
methods.persistAgentBackgroundToolResult({
|
||||
...input,
|
||||
result: { ...input.result, output: 'conflicting output' },
|
||||
}),
|
||||
).resolves.toBe(false);
|
||||
const ordinaryRead = await Delivery.findById(queued.delivery.id).lean();
|
||||
expect(ordinaryRead).not.toHaveProperty('backgroundToolResult');
|
||||
});
|
||||
|
||||
it('erases private result receipts when their conversation is deleted', async () => {
|
||||
const source = { id: 'background-tool-completion', type: 'internal' };
|
||||
const user = new mongoose.Types.ObjectId();
|
||||
const queued = await methods.enqueueAgentTriggerDelivery(
|
||||
enqueueInput({
|
||||
deliveryKey: 'background-completion-result-to-erase',
|
||||
user,
|
||||
envelope: {
|
||||
event: { source },
|
||||
target: { conversationId: 'conversation-to-delete' },
|
||||
},
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
}),
|
||||
);
|
||||
await methods.persistAgentBackgroundToolResult({
|
||||
deliveryKey: queued.delivery.deliveryKey,
|
||||
sourceId: source.id,
|
||||
result: { status: 'completed', output: 'private output', settledAt: START },
|
||||
});
|
||||
|
||||
await methods.eraseAgentTriggerDeliveryConversationResults(user, ['conversation-to-delete']);
|
||||
|
||||
await expect(
|
||||
methods.getAgentBackgroundToolResult({
|
||||
deliveryKey: queued.delivery.deliveryKey,
|
||||
sourceId: source.id,
|
||||
}),
|
||||
).resolves.toBeNull();
|
||||
await expect(Delivery.exists({ _id: queued.delivery.id })).resolves.not.toBeNull();
|
||||
await expect(
|
||||
methods.persistAgentBackgroundToolResult({
|
||||
deliveryKey: queued.delivery.deliveryKey,
|
||||
sourceId: source.id,
|
||||
result: { status: 'completed', output: 'late private output', settledAt: START },
|
||||
}),
|
||||
).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it('does not make legacy string-owned conversation cleanup depend on an ObjectId cast', async () => {
|
||||
await expect(
|
||||
methods.prepareAgentTriggerConversationResultErasure('legacy-user', ['conversation']),
|
||||
).resolves.toBeUndefined();
|
||||
await expect(
|
||||
methods.eraseAgentTriggerDeliveryConversationResults('legacy-user', ['conversation']),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('recovers receipt erasure after a descendant row was already deleted', async () => {
|
||||
const user = new mongoose.Types.ObjectId();
|
||||
const source = { id: 'background-tool-completion', type: 'internal' };
|
||||
const queued = await methods.enqueueAgentTriggerDelivery(
|
||||
enqueueInput({
|
||||
user,
|
||||
envelope: { event: { source }, target: { conversationId: 'deleted-child' } },
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
}),
|
||||
);
|
||||
const lookup = { deliveryKey: queued.delivery.deliveryKey, sourceId: source.id };
|
||||
await methods.persistAgentBackgroundToolResult({
|
||||
...lookup,
|
||||
result: { status: 'completed', output: 'private child output', settledAt: START },
|
||||
});
|
||||
await methods.prepareAgentTriggerConversationResultErasure(user, ['deleted-child']);
|
||||
await mongoose.models.Conversation.collection.insertOne({
|
||||
user: user.toString(),
|
||||
conversationId: 'deleted-child',
|
||||
});
|
||||
await methods.recoverAgentTriggerUserPurges();
|
||||
await expect(methods.getAgentBackgroundToolResult(lookup)).resolves.not.toBeNull();
|
||||
await mongoose.models.Conversation.deleteOne({ user, conversationId: 'deleted-child' });
|
||||
/** The conversation deletion committed, but the follow-up receipt erasure did not. */
|
||||
await methods.recoverAgentTriggerUserPurges();
|
||||
await expect(methods.getAgentBackgroundToolResult(lookup)).resolves.toBeNull();
|
||||
await expect(
|
||||
Delivery.exists({
|
||||
_id: queued.delivery.id,
|
||||
backgroundToolResultDeletionPendingAt: { $exists: true },
|
||||
}),
|
||||
).resolves.toBeNull();
|
||||
await expect(
|
||||
methods.persistAgentBackgroundToolResult({
|
||||
...lookup,
|
||||
result: { status: 'completed', output: 'late write', settledAt: START },
|
||||
}),
|
||||
).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['receipt', 'BACKGROUND_TOOL_PRODUCER_LOST'],
|
||||
['death', 'BACKGROUND_TOOL_PRODUCER_LOST'],
|
||||
['receipt', 'PARENT_STATE_UNAVAILABLE'],
|
||||
['death', 'PARENT_STATE_UNAVAILABLE'],
|
||||
])('atomically fences result publication when %s wins before %s', async (winner, code) => {
|
||||
const source = { id: 'background-tool-completion', type: 'internal' };
|
||||
const queued = await methods.enqueueAgentTriggerDelivery(
|
||||
enqueueInput({
|
||||
envelope: { event: { source } },
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
}),
|
||||
);
|
||||
const owner = { workerId: 'worker', claimToken: 'claim', now: START };
|
||||
const claimed = await methods.claimNextAgentTriggerDelivery({
|
||||
...owner,
|
||||
leaseUntil: new Date(START.getTime() + 60_000),
|
||||
workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2],
|
||||
});
|
||||
expect(claimed).not.toBeNull();
|
||||
const attempt = await methods.beginAgentTriggerDeliveryAttempt({ ...owner, id: claimed!.id });
|
||||
const write = () =>
|
||||
methods.persistAgentBackgroundToolResult({
|
||||
deliveryKey: queued.delivery.deliveryKey,
|
||||
sourceId: source.id,
|
||||
result: { status: 'completed', output: 'late output', settledAt: START },
|
||||
});
|
||||
const dead = () =>
|
||||
methods.deadLetterAgentTriggerDelivery({
|
||||
...owner,
|
||||
id: claimed!.id,
|
||||
attempt: attempt!,
|
||||
settledAt: START,
|
||||
error: transientFailure({ code, retryable: code !== 'BACKGROUND_TOOL_PRODUCER_LOST' }),
|
||||
});
|
||||
await expect(winner === 'receipt' ? write() : dead()).resolves.toBe(true);
|
||||
await expect(winner === 'receipt' ? dead() : write()).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it.each(['shield', 'legacy'] as const)(
|
||||
'recovers a committed receipt after generic retry exhaustion through the %s fence',
|
||||
async (format) => {
|
||||
const source = { id: 'background-tool-completion', type: 'internal' };
|
||||
const queued = await methods.enqueueAgentTriggerDelivery(
|
||||
enqueueInput({
|
||||
envelope: { event: { source } },
|
||||
requiredWorkerCapability:
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
}),
|
||||
);
|
||||
const owner = { workerId: 'worker', claimToken: 'claim', now: START };
|
||||
const claimed = await methods.claimNextAgentTriggerDelivery({
|
||||
...owner,
|
||||
leaseUntil: new Date(START.getTime() + 60_000),
|
||||
workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2],
|
||||
});
|
||||
const id = claimed!.id;
|
||||
const attempt = await methods.beginAgentTriggerDeliveryAttempt({ ...owner, id });
|
||||
if (format !== 'shield') {
|
||||
await Delivery.updateOne(
|
||||
{ _id: id },
|
||||
{
|
||||
$set: {
|
||||
status: 'capability_leased',
|
||||
leaseBy: owner.workerId,
|
||||
claimToken: owner.claimToken,
|
||||
leaseUntil: new Date(START.getTime() + 60_000),
|
||||
},
|
||||
$unset: {
|
||||
capabilityStatus: 1,
|
||||
capabilityLeaseBy: 1,
|
||||
capabilityLeaseUntil: 1,
|
||||
capabilityClaimToken: 1,
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
const receipt = { deliveryKey: queued.delivery.deliveryKey, sourceId: source.id };
|
||||
await methods.persistAgentBackgroundToolResult({
|
||||
...receipt,
|
||||
result: { status: 'completed', output: 'saved output', settledAt: START },
|
||||
});
|
||||
const receiptRetryAt = new Date(START.getTime() + 300_000);
|
||||
const terminal = {
|
||||
...owner,
|
||||
id,
|
||||
attempt: attempt!,
|
||||
settledAt: START,
|
||||
receiptRetryAt,
|
||||
error: transientFailure({ code: 'PARENT_STATE_UNAVAILABLE' }),
|
||||
};
|
||||
await expect(
|
||||
methods.deadLetterAgentTriggerDelivery({ ...terminal, claimToken: 'stale' }),
|
||||
).resolves.toBe(false);
|
||||
expect((await Delivery.findById(id).lean())?.attempts).toBe(attempt);
|
||||
await expect(methods.deadLetterAgentTriggerDelivery(terminal)).resolves.toBe(false);
|
||||
expect(await Delivery.findById(id).lean()).toMatchObject({
|
||||
attempts: 0,
|
||||
claimAvailableAt: receiptRetryAt,
|
||||
...(format === 'shield'
|
||||
? { capabilityStatus: 'pending' }
|
||||
: { status: 'capability_pending' }),
|
||||
});
|
||||
await expect(methods.getAgentBackgroundToolResult(receipt)).resolves.toMatchObject({
|
||||
output: 'saved output',
|
||||
});
|
||||
const recovered = await methods.claimNextAgentTriggerDelivery({
|
||||
...owner,
|
||||
claimToken: 'recovered',
|
||||
now: receiptRetryAt,
|
||||
leaseUntil: new Date(receiptRetryAt.getTime() + 60_000),
|
||||
workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2],
|
||||
});
|
||||
expect(recovered?.id).toBe(id);
|
||||
expect(
|
||||
await methods.beginAgentTriggerDeliveryAttempt({
|
||||
...owner,
|
||||
id,
|
||||
claimToken: 'recovered',
|
||||
now: receiptRetryAt,
|
||||
}),
|
||||
).toBe(1);
|
||||
await expect(
|
||||
methods.deadLetterAgentTriggerDelivery({
|
||||
...terminal,
|
||||
claimToken: 'recovered',
|
||||
settledAt: receiptRetryAt,
|
||||
error: transientFailure({ code: 'PARENT_NOT_FOUND', retryable: false }),
|
||||
}),
|
||||
).resolves.toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it('claims an independent result receipt before returning its output', async () => {
|
||||
const source = { id: 'background-tool-completion', type: 'internal' };
|
||||
const user = new mongoose.Types.ObjectId();
|
||||
const queued = await methods.enqueueAgentTriggerDelivery(
|
||||
enqueueInput({
|
||||
deliveryKey: 'background-completion-result-to-claim',
|
||||
user,
|
||||
envelope: {
|
||||
event: {
|
||||
source,
|
||||
payload: { taskId: 'task-1', toolCallId: 'call-1', toolName: 'slow_tool' },
|
||||
},
|
||||
target: {
|
||||
agentId: 'agent-1',
|
||||
conversationId: 'conversation-1',
|
||||
parentMessageId: 'response-1',
|
||||
},
|
||||
},
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
}),
|
||||
);
|
||||
await methods.persistAgentBackgroundToolResult({
|
||||
deliveryKey: queued.delivery.deliveryKey,
|
||||
sourceId: source.id,
|
||||
result: { status: 'completed', output: 'private output', settledAt: START },
|
||||
});
|
||||
const claim = {
|
||||
deliveryKey: queued.delivery.deliveryKey,
|
||||
sourceId: source.id,
|
||||
userId: user.toString(),
|
||||
conversationId: 'conversation-1',
|
||||
parentMessageId: 'response-1',
|
||||
agentId: 'agent-1',
|
||||
claimId: 'delivery-claim-1',
|
||||
limit: 1,
|
||||
};
|
||||
|
||||
await expect(methods.claimAgentBackgroundToolResults(claim)).resolves.toMatchObject({
|
||||
status: 'acquired',
|
||||
results: [{ taskId: 'task-1', output: 'private output' }],
|
||||
});
|
||||
await expect(
|
||||
methods.claimAgentBackgroundToolResults({ ...claim, claimId: 'delivery-claim-2' }),
|
||||
).resolves.toEqual({ status: 'claimed', claimId: 'delivery-claim-1' });
|
||||
await expect(
|
||||
methods.getAgentBackgroundToolResultClaim({
|
||||
sourceId: source.id,
|
||||
userId: user.toString(),
|
||||
conversationId: 'conversation-1',
|
||||
parentMessageId: 'response-1',
|
||||
taskId: 'task-1',
|
||||
}),
|
||||
).resolves.toMatchObject({ kind: 'wakeup', claimId: 'delivery-claim-1' });
|
||||
await expect(
|
||||
methods.releaseAgentBackgroundToolResultClaims({
|
||||
sourceId: source.id,
|
||||
userId: user.toString(),
|
||||
conversationId: 'conversation-1',
|
||||
parentMessageId: 'response-1',
|
||||
claimId: 'delivery-claim-1',
|
||||
}),
|
||||
).resolves.toBe(true);
|
||||
});
|
||||
|
||||
it('keeps capability-fenced work limited to capable workers through lease recovery', async () => {
|
||||
const queued = await methods.enqueueAgentTriggerDelivery(
|
||||
enqueueInput({
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import type { FilterQuery, Model, Types } from 'mongoose';
|
||||
import type {
|
||||
AgentBackgroundToolResultReceipt,
|
||||
AgentEventActorDetachedAction,
|
||||
AgentTriggerDeliveryClaim,
|
||||
AgentEventActorReceipt,
|
||||
|
|
@ -17,6 +18,8 @@ import type {
|
|||
IAgentTriggerUserPurgeDocument,
|
||||
} from '~/types/triggerDelivery';
|
||||
import {
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
|
||||
|
|
@ -129,6 +132,26 @@ export type AgentTriggerProducerLeaseStatus =
|
|||
| { status: 'expired'; leaseUntil: Date }
|
||||
| { status: 'missing' };
|
||||
|
||||
export interface PersistAgentBackgroundToolResultInput {
|
||||
deliveryKey: string;
|
||||
sourceId: string;
|
||||
result: AgentBackgroundToolResultReceipt;
|
||||
}
|
||||
|
||||
export type AgentBackgroundToolResultClaim =
|
||||
| { status: 'not_ready' }
|
||||
| { status: 'claimed'; claimId: string }
|
||||
| {
|
||||
status: 'acquired';
|
||||
results: Array<{
|
||||
taskId: string;
|
||||
toolCallId: string;
|
||||
toolName: string;
|
||||
status: AgentBackgroundToolResultReceipt['status'];
|
||||
output: string;
|
||||
}>;
|
||||
};
|
||||
|
||||
export interface AgentTriggerDeliveryFence {
|
||||
id: string;
|
||||
workerId: string;
|
||||
|
|
@ -277,6 +300,37 @@ export interface AgentTriggerDeliveryMethods {
|
|||
sourceId: string;
|
||||
now: Date;
|
||||
}) => Promise<AgentTriggerProducerLeaseStatus>;
|
||||
persistAgentBackgroundToolResult: (
|
||||
input: PersistAgentBackgroundToolResultInput,
|
||||
) => Promise<boolean>;
|
||||
getAgentBackgroundToolResult: (input: {
|
||||
deliveryKey: string;
|
||||
sourceId: string;
|
||||
}) => Promise<AgentBackgroundToolResultReceipt | null>;
|
||||
getAgentBackgroundToolResultClaim: (input: {
|
||||
sourceId: string;
|
||||
userId: string;
|
||||
conversationId: string;
|
||||
parentMessageId: string;
|
||||
taskId: string;
|
||||
}) => Promise<AgentBackgroundToolResultReceipt['resultClaim'] | null>;
|
||||
claimAgentBackgroundToolResults: (input: {
|
||||
deliveryKey: string;
|
||||
sourceId: string;
|
||||
userId: string;
|
||||
conversationId: string;
|
||||
parentMessageId: string;
|
||||
agentId?: string;
|
||||
claimId: string;
|
||||
limit?: number;
|
||||
}) => Promise<AgentBackgroundToolResultClaim>;
|
||||
releaseAgentBackgroundToolResultClaims: (input: {
|
||||
sourceId: string;
|
||||
userId: string;
|
||||
conversationId: string;
|
||||
parentMessageId: string;
|
||||
claimId: string;
|
||||
}) => Promise<boolean>;
|
||||
settleAgentTriggerHandlingOutcome: (
|
||||
input: SettleAgentTriggerHandlingOutcomeInput,
|
||||
) => Promise<boolean>;
|
||||
|
|
@ -326,6 +380,7 @@ export interface AgentTriggerDeliveryMethods {
|
|||
attempt: number;
|
||||
error: AgentTriggerDeliveryFailure;
|
||||
settledAt: Date;
|
||||
receiptRetryAt?: Date;
|
||||
},
|
||||
) => Promise<boolean>;
|
||||
getAgentTriggerDelivery: (deliveryKey: string) => Promise<AgentTriggerDeliveryRecord | null>;
|
||||
|
|
@ -358,6 +413,14 @@ export interface AgentTriggerDeliveryMethods {
|
|||
) => Promise<boolean>;
|
||||
recoverAgentTriggerUserPurges: (limit?: number) => Promise<number>;
|
||||
deleteAgentTriggerDeliveriesByUser: (user: string | Types.ObjectId) => Promise<void>;
|
||||
eraseAgentTriggerDeliveryConversationResults: (
|
||||
user: string | Types.ObjectId,
|
||||
conversationIds: string[],
|
||||
) => Promise<void>;
|
||||
prepareAgentTriggerConversationResultErasure: (
|
||||
user: string | Types.ObjectId,
|
||||
conversationIds: string[],
|
||||
) => Promise<void>;
|
||||
}
|
||||
|
||||
function normalizeFailure(error: AgentTriggerDeliveryFailure): AgentTriggerDeliveryFailure {
|
||||
|
|
@ -960,6 +1023,8 @@ export function createAgentTriggerDeliveryMethods(
|
|||
if (
|
||||
input.requiredWorkerCapability != null &&
|
||||
input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 &&
|
||||
input.requiredWorkerCapability !==
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 &&
|
||||
input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 &&
|
||||
input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1
|
||||
) {
|
||||
|
|
@ -2122,7 +2187,12 @@ export function createAgentTriggerDeliveryMethods(
|
|||
const renewed = await Delivery().updateOne(
|
||||
{
|
||||
deliveryKey: input.deliveryKey,
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
requiredWorkerCapability: {
|
||||
$in: [
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
],
|
||||
},
|
||||
'envelope.event.source.type': 'internal',
|
||||
'envelope.event.source.id': input.sourceId,
|
||||
status: {
|
||||
|
|
@ -2161,7 +2231,12 @@ export function createAgentTriggerDeliveryMethods(
|
|||
const delivery = await Delivery()
|
||||
.findOne({
|
||||
deliveryKey: input.deliveryKey,
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
requiredWorkerCapability: {
|
||||
$in: [
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
],
|
||||
},
|
||||
'envelope.event.source.type': 'internal',
|
||||
'envelope.event.source.id': input.sourceId,
|
||||
})
|
||||
|
|
@ -2175,6 +2250,262 @@ export function createAgentTriggerDeliveryMethods(
|
|||
: { status: 'expired', leaseUntil: delivery.producerLeaseUntil };
|
||||
}
|
||||
|
||||
/** Stores terminal output on the pre-admitted delivery before attempting the
|
||||
* parent-message projection. The first terminal receipt wins; exact retries
|
||||
* are idempotent and conflicting rewrites fail closed. */
|
||||
async function persistAgentBackgroundToolResult(
|
||||
input: PersistAgentBackgroundToolResultInput,
|
||||
): Promise<boolean> {
|
||||
const { result } = input;
|
||||
if (
|
||||
input.deliveryKey.length === 0 ||
|
||||
input.deliveryKey.length > 256 ||
|
||||
input.sourceId.length === 0 ||
|
||||
input.sourceId.length > 256 ||
|
||||
!['completed', 'error', 'cancelled'].includes(result.status) ||
|
||||
result.output.length > AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS ||
|
||||
!(result.settledAt instanceof Date) ||
|
||||
!Number.isFinite(result.settledAt.getTime())
|
||||
) {
|
||||
throw new TypeError('Invalid background tool result receipt');
|
||||
}
|
||||
const identity = {
|
||||
deliveryKey: input.deliveryKey,
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
'envelope.event.source.type': 'internal',
|
||||
'envelope.event.source.id': input.sourceId,
|
||||
backgroundToolResultErasedAt: { $exists: false },
|
||||
};
|
||||
const updated = await Delivery().updateOne(
|
||||
{
|
||||
...identity,
|
||||
backgroundToolResult: { $exists: false },
|
||||
status: { $nin: ['dead', 'capability_dead', 'succeeded'] },
|
||||
capabilityStatus: { $ne: 'dead' },
|
||||
},
|
||||
{ $set: { backgroundToolResult: result } },
|
||||
{ timestamps: false },
|
||||
);
|
||||
if (updated.matchedCount === 1) {
|
||||
return true;
|
||||
}
|
||||
const existing = await Delivery()
|
||||
.findOne(identity)
|
||||
.select('+backgroundToolResult')
|
||||
.lean<Pick<IAgentTriggerDelivery, 'backgroundToolResult'>>();
|
||||
return (
|
||||
existing?.backgroundToolResult?.status === result.status &&
|
||||
existing.backgroundToolResult.output === result.output &&
|
||||
existing.backgroundToolResult.settledAt.getTime() === result.settledAt.getTime()
|
||||
);
|
||||
}
|
||||
|
||||
async function getAgentBackgroundToolResult(input: {
|
||||
deliveryKey: string;
|
||||
sourceId: string;
|
||||
}): Promise<AgentBackgroundToolResultReceipt | null> {
|
||||
if (
|
||||
input.deliveryKey.length === 0 ||
|
||||
input.deliveryKey.length > 256 ||
|
||||
input.sourceId.length === 0 ||
|
||||
input.sourceId.length > 256
|
||||
) {
|
||||
throw new TypeError('Invalid background tool result receipt lookup');
|
||||
}
|
||||
const delivery = await Delivery()
|
||||
.findOne({
|
||||
deliveryKey: input.deliveryKey,
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
'envelope.event.source.type': 'internal',
|
||||
'envelope.event.source.id': input.sourceId,
|
||||
})
|
||||
.select('+backgroundToolResult')
|
||||
.lean<Pick<IAgentTriggerDelivery, 'backgroundToolResult'>>();
|
||||
return delivery?.backgroundToolResult ?? null;
|
||||
}
|
||||
|
||||
async function getAgentBackgroundToolResultClaim(input: {
|
||||
sourceId: string;
|
||||
userId: string;
|
||||
conversationId: string;
|
||||
parentMessageId: string;
|
||||
taskId: string;
|
||||
}): Promise<AgentBackgroundToolResultReceipt['resultClaim'] | null> {
|
||||
const delivery = await Delivery()
|
||||
.findOne({
|
||||
...backgroundResultIdentity(input),
|
||||
'envelope.event.payload.taskId': input.taskId,
|
||||
'backgroundToolResult.resultClaim': { $exists: true },
|
||||
})
|
||||
.select('+backgroundToolResult')
|
||||
.lean<Pick<IAgentTriggerDelivery, 'backgroundToolResult'>>();
|
||||
return delivery?.backgroundToolResult?.resultClaim ?? null;
|
||||
}
|
||||
|
||||
function backgroundResultIdentity(input: {
|
||||
sourceId: string;
|
||||
userId: string;
|
||||
conversationId: string;
|
||||
parentMessageId: string;
|
||||
agentId?: string;
|
||||
}) {
|
||||
return {
|
||||
user: input.userId,
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
'envelope.event.source.type': 'internal',
|
||||
'envelope.event.source.id': input.sourceId,
|
||||
'envelope.target.conversationId': input.conversationId,
|
||||
'envelope.target.parentMessageId': input.parentMessageId,
|
||||
...(input.agentId == null ? {} : { 'envelope.target.agentId': input.agentId }),
|
||||
};
|
||||
}
|
||||
|
||||
function projectBackgroundResult(delivery: {
|
||||
envelope?: unknown;
|
||||
backgroundToolResult?: AgentBackgroundToolResultReceipt;
|
||||
}) {
|
||||
const envelope = delivery.envelope;
|
||||
if (envelope == null || typeof envelope !== 'object' || Array.isArray(envelope)) {
|
||||
return null;
|
||||
}
|
||||
const event = (envelope as { event?: unknown }).event;
|
||||
if (event == null || typeof event !== 'object' || Array.isArray(event)) {
|
||||
return null;
|
||||
}
|
||||
const payload = (event as { payload?: unknown }).payload;
|
||||
if (payload == null || typeof payload !== 'object' || Array.isArray(payload)) {
|
||||
return null;
|
||||
}
|
||||
const { taskId, toolCallId, toolName } = payload as Record<string, unknown>;
|
||||
const receipt = delivery.backgroundToolResult;
|
||||
if (
|
||||
receipt == null ||
|
||||
typeof taskId !== 'string' ||
|
||||
typeof toolCallId !== 'string' ||
|
||||
typeof toolName !== 'string'
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
taskId,
|
||||
toolCallId,
|
||||
toolName,
|
||||
status: receipt.status,
|
||||
output: receipt.output,
|
||||
};
|
||||
}
|
||||
|
||||
/** Elects exactly one automatic continuation for an independently persisted
|
||||
* result. A later stack layer may coalesce siblings after this ownership
|
||||
* boundary, but the durable receipt itself is never dispatched unclaimed. */
|
||||
async function claimAgentBackgroundToolResults(input: {
|
||||
deliveryKey: string;
|
||||
sourceId: string;
|
||||
userId: string;
|
||||
conversationId: string;
|
||||
parentMessageId: string;
|
||||
agentId?: string;
|
||||
claimId: string;
|
||||
limit?: number;
|
||||
}): Promise<AgentBackgroundToolResultClaim> {
|
||||
if (
|
||||
input.deliveryKey.length === 0 ||
|
||||
input.deliveryKey.length > 256 ||
|
||||
input.sourceId.length === 0 ||
|
||||
input.sourceId.length > 256 ||
|
||||
input.userId.length === 0 ||
|
||||
input.conversationId.length === 0 ||
|
||||
input.conversationId.length > 256 ||
|
||||
input.parentMessageId.length === 0 ||
|
||||
input.parentMessageId.length > 256 ||
|
||||
input.claimId.length === 0 ||
|
||||
input.claimId.length > 128
|
||||
) {
|
||||
throw new TypeError('Invalid background tool result receipt claim');
|
||||
}
|
||||
const scope = backgroundResultIdentity(input);
|
||||
const projection = 'envelope +backgroundToolResult';
|
||||
let requested = await Delivery()
|
||||
.findOne({ ...scope, deliveryKey: input.deliveryKey })
|
||||
.select(projection)
|
||||
.lean<Pick<IAgentTriggerDelivery, 'envelope' | 'backgroundToolResult'>>();
|
||||
if (requested?.backgroundToolResult == null) {
|
||||
return { status: 'not_ready' };
|
||||
}
|
||||
const existingClaim = requested.backgroundToolResult.resultClaim;
|
||||
if (existingClaim != null && existingClaim.claimId !== input.claimId) {
|
||||
return { status: 'claimed', claimId: existingClaim.claimId };
|
||||
}
|
||||
if (existingClaim == null) {
|
||||
const claimed = await Delivery().updateOne(
|
||||
{
|
||||
...scope,
|
||||
deliveryKey: input.deliveryKey,
|
||||
backgroundToolResult: { $exists: true },
|
||||
'backgroundToolResult.resultClaim': { $exists: false },
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
'backgroundToolResult.resultClaim': {
|
||||
kind: 'wakeup',
|
||||
claimId: input.claimId,
|
||||
claimedAt: new Date(),
|
||||
},
|
||||
},
|
||||
},
|
||||
{ timestamps: false },
|
||||
);
|
||||
if (claimed.modifiedCount !== 1) {
|
||||
requested = await Delivery()
|
||||
.findOne({ ...scope, deliveryKey: input.deliveryKey })
|
||||
.select(projection)
|
||||
.lean<Pick<IAgentTriggerDelivery, 'envelope' | 'backgroundToolResult'>>();
|
||||
const winner = requested?.backgroundToolResult?.resultClaim;
|
||||
if (winner?.claimId !== input.claimId) {
|
||||
return winner == null
|
||||
? { status: 'not_ready' }
|
||||
: { status: 'claimed', claimId: winner.claimId };
|
||||
}
|
||||
}
|
||||
}
|
||||
const result = requested == null ? null : projectBackgroundResult(requested);
|
||||
return result == null ? { status: 'not_ready' } : { status: 'acquired', results: [result] };
|
||||
}
|
||||
|
||||
async function releaseAgentBackgroundToolResultClaims(input: {
|
||||
sourceId: string;
|
||||
userId: string;
|
||||
conversationId: string;
|
||||
parentMessageId: string;
|
||||
claimId: string;
|
||||
}): Promise<boolean> {
|
||||
if (
|
||||
input.sourceId.length === 0 ||
|
||||
input.sourceId.length > 256 ||
|
||||
input.userId.length === 0 ||
|
||||
input.conversationId.length === 0 ||
|
||||
input.conversationId.length > 256 ||
|
||||
input.parentMessageId.length === 0 ||
|
||||
input.parentMessageId.length > 256 ||
|
||||
input.claimId.length === 0 ||
|
||||
input.claimId.length > 128
|
||||
) {
|
||||
throw new TypeError('Invalid background tool result receipt claim release');
|
||||
}
|
||||
const scope = backgroundResultIdentity(input);
|
||||
await Delivery().updateMany(
|
||||
{ ...scope, 'backgroundToolResult.resultClaim.claimId': input.claimId },
|
||||
{ $unset: { 'backgroundToolResult.resultClaim': 1 } },
|
||||
{ timestamps: false },
|
||||
);
|
||||
return (
|
||||
(await Delivery().exists({
|
||||
...scope,
|
||||
'backgroundToolResult.resultClaim.claimId': input.claimId,
|
||||
})) == null
|
||||
);
|
||||
}
|
||||
|
||||
async function settleAgentTriggerHandlingOutcome(
|
||||
input: SettleAgentTriggerHandlingOutcomeInput,
|
||||
): Promise<boolean> {
|
||||
|
|
@ -3076,10 +3407,26 @@ export function createAgentTriggerDeliveryMethods(
|
|||
error: AgentTriggerDeliveryFailure;
|
||||
availableAt: Date;
|
||||
},
|
||||
): Promise<boolean> {
|
||||
return retryDelivery(input);
|
||||
}
|
||||
|
||||
async function retryDelivery(
|
||||
input: Parameters<AgentTriggerDeliveryMethods['retryAgentTriggerDelivery']>[0],
|
||||
recoverReceipt = false,
|
||||
): Promise<boolean> {
|
||||
const error = normalizeFailure(input.error);
|
||||
const receiptFence = recoverReceipt
|
||||
? {
|
||||
requiredWorkerCapability:
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
backgroundToolResult: { $exists: true },
|
||||
}
|
||||
: {};
|
||||
const recoveredAttempt = recoverReceipt ? { attempts: 0 } : {};
|
||||
const retryUpdate = (status: 'pending' | 'capability_pending') => ({
|
||||
$set: {
|
||||
...recoveredAttempt,
|
||||
status,
|
||||
availableAt: input.availableAt,
|
||||
claimAvailableAt: input.availableAt,
|
||||
|
|
@ -3108,9 +3455,10 @@ export function createAgentTriggerDeliveryMethods(
|
|||
},
|
||||
});
|
||||
const shieldResult = await Delivery().updateOne(
|
||||
{ _id: input.id, ...shieldCapabilityFence(input) },
|
||||
{ _id: input.id, ...shieldCapabilityFence(input), ...receiptFence },
|
||||
{
|
||||
$set: {
|
||||
...recoveredAttempt,
|
||||
status: 'leased',
|
||||
availableAt: input.availableAt,
|
||||
capabilityStatus: 'pending',
|
||||
|
|
@ -3134,14 +3482,14 @@ export function createAgentTriggerDeliveryMethods(
|
|||
return true;
|
||||
}
|
||||
const capabilityResult = await Delivery().updateOne(
|
||||
{ _id: input.id, ...legacyCapabilityFence(input) },
|
||||
{ _id: input.id, ...legacyCapabilityFence(input), ...receiptFence },
|
||||
retryUpdate('capability_pending'),
|
||||
);
|
||||
if (capabilityResult.modifiedCount === 1) {
|
||||
return true;
|
||||
}
|
||||
const result = await Delivery().updateOne(
|
||||
{ _id: input.id, ...ordinaryFence(input) },
|
||||
{ _id: input.id, ...ordinaryFence(input), ...receiptFence },
|
||||
retryUpdate('pending'),
|
||||
);
|
||||
return result.modifiedCount === 1;
|
||||
|
|
@ -3152,9 +3500,23 @@ export function createAgentTriggerDeliveryMethods(
|
|||
attempt: number;
|
||||
error: AgentTriggerDeliveryFailure;
|
||||
settledAt: Date;
|
||||
receiptRetryAt?: Date;
|
||||
},
|
||||
): Promise<boolean> {
|
||||
const error = normalizeFailure(input.error);
|
||||
if (
|
||||
error.retryable &&
|
||||
input.receiptRetryAt != null &&
|
||||
(await retryDelivery({ ...input, availableAt: input.receiptRetryAt }, true))
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
/** Publication must contend with every retryable terminal transition,
|
||||
* including a receipt committed between recovery and the dead-letter CAS. */
|
||||
const receiptFence =
|
||||
error.retryable || error.code === 'BACKGROUND_TOOL_PRODUCER_LOST'
|
||||
? { backgroundToolResult: { $exists: false } }
|
||||
: {};
|
||||
const deadUpdate = (status: 'dead' | 'capability_dead') => ({
|
||||
$set: { status, settledAt: input.settledAt, lastError: error },
|
||||
$unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1, expiresAt: 1 },
|
||||
|
|
@ -3174,7 +3536,7 @@ export function createAgentTriggerDeliveryMethods(
|
|||
},
|
||||
});
|
||||
const shieldDead = await Delivery().updateOne(
|
||||
{ _id: input.id, ...shieldCapabilityFence(input) },
|
||||
{ _id: input.id, ...shieldCapabilityFence(input), ...receiptFence },
|
||||
{
|
||||
$set: {
|
||||
// The pre-capability runtime already treats `capability_dead` as a
|
||||
|
|
@ -3201,16 +3563,20 @@ export function createAgentTriggerDeliveryMethods(
|
|||
return true;
|
||||
}
|
||||
const capabilityDead = await Delivery().updateOne(
|
||||
{ _id: input.id, ...legacyCapabilityFence(input) },
|
||||
{ _id: input.id, ...legacyCapabilityFence(input), ...receiptFence },
|
||||
deadUpdate('capability_dead'),
|
||||
);
|
||||
if (capabilityDead.modifiedCount === 1) {
|
||||
return true;
|
||||
}
|
||||
const dead = await Delivery()
|
||||
.findOneAndUpdate({ _id: input.id, ...ordinaryFence(input) }, deadUpdate('dead'), {
|
||||
new: true,
|
||||
})
|
||||
.findOneAndUpdate(
|
||||
{ _id: input.id, ...ordinaryFence(input), ...receiptFence },
|
||||
deadUpdate('dead'),
|
||||
{
|
||||
new: true,
|
||||
},
|
||||
)
|
||||
.select('_id orderingKey batchMemberIds batchMembersSettledAt requeueCount')
|
||||
.lean<
|
||||
Pick<
|
||||
|
|
@ -3512,6 +3878,31 @@ export function createAgentTriggerDeliveryMethods(
|
|||
if (!Number.isSafeInteger(limit) || limit <= 0) {
|
||||
throw new TypeError('Agent trigger purge recovery limit must be a positive integer');
|
||||
}
|
||||
const pending = await Delivery()
|
||||
.find({
|
||||
backgroundToolResultDeletionPendingAt: { $exists: true },
|
||||
})
|
||||
.select('user envelope')
|
||||
.sort({ backgroundToolResultDeletionPendingAt: 1, _id: 1 })
|
||||
.limit(Math.min(limit, MAX_PURGE_RECOVERY_LIMIT))
|
||||
.lean<IAgentTriggerDelivery[]>();
|
||||
for (const row of pending) {
|
||||
const conversationId = (row.envelope as { target?: { conversationId?: string } }).target
|
||||
?.conversationId;
|
||||
if (conversationId == null) {
|
||||
continue;
|
||||
}
|
||||
const exists = await mongoose.models.Conversation.exists({ user: row.user, conversationId });
|
||||
if (exists == null) {
|
||||
await eraseAgentTriggerDeliveryConversationResults(row.user, [conversationId]);
|
||||
} else {
|
||||
await Delivery().updateOne(
|
||||
{ _id: row._id, backgroundToolResultDeletionPendingAt: { $exists: true } },
|
||||
{ $currentDate: { backgroundToolResultDeletionPendingAt: true } },
|
||||
{ timestamps: false },
|
||||
);
|
||||
}
|
||||
}
|
||||
const markers = await UserPurge()
|
||||
.find({})
|
||||
.sort({ updatedAt: 1, _id: 1 })
|
||||
|
|
@ -3565,6 +3956,44 @@ export function createAgentTriggerDeliveryMethods(
|
|||
await UserPurge().deleteOne({ _id: user });
|
||||
}
|
||||
|
||||
async function eraseAgentTriggerDeliveryConversationResults(
|
||||
user: string | Types.ObjectId,
|
||||
conversationIds: string[],
|
||||
): Promise<void> {
|
||||
if (conversationIds.length === 0 || !mongoose.isObjectIdOrHexString(user)) {
|
||||
return;
|
||||
}
|
||||
const erasedAt = new Date();
|
||||
await Delivery().updateMany(
|
||||
{
|
||||
user,
|
||||
'envelope.target.conversationId': { $in: conversationIds },
|
||||
},
|
||||
{
|
||||
$set: { backgroundToolResultErasedAt: erasedAt },
|
||||
$unset: { backgroundToolResult: 1, backgroundToolResultDeletionPendingAt: 1 },
|
||||
},
|
||||
{ timestamps: false },
|
||||
);
|
||||
}
|
||||
|
||||
async function prepareAgentTriggerConversationResultErasure(
|
||||
user: string | Types.ObjectId,
|
||||
conversationIds: string[],
|
||||
): Promise<void> {
|
||||
if (conversationIds.length === 0 || !mongoose.isObjectIdOrHexString(user)) return;
|
||||
await Delivery().updateMany(
|
||||
{
|
||||
user,
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
'envelope.target.conversationId': { $in: conversationIds },
|
||||
backgroundToolResultErasedAt: { $exists: false },
|
||||
},
|
||||
{ $set: { backgroundToolResultDeletionPendingAt: new Date() } },
|
||||
{ timestamps: false },
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
ensureAgentTriggerDeliveryIndexes,
|
||||
enqueueAgentTriggerDelivery,
|
||||
|
|
@ -3578,6 +4007,11 @@ export function createAgentTriggerDeliveryMethods(
|
|||
retireAgentTriggerDelivery,
|
||||
renewAgentTriggerDeliveryProducerLease,
|
||||
getAgentTriggerDeliveryProducerLease,
|
||||
persistAgentBackgroundToolResult,
|
||||
getAgentBackgroundToolResult,
|
||||
getAgentBackgroundToolResultClaim,
|
||||
claimAgentBackgroundToolResults,
|
||||
releaseAgentBackgroundToolResultClaims,
|
||||
settleAgentTriggerHandlingOutcome,
|
||||
admitAgentEventActorAction,
|
||||
releaseAgentEventActorAction,
|
||||
|
|
@ -3606,5 +4040,7 @@ export function createAgentTriggerDeliveryMethods(
|
|||
cancelAgentTriggerUserPurge,
|
||||
recoverAgentTriggerUserPurges,
|
||||
deleteAgentTriggerDeliveriesByUser,
|
||||
eraseAgentTriggerDeliveryConversationResults,
|
||||
prepareAgentTriggerConversationResultErasure,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { Schema } from 'mongoose';
|
||||
import type { IAgentTriggerDeliveryDocument } from '~/types/triggerDelivery';
|
||||
import { AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS } from '~/types/triggerDelivery';
|
||||
|
||||
const failureSchema = new Schema(
|
||||
{
|
||||
|
|
@ -147,6 +148,38 @@ const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(
|
|||
capabilityClaimToken: { type: String },
|
||||
/** Private process-owner heartbeat; never projected to legacy consumers. */
|
||||
producerLeaseUntil: { type: Date, select: false },
|
||||
backgroundToolResult: {
|
||||
type: new Schema(
|
||||
{
|
||||
status: {
|
||||
type: String,
|
||||
enum: ['completed', 'error', 'cancelled'],
|
||||
required: true,
|
||||
},
|
||||
output: {
|
||||
type: String,
|
||||
required: true,
|
||||
maxlength: AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
|
||||
},
|
||||
settledAt: { type: Date, required: true },
|
||||
resultClaim: {
|
||||
type: new Schema(
|
||||
{
|
||||
kind: { type: String, enum: ['wakeup'], required: true },
|
||||
claimId: { type: String, required: true, maxlength: 128 },
|
||||
claimedAt: { type: Date, required: true },
|
||||
},
|
||||
{ _id: false },
|
||||
),
|
||||
required: false,
|
||||
},
|
||||
},
|
||||
{ _id: false },
|
||||
),
|
||||
select: false,
|
||||
},
|
||||
backgroundToolResultErasedAt: { type: Date, select: false },
|
||||
backgroundToolResultDeletionPendingAt: { type: Date, select: false },
|
||||
attempts: { type: Number, required: true, default: 0, min: 0 },
|
||||
availableAt: { type: Date, required: true },
|
||||
envelopeBytes: { type: Number, min: 0 },
|
||||
|
|
@ -224,6 +257,7 @@ triggerDeliverySchema.index({ 'actorReceipt.resolution': 1 }, { sparse: true });
|
|||
triggerDeliverySchema.index({ user: 1, actorActionAdmittedAt: 1 }, { sparse: true });
|
||||
triggerDeliverySchema.index({ stagingRecoveryAt: 1 }, { sparse: true });
|
||||
triggerDeliverySchema.index({ laneCleanupPendingAt: 1 }, { sparse: true });
|
||||
triggerDeliverySchema.index({ backgroundToolResultDeletionPendingAt: 1 }, { sparse: true });
|
||||
// Only successful rows receive expiresAt. Dead letters remain available until
|
||||
// an operator explicitly requeues or removes them.
|
||||
triggerDeliverySchema.index({ expiresAt: 1 }, { expireAfterSeconds: 0 });
|
||||
|
|
|
|||
|
|
@ -15,7 +15,10 @@ export type AgentTriggerDeliveryStatus =
|
|||
export const AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 = 'event_actor_detached_action_v1';
|
||||
export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 =
|
||||
'background_tool_completion_v1';
|
||||
export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 =
|
||||
'background_tool_completion_receipt_v2';
|
||||
export const AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1 = 'agent_queued_turn_v1';
|
||||
export const AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS: number = 64 * 1024;
|
||||
export type AgentTriggerDeliveryOutcome = 'succeeded' | 'retry' | 'dead';
|
||||
|
||||
export interface AgentTriggerHandlingState {
|
||||
|
|
@ -77,6 +80,20 @@ export interface AgentTriggerDeliveryFailure {
|
|||
status?: number;
|
||||
}
|
||||
|
||||
/** Private terminal result produced before a background-completion delivery is
|
||||
* resolved. It is independent of the parent message projection because a fast
|
||||
* task can settle before that response row exists. */
|
||||
export interface AgentBackgroundToolResultReceipt {
|
||||
status: 'completed' | 'error' | 'cancelled';
|
||||
output: string;
|
||||
settledAt: Date;
|
||||
resultClaim?: {
|
||||
kind: 'wakeup';
|
||||
claimId: string;
|
||||
claimedAt: Date;
|
||||
};
|
||||
}
|
||||
|
||||
export interface AgentTriggerDeliveryHistoryEntry {
|
||||
attempt: number;
|
||||
outcome: AgentTriggerDeliveryOutcome;
|
||||
|
|
@ -108,6 +125,11 @@ export interface IAgentTriggerDelivery {
|
|||
capabilityClaimToken?: string;
|
||||
/** Durable liveness evidence for process-owned capability work. */
|
||||
producerLeaseUntil?: Date;
|
||||
/** Durable source of truth for a background completion. */
|
||||
backgroundToolResult?: AgentBackgroundToolResultReceipt;
|
||||
/** Conversation-deletion fence preventing a late producer from restoring private output. */
|
||||
backgroundToolResultErasedAt?: Date;
|
||||
backgroundToolResultDeletionPendingAt?: Date;
|
||||
attempts: number;
|
||||
availableAt: Date;
|
||||
envelopeBytes?: number;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue