mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-19 22:43:38 +00:00
🪇 perf: Coalesce Ready Background Completions (#16650)
* 🪇 perf: Coalesce Ready Background Completions
* fix: Recover Receipt Batches and Gate Rolling Deploys
* fix: Fence Background Batch Handoffs and Cleanup Epochs
* fix: Bind Resumed Cleanup to Its Observed Batch
* fix: Preserve Background Results Through Recovery Handoffs
* fix: Recover Unconfirmed Manual Background Claims
* fix: Fence Automatic Recovery of Manual Handoffs
* fix: Preserve Background Reconciliation and Recovery Retention
* fix: Preserve Replayed Claims and Finish Follower Proofs
* fix: Recover Stale Background Projection Epochs
---------
Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
parent
35a2bee409
commit
f88f3fb2fe
35 changed files with 4610 additions and 196 deletions
|
|
@ -709,6 +709,8 @@ if (cluster.isMaster) {
|
|||
address: server.address(),
|
||||
completionResultBatchSize:
|
||||
baseAppConfig?.endpoints?.agents?.backgroundTasks?.completionResultBatchSize,
|
||||
completionReceiptBatching:
|
||||
baseAppConfig?.endpoints?.agents?.backgroundTasks?.completionReceiptBatching,
|
||||
idlePolling: baseAppConfig?.endpoints?.agents?.eventDriven?.idlePolling,
|
||||
});
|
||||
} catch (initErr) {
|
||||
|
|
|
|||
|
|
@ -508,6 +508,8 @@ const startServer = async () => {
|
|||
address: server.address(),
|
||||
completionResultBatchSize:
|
||||
appConfig?.endpoints?.agents?.backgroundTasks?.completionResultBatchSize,
|
||||
completionReceiptBatching:
|
||||
appConfig?.endpoints?.agents?.backgroundTasks?.completionReceiptBatching,
|
||||
idlePolling: appConfig?.endpoints?.agents?.eventDriven?.idlePolling,
|
||||
});
|
||||
const scheduleEngineArmed = (await initializeScheduleEngine()) != null;
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ const {
|
|||
createSubagentCompletionWakeupResolver,
|
||||
SUBAGENT_COMPLETION_SOURCE,
|
||||
createBackgroundToolCompletionWakeupResolver,
|
||||
createBackgroundToolDeadClaimRecovery,
|
||||
BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
createAgentQueuedTurnLifecycle,
|
||||
AGENT_QUEUED_TURN_SOURCE,
|
||||
|
|
@ -30,6 +31,22 @@ const backgroundToolCompletionAdapter = createBackgroundToolCompletionWakeupReso
|
|||
methods,
|
||||
getGenerationJob: (conversationId) => GenerationJobManager.getJob(conversationId),
|
||||
getResultBatchSize: () => service.getBackgroundCompletionResultBatchSize(),
|
||||
getGenerationAdmissionEvidence,
|
||||
recoverDeadClaim: createBackgroundToolDeadClaimRecovery(
|
||||
(...args) => service.retire(...args),
|
||||
methods.releaseBackgroundToolResultClaims,
|
||||
(conversationId) => GenerationJobManager.getJob(conversationId),
|
||||
({ userId, conversationId, claimId }) =>
|
||||
GenerationJobManager.fenceGenerationClaimForRecovery(
|
||||
userId,
|
||||
claimId,
|
||||
conversationId,
|
||||
conversationId,
|
||||
),
|
||||
methods.releaseAgentBackgroundToolResultClaims,
|
||||
methods,
|
||||
getGenerationAdmissionEvidence,
|
||||
),
|
||||
getWaitMaxIntervalMs: () => service.getCompletionWaitMaxIntervalMs(),
|
||||
});
|
||||
const eventActorAdapter = createAgentEventContinueResolver({
|
||||
|
|
@ -95,6 +112,7 @@ module.exports = {
|
|||
persistAgentBackgroundToolResult: service.persistBackgroundToolResult,
|
||||
expediteCompletionWakeups: service.expediteCompletionWakeups,
|
||||
getAgentBackgroundToolResultClaim: service.getBackgroundToolResultClaim,
|
||||
getBackgroundCompletionReceiptBatching: service.getBackgroundCompletionReceiptBatching,
|
||||
releaseAgentBackgroundToolResultClaims: service.releaseBackgroundToolResultClaims,
|
||||
drainAgentTriggerDeliveriesForUser: service.drainUser,
|
||||
prepareAgentTriggerUserPurge: service.prepareUserPurge,
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ const mockQueuedTurnLifecycle = {
|
|||
|
||||
jest.mock('@librechat/api', () => ({
|
||||
createCheckpointDeletionReclaimer: jest.fn((getJobs) => () => getJobs('owner', 'tenant')),
|
||||
createBackgroundToolDeadClaimRecovery: jest.fn(() => jest.fn()),
|
||||
createAgentTriggerService: (...args) => mockCreateAgentTriggerService(...args),
|
||||
createAgentContinuationResolver: jest.fn(() => jest.fn()),
|
||||
createAgentEventContinueResolver: jest.fn(() => jest.fn()),
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
const {
|
||||
createBackgroundToolCompletionWakeupHandler,
|
||||
GenerationJobManager,
|
||||
createBackgroundToolDeadClaimRecovery,
|
||||
createPendingBackgroundCompletions,
|
||||
createBackgroundToolResultHandler,
|
||||
|
|
@ -7,12 +8,15 @@ const {
|
|||
} = require('@librechat/api');
|
||||
const {
|
||||
listPendingAgentBackgroundToolCompletions,
|
||||
getAgentBackgroundToolResultBatch,
|
||||
confirmAgentBackgroundToolResultBatch,
|
||||
listUndeliveredAgentTriggerTaskIds,
|
||||
} = require('~/models');
|
||||
const {
|
||||
enqueueAgentTrigger,
|
||||
persistAgentBackgroundToolResult,
|
||||
getAgentBackgroundToolResultClaim,
|
||||
getBackgroundCompletionReceiptBatching,
|
||||
releaseAgentBackgroundToolResultClaims,
|
||||
renewAgentTriggerProducerLease,
|
||||
retireAgentTrigger,
|
||||
|
|
@ -26,6 +30,7 @@ const preregisterBackgroundToolCompletion = createBackgroundToolCompletionWakeup
|
|||
(deliveryKey, sourceId, result) =>
|
||||
persistAgentBackgroundToolResult({ deliveryKey, sourceId, result }),
|
||||
(deliveryKey) => expediteCompletionWakeups({ deliveryKeys: [deliveryKey] }),
|
||||
getBackgroundCompletionReceiptBatching,
|
||||
);
|
||||
|
||||
const pendingBackgroundToolCompletions = createPendingBackgroundCompletions({
|
||||
|
|
@ -52,6 +57,8 @@ function createDeadBackgroundToolClaimRecovery(
|
|||
getGenerationJob,
|
||||
fenceGenerationClaim,
|
||||
releaseAgentBackgroundToolResultClaims,
|
||||
{ getAgentBackgroundToolResultBatch, confirmAgentBackgroundToolResultBatch },
|
||||
(...args) => GenerationJobManager.getGenerationAdmissionEvidence(...args),
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -53,6 +53,42 @@ async function waitForSubagentTaskToSettle(
|
|||
throw new Error('Timed out waiting for the detached subagent task.');
|
||||
}
|
||||
|
||||
describe('manual reconciliation restoration', () => {
|
||||
it('upgrades only the confirmed local owner and preserves the marker on replay', () => {
|
||||
const registry = new BackgroundTaskRegistryClass();
|
||||
const created = registry.create({
|
||||
userId: 'marker-user',
|
||||
conversationId: 'marker-convo',
|
||||
toolCallId: 'marker-call',
|
||||
toolName: 'tool',
|
||||
});
|
||||
if ('atCapacity' in created) throw new Error('Unexpected capacity');
|
||||
registry.complete('marker-user', 'marker-convo', created.task.id, { content: 'done' });
|
||||
const claim = { kind: 'manual' as const, claimId: 'poll' };
|
||||
expect(registry.claimResult('marker-user', 'marker-convo', created.task.id, claim)).toBe(
|
||||
'acquired',
|
||||
);
|
||||
expect(
|
||||
registry.claimResult('marker-user', 'marker-convo', created.task.id, {
|
||||
...claim,
|
||||
claimId: 'foreign',
|
||||
receiptReconciled: true,
|
||||
}),
|
||||
).toBe('claimed');
|
||||
expect(created.task.resultClaim?.receiptReconciled).toBeUndefined();
|
||||
expect(
|
||||
registry.claimResult('marker-user', 'marker-convo', created.task.id, {
|
||||
...claim,
|
||||
receiptReconciled: true,
|
||||
}),
|
||||
).toBe('replay');
|
||||
expect(registry.claimResult('marker-user', 'marker-convo', created.task.id, claim)).toBe(
|
||||
'replay',
|
||||
);
|
||||
expect(created.task.resultClaim?.receiptReconciled).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isBackgroundEligibleToolName', () => {
|
||||
it('excludes direct-path, host-special, and machinery tools', () => {
|
||||
for (const name of [
|
||||
|
|
|
|||
|
|
@ -668,6 +668,7 @@ export interface BackgroundTask {
|
|||
claimId: string;
|
||||
claimedAt: number;
|
||||
generationId?: string;
|
||||
receiptReconciled?: true;
|
||||
};
|
||||
/** The declared tool may return a process-local live artifact. A terminal
|
||||
* same-generation poll may therefore deliver from the local claim after it
|
||||
|
|
@ -1823,7 +1824,12 @@ export class BackgroundTaskRegistryClass {
|
|||
userId: string,
|
||||
conversationId: string,
|
||||
taskId: string,
|
||||
claim: { kind: 'manual' | 'wakeup'; claimId: string; generationId?: string },
|
||||
claim: {
|
||||
kind: 'manual' | 'wakeup';
|
||||
claimId: string;
|
||||
generationId?: string;
|
||||
receiptReconciled?: true;
|
||||
},
|
||||
): 'acquired' | 'replay' | 'claimed' | 'not_ready' {
|
||||
const task = this.get(userId, conversationId, taskId);
|
||||
if (task == null || task.status === 'running') {
|
||||
|
|
@ -1834,9 +1840,14 @@ export class BackgroundTaskRegistryClass {
|
|||
task.updatedAt = Date.now();
|
||||
return 'acquired';
|
||||
}
|
||||
return task.resultClaim.kind === claim.kind && task.resultClaim.claimId === claim.claimId
|
||||
? 'replay'
|
||||
: 'claimed';
|
||||
if (task.resultClaim.kind !== claim.kind || task.resultClaim.claimId !== claim.claimId) {
|
||||
return 'claimed';
|
||||
}
|
||||
if (claim.receiptReconciled === true && task.resultClaim.receiptReconciled !== true) {
|
||||
task.resultClaim.receiptReconciled = true;
|
||||
task.updatedAt = Date.now();
|
||||
}
|
||||
return 'replay';
|
||||
}
|
||||
|
||||
releaseResultClaim(
|
||||
|
|
@ -2554,6 +2565,7 @@ export async function runCheckBackgroundTask(params: {
|
|||
conversationId,
|
||||
messageId: task.messageId,
|
||||
claimId: existingClaim.claimId,
|
||||
...(existingClaim.batchId != null && { batchId: existingClaim.batchId }),
|
||||
...(existingClaim.kind === 'manual'
|
||||
? {
|
||||
kind: 'manual' as const,
|
||||
|
|
@ -2674,6 +2686,7 @@ export async function runCheckBackgroundTask(params: {
|
|||
taskId,
|
||||
{
|
||||
kind: 'manual',
|
||||
receiptReconciled: true,
|
||||
claimId: invocationId,
|
||||
...(params.generationId == null ? {} : { generationId: params.generationId }),
|
||||
},
|
||||
|
|
@ -2744,6 +2757,7 @@ export async function runCheckBackgroundTask(params: {
|
|||
taskId,
|
||||
{
|
||||
kind: 'manual',
|
||||
receiptReconciled: true,
|
||||
claimId: invocationId,
|
||||
...(params.generationId == null ? {} : { generationId: params.generationId }),
|
||||
},
|
||||
|
|
@ -2854,6 +2868,7 @@ export async function runCheckBackgroundTask(params: {
|
|||
conversationId,
|
||||
messageId: claimedResult.messageId,
|
||||
claimId: existingClaim.claimId,
|
||||
...(existingClaim.batchId != null && { batchId: existingClaim.batchId }),
|
||||
...(existingClaim.kind === 'manual'
|
||||
? {
|
||||
kind: 'manual' as const,
|
||||
|
|
@ -3242,6 +3257,7 @@ export function getBackgroundCodeDelivery(params: {
|
|||
kind: task.resultClaim.kind,
|
||||
claimId: task.resultClaim.claimId,
|
||||
claimedAt: new Date(task.resultClaim.claimedAt),
|
||||
...(task.resultClaim.receiptReconciled === true && { receiptReconciled: true }),
|
||||
...(task.resultClaim.generationId == null
|
||||
? {}
|
||||
: { generationId: task.resultClaim.generationId }),
|
||||
|
|
|
|||
|
|
@ -4,7 +4,11 @@ import { BACKGROUND_TOOL_COMPLETION_SOURCE } from './backgroundCompletionWakeup'
|
|||
/** Reconciles manual polling with the independent automatic-delivery receipt,
|
||||
* including polls reconstructed after the process-local registry was lost. */
|
||||
export async function claimBackgroundToolResult(
|
||||
methods: Pick<MessageMethods, 'claimBackgroundToolResults' | 'releaseBackgroundToolResultClaims'>,
|
||||
methods: Pick<
|
||||
MessageMethods,
|
||||
'claimBackgroundToolResults' | 'releaseBackgroundToolResultClaims'
|
||||
> &
|
||||
Partial<Pick<MessageMethods, 'confirmBackgroundToolResultClaim'>>,
|
||||
getReceiptClaim: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'],
|
||||
input: Parameters<MessageMethods['claimBackgroundToolResults']>[0],
|
||||
): ReturnType<MessageMethods['claimBackgroundToolResults']> {
|
||||
|
|
@ -30,7 +34,7 @@ export async function claimBackgroundToolResult(
|
|||
if (messageId == null) {
|
||||
throw new Error('The background result claim has no parent message identity');
|
||||
}
|
||||
const release = async () => {
|
||||
const release = async (allowReconciled = false) => {
|
||||
const released = await methods.releaseBackgroundToolResultClaims({
|
||||
userId: input.userId,
|
||||
conversationId: input.conversationId,
|
||||
|
|
@ -38,8 +42,10 @@ export async function claimBackgroundToolResult(
|
|||
taskIds: messageClaim.results.map((result) => result.taskId),
|
||||
kind: input.kind,
|
||||
claimId: input.claimId,
|
||||
...(input.batchId != null && { batchId: input.batchId }),
|
||||
...(input.kind === 'manual' && { onlyIfUnreconciled: true }),
|
||||
});
|
||||
if (!released) {
|
||||
if (!released && !allowReconciled) {
|
||||
throw new Error('The background result claim could not be released');
|
||||
}
|
||||
};
|
||||
|
|
@ -51,6 +57,26 @@ export async function claimBackgroundToolResult(
|
|||
throw error;
|
||||
}
|
||||
if (receiptClaim == null || receiptClaim.claimId === input.claimId) {
|
||||
if (input.kind === 'manual' && methods.confirmBackgroundToolResultClaim != null) {
|
||||
try {
|
||||
if (
|
||||
!(await methods.confirmBackgroundToolResultClaim({
|
||||
userId: input.userId,
|
||||
conversationId: input.conversationId,
|
||||
messageId,
|
||||
taskId: input.taskId,
|
||||
claimId: input.claimId,
|
||||
}))
|
||||
) {
|
||||
throw new Error('The reconciled background result claim was superseded');
|
||||
}
|
||||
} catch (error) {
|
||||
// This CAS removes only speculative ownership. A committed or unknown
|
||||
// handoff remains owned for generation-fenced recovery.
|
||||
await release(true);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return messageClaim;
|
||||
}
|
||||
await release();
|
||||
|
|
|
|||
|
|
@ -65,6 +65,9 @@ export interface BackgroundToolDeadClaimRecoveryInput {
|
|||
conversationId: string;
|
||||
messageId: string;
|
||||
claimId: string;
|
||||
batchId?: string;
|
||||
/** Automatic repair cannot reopen a committed manual handoff. */
|
||||
onlyIfUnreconciled?: true;
|
||||
/** Omitted for the legacy automatic-wakeup recovery path. */
|
||||
kind?: 'manual' | 'wakeup';
|
||||
/** Required to prove that a manual claim's owning generation is no longer active. */
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,4 +1,7 @@
|
|||
import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 } from '@librechat/data-schemas';
|
||||
import {
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
} from '@librechat/data-schemas';
|
||||
import type { AgentTriggerProducerLeaseStatus } from '@librechat/data-schemas';
|
||||
import type { CodeApprovalMode } from 'librechat-data-provider';
|
||||
import type { EnqueueBackgroundToolCompletion } from './backgroundCompletionWakeup';
|
||||
|
|
@ -113,6 +116,37 @@ describe('background tool completion wakeups', () => {
|
|||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
it('admits v3 receipts only when the deployment rollout gate is enabled', async () => {
|
||||
const enqueue = jest.fn<
|
||||
ReturnType<EnqueueBackgroundToolCompletion>,
|
||||
Parameters<EnqueueBackgroundToolCompletion>
|
||||
>(async () => ({ deliveryKey: 'key' }));
|
||||
let enabled = false;
|
||||
const notify = createBackgroundToolCompletionWakeupHandler(
|
||||
enqueue,
|
||||
async () => true,
|
||||
async () => true,
|
||||
undefined,
|
||||
undefined,
|
||||
() => enabled,
|
||||
);
|
||||
await notify(registration());
|
||||
expect(enqueue).toHaveBeenLastCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
}),
|
||||
);
|
||||
enabled = true;
|
||||
await notify(registration({ taskId: 'task-2' }));
|
||||
expect(enqueue).toHaveBeenLastCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('expedites its own delivery when a result it can consume appears', async () => {
|
||||
const expedite = jest.fn();
|
||||
const notify = createBackgroundToolCompletionWakeupHandler(
|
||||
|
|
@ -265,6 +299,7 @@ describe('background tool completion wakeups', () => {
|
|||
userId: 'user-1',
|
||||
conversationId: 'conversation-1',
|
||||
messageId: 'response-1',
|
||||
allowMissingMessage: true,
|
||||
kind: 'wakeup',
|
||||
claimId: 'batch-root-delivery',
|
||||
});
|
||||
|
|
@ -274,6 +309,7 @@ describe('background tool completion wakeups', () => {
|
|||
conversationId: 'conversation-1',
|
||||
parentMessageId: 'response-1',
|
||||
claimId: 'batch-root-delivery',
|
||||
recoveryFenced: true,
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -1034,3 +1070,204 @@ describe('pending background completions', () => {
|
|||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('capability-gated receipt batch resolution', () => {
|
||||
function batchMethods() {
|
||||
const { methods } = resolverMethods();
|
||||
return {
|
||||
...methods,
|
||||
claimAgentBackgroundToolResultBatch: jest.fn(async () => ({
|
||||
status: 'acquired' as const,
|
||||
batchId: 'batch-1',
|
||||
results: [
|
||||
{
|
||||
taskId: 'task-1',
|
||||
toolCallId: 'call-1',
|
||||
toolName: 'tool',
|
||||
status: 'completed' as const,
|
||||
output: 'one',
|
||||
},
|
||||
{
|
||||
taskId: 'task-2',
|
||||
toolCallId: 'call-2',
|
||||
toolName: 'tool',
|
||||
status: 'completed' as const,
|
||||
output: 'two',
|
||||
},
|
||||
],
|
||||
})),
|
||||
beginAgentBackgroundToolResultBatchDispatch: jest.fn(async () => true),
|
||||
confirmAgentBackgroundToolResultBatch: jest.fn(async () => true),
|
||||
};
|
||||
}
|
||||
|
||||
it('does not probe batch storage for rollout-default v2 deliveries', async () => {
|
||||
const methods = batchMethods();
|
||||
const resolve = createBackgroundToolCompletionWakeupResolver({
|
||||
methods: methods as never,
|
||||
getGenerationJob: async () => null,
|
||||
});
|
||||
await resolve(await envelope(), {
|
||||
idempotencyKey: 'delivery-1',
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
});
|
||||
expect(methods.claimAgentBackgroundToolResultBatch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('reconciles every member before dispatch and confirms only after admission', async () => {
|
||||
const methods = batchMethods();
|
||||
const resolve = createBackgroundToolCompletionWakeupResolver({
|
||||
methods: methods as never,
|
||||
getGenerationJob: async () => null,
|
||||
});
|
||||
const prepared = await resolve(await envelope(), { idempotencyKey: 'delivery-1' });
|
||||
expect(methods.claimBackgroundToolResults).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({ taskId: 'task-1' }),
|
||||
);
|
||||
expect(methods.claimBackgroundToolResults).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({ taskId: 'task-2' }),
|
||||
);
|
||||
expect(methods.confirmAgentBackgroundToolResultBatch).not.toHaveBeenCalled();
|
||||
expect(methods.claimAgentBackgroundToolResultBatch).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
tenantId: 'tenant-1',
|
||||
limit: 8,
|
||||
maxMetadataChars: BACKGROUND_TOOL_WAKEUP_INPUT_MAX_CHARS - 256,
|
||||
}),
|
||||
);
|
||||
if (prepared?.status !== 'ready') throw new Error('Expected ready');
|
||||
expect(prepared.input).toContain('task-2');
|
||||
await prepared.settleOnAdmission?.({
|
||||
mode: 'continue',
|
||||
status: 'started',
|
||||
conversationId: 'conversation-1',
|
||||
});
|
||||
expect(methods.confirmAgentBackgroundToolResultBatch).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('defers followers until the root has confirmed admission', async () => {
|
||||
const methods = batchMethods();
|
||||
methods.claimAgentBackgroundToolResultBatch
|
||||
.mockResolvedValueOnce({
|
||||
status: 'claimed',
|
||||
claimId: 'root',
|
||||
ownerStatus: 'pending',
|
||||
} as never)
|
||||
.mockResolvedValueOnce({
|
||||
status: 'claimed',
|
||||
claimId: 'root',
|
||||
ownerStatus: 'applied',
|
||||
} as never);
|
||||
const resolve = createBackgroundToolCompletionWakeupResolver({
|
||||
methods: methods as never,
|
||||
getGenerationJob: async () => null,
|
||||
});
|
||||
await expect(resolve(await envelope(), { idempotencyKey: 'follower' })).rejects.toMatchObject({
|
||||
code: 'BACKGROUND_TOOL_BATCH_PENDING',
|
||||
deferWithoutAttempt: true,
|
||||
});
|
||||
await expect(resolve(await envelope(), { idempotencyKey: 'follower' })).resolves.toEqual({
|
||||
status: 'settled',
|
||||
});
|
||||
expect(methods.claimBackgroundToolResults).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('uses fenced dead-owner recovery before retrying a follower as its own root', async () => {
|
||||
const methods = batchMethods();
|
||||
const recoverDeadClaim = jest.fn(async () => true);
|
||||
methods.claimAgentBackgroundToolResultBatch.mockResolvedValueOnce({
|
||||
status: 'claimed',
|
||||
claimId: 'root',
|
||||
ownerStatus: 'recoverable',
|
||||
} as never);
|
||||
const resolve = createBackgroundToolCompletionWakeupResolver({
|
||||
methods: methods as never,
|
||||
getGenerationJob: async () => null,
|
||||
recoverDeadClaim,
|
||||
});
|
||||
await expect(resolve(await envelope(), { idempotencyKey: 'follower' })).rejects.toMatchObject({
|
||||
code: 'BACKGROUND_TOOL_BATCH_PENDING',
|
||||
});
|
||||
expect(recoverDeadClaim).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ claimId: 'root', messageId: 'response-1' }),
|
||||
);
|
||||
expect((await resolve(await envelope(), { idempotencyKey: 'follower' }))?.status).toBe('ready');
|
||||
});
|
||||
|
||||
it('releases the entire frozen batch when a sibling manual poll wins', async () => {
|
||||
const methods = batchMethods();
|
||||
methods.claimBackgroundToolResults
|
||||
.mockResolvedValueOnce({ status: 'acquired', results: [] })
|
||||
.mockResolvedValueOnce({
|
||||
status: 'claimed',
|
||||
claim: { kind: 'manual', claimId: 'poll' },
|
||||
} as never);
|
||||
const resolve = createBackgroundToolCompletionWakeupResolver({
|
||||
methods: methods as never,
|
||||
getGenerationJob: async () => null,
|
||||
});
|
||||
await expect(resolve(await envelope(), { idempotencyKey: 'root' })).rejects.toMatchObject({
|
||||
code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
|
||||
});
|
||||
expect(methods.releaseAgentBackgroundToolResultClaims).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ claimId: 'root' }),
|
||||
);
|
||||
expect(methods.confirmAgentBackgroundToolResultBatch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('releases receipt ownership even when no message projection has landed', async () => {
|
||||
const methods = batchMethods();
|
||||
methods.claimBackgroundToolResults.mockResolvedValue({ status: 'not_found' } as never);
|
||||
const resolve = createBackgroundToolCompletionWakeupResolver({
|
||||
methods: methods as never,
|
||||
getGenerationJob: async () => null,
|
||||
});
|
||||
const prepared = await resolve(await envelope(), { idempotencyKey: 'root' });
|
||||
if (prepared?.status !== 'ready') throw new Error('Expected ready');
|
||||
await prepared.releaseOnDefiniteFailure?.();
|
||||
expect(methods.releaseBackgroundToolResultClaims).not.toHaveBeenCalled();
|
||||
expect(methods.releaseAgentBackgroundToolResultClaims).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('retains ownership when admission confirmation fails after dispatch', async () => {
|
||||
const methods = batchMethods();
|
||||
methods.confirmAgentBackgroundToolResultBatch.mockRejectedValueOnce(new Error('lost reply'));
|
||||
const resolve = createBackgroundToolCompletionWakeupResolver({
|
||||
methods: methods as never,
|
||||
getGenerationJob: async () => null,
|
||||
});
|
||||
const prepared = await resolve(await envelope(), { idempotencyKey: 'root' });
|
||||
if (prepared?.status !== 'ready') throw new Error('Expected ready');
|
||||
await expect(
|
||||
prepared.settleOnAdmission?.({
|
||||
mode: 'continue',
|
||||
status: 'started',
|
||||
conversationId: 'conversation-1',
|
||||
}),
|
||||
).rejects.toThrow('lost reply');
|
||||
expect(methods.releaseAgentBackgroundToolResultClaims).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('still detects a lost producer while a v3 receipt is not ready', async () => {
|
||||
const methods = batchMethods();
|
||||
methods.claimBackgroundToolResults.mockResolvedValueOnce({ status: 'not_ready', results: [] });
|
||||
methods.claimAgentBackgroundToolResultBatch.mockResolvedValueOnce({
|
||||
status: 'not_ready',
|
||||
waitingForResult: true,
|
||||
} as never);
|
||||
methods.getAgentTriggerDeliveryProducerLease.mockResolvedValueOnce({
|
||||
status: 'expired',
|
||||
leaseUntil: new Date(0),
|
||||
});
|
||||
const resolve = createBackgroundToolCompletionWakeupResolver({
|
||||
methods: methods as never,
|
||||
getGenerationJob: async () => null,
|
||||
});
|
||||
await expect(resolve(await envelope(), { idempotencyKey: 'root' })).rejects.toMatchObject({
|
||||
code: 'BACKGROUND_TOOL_PRODUCER_LOST',
|
||||
retryable: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,10 @@
|
|||
import { randomUUID } from 'node:crypto';
|
||||
import { backgroundResultMetadata, isEphemeralAgentId } from 'librechat-data-provider';
|
||||
import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 } from '@librechat/data-schemas';
|
||||
import {
|
||||
AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
} from '@librechat/data-schemas';
|
||||
import type {
|
||||
AgentTriggerProducerLeaseStatus,
|
||||
AgentTriggerDeliveryMethods,
|
||||
|
|
@ -80,6 +84,10 @@ type WakeupMethods = Pick<ConversationMethods, 'getConvo'> &
|
|||
output: string;
|
||||
settledAt: Date;
|
||||
} | null>;
|
||||
persistAgentBackgroundToolResult?: AgentTriggerDeliveryMethods['persistAgentBackgroundToolResult'];
|
||||
beginAgentBackgroundToolResultBatchDispatch?: AgentTriggerDeliveryMethods['beginAgentBackgroundToolResultBatchDispatch'];
|
||||
claimAgentBackgroundToolResultBatch?: AgentTriggerDeliveryMethods['claimAgentBackgroundToolResultBatch'];
|
||||
confirmAgentBackgroundToolResultBatch?: AgentTriggerDeliveryMethods['confirmAgentBackgroundToolResultBatch'];
|
||||
claimAgentBackgroundToolResults?: AgentTriggerDeliveryMethods['claimAgentBackgroundToolResults'];
|
||||
getAgentBackgroundToolResultClaim?: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'];
|
||||
releaseAgentBackgroundToolResultClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];
|
||||
|
|
@ -98,6 +106,13 @@ export interface BackgroundToolCompletionWakeupResolverDeps {
|
|||
methods: WakeupMethods;
|
||||
getGenerationJob: (conversationId: string) => Promise<GenerationState | null>;
|
||||
getResultBatchSize?: () => number | undefined;
|
||||
recoverDeadClaim?: BackgroundToolDeadClaimRecovery;
|
||||
getGenerationAdmissionEvidence?: (
|
||||
userId: string,
|
||||
clientRequestId: string,
|
||||
streamId: string,
|
||||
conversationId?: string,
|
||||
) => Promise<{ generationId: string; generationCreatedAt: number } | null>;
|
||||
/** Longest a waiting delivery re-checks readiness; the backoff default otherwise. */
|
||||
getWaitMaxIntervalMs?: () => number | undefined;
|
||||
}
|
||||
|
|
@ -265,12 +280,14 @@ export function createBackgroundToolCompletionWakeupResolver({
|
|||
getGenerationJob,
|
||||
getResultBatchSize,
|
||||
getWaitMaxIntervalMs,
|
||||
recoverDeadClaim,
|
||||
getGenerationAdmissionEvidence,
|
||||
}: BackgroundToolCompletionWakeupResolverDeps): NonNullable<
|
||||
AgentTriggerExecutionHostDeps['prepareContinue']
|
||||
> {
|
||||
const waitingRetry = (receivedAt: number): string =>
|
||||
waitingRetryAfter(receivedAt, Date.now(), getWaitMaxIntervalMs?.() ?? WAITING_RETRY_CAP_MS);
|
||||
return async (
|
||||
const resolve = async (
|
||||
envelope: AgentContinueTriggerEnvelope,
|
||||
context: AgentTriggerDispatchContext,
|
||||
): Promise<AgentTriggerContinuePreparation | undefined> => {
|
||||
|
|
@ -332,6 +349,313 @@ export function createBackgroundToolCompletionWakeupResolver({
|
|||
status: 404,
|
||||
});
|
||||
}
|
||||
const batchScope = {
|
||||
deliveryKey: context.idempotencyKey,
|
||||
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
userId,
|
||||
...(envelope.principal.tenantId != null && { tenantId: envelope.principal.tenantId }),
|
||||
conversationId: envelope.target.conversationId,
|
||||
parentMessageId: envelope.target.parentMessageId,
|
||||
agentId: envelope.target.agentId,
|
||||
...(context.deliveryClaimToken != null && { deliveryClaimToken: context.deliveryClaimToken }),
|
||||
};
|
||||
const batch =
|
||||
context.requiredWorkerCapability != null &&
|
||||
context.requiredWorkerCapability !==
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3
|
||||
? undefined
|
||||
: await methods.claimAgentBackgroundToolResultBatch?.({
|
||||
...batchScope,
|
||||
limit: getResultBatchSize?.() ?? 8,
|
||||
maxMetadataChars: BACKGROUND_TOOL_WAKEUP_INPUT_MAX_CHARS - 256,
|
||||
});
|
||||
if (batch?.status === 'claimed') {
|
||||
if (batch.ownerStatus === 'applied') return { status: 'settled' };
|
||||
if (batch.ownerStatus === 'recoverable') {
|
||||
await recoverDeadClaim?.({
|
||||
userId,
|
||||
conversationId: envelope.target.conversationId,
|
||||
messageId: envelope.target.parentMessageId,
|
||||
claimId: batch.claimId,
|
||||
batchId: batch.batchId,
|
||||
});
|
||||
}
|
||||
throw executionError('The background result is owned by another completion delivery.', {
|
||||
code: 'BACKGROUND_TOOL_BATCH_PENDING',
|
||||
retryable: true,
|
||||
deferWithoutAttempt: true,
|
||||
retryAfter: '1',
|
||||
});
|
||||
}
|
||||
if (batch?.status === 'not_ready') {
|
||||
if (batch.waitingForResult === true) {
|
||||
// A producer can fail its receipt write while its terminal message
|
||||
// projection succeeds. Reconstruct from that already-durable result.
|
||||
const projection = await methods.claimBackgroundToolResults({
|
||||
userId,
|
||||
conversationId: envelope.target.conversationId,
|
||||
messageId: envelope.target.parentMessageId,
|
||||
taskId: registration.taskId,
|
||||
agentId: envelope.target.agentId,
|
||||
kind: 'wakeup',
|
||||
claimId: context.idempotencyKey,
|
||||
limit: 1,
|
||||
});
|
||||
if (
|
||||
projection.status === 'claimed' &&
|
||||
projection.claim?.kind === 'manual' &&
|
||||
projection.claim.receiptReconciled === true
|
||||
)
|
||||
return { status: 'settled' };
|
||||
if (projection.status === 'acquired') {
|
||||
const result = projection.results.find((result) => result.taskId === registration.taskId);
|
||||
const releaseProjection = async (): Promise<void> => {
|
||||
if (
|
||||
!(await methods.releaseBackgroundToolResultClaims({
|
||||
userId,
|
||||
conversationId: envelope.target.conversationId,
|
||||
messageId: envelope.target.parentMessageId,
|
||||
taskIds: [registration.taskId],
|
||||
kind: 'wakeup',
|
||||
claimId: context.idempotencyKey,
|
||||
allowMissingMessage: true,
|
||||
}))
|
||||
) {
|
||||
throw new Error('Background projection reconstruction release was not confirmed');
|
||||
}
|
||||
};
|
||||
let restored = false;
|
||||
try {
|
||||
if (result != null)
|
||||
restored =
|
||||
(await methods.persistAgentBackgroundToolResult?.({
|
||||
deliveryKey: context.idempotencyKey,
|
||||
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
result: {
|
||||
status: result.status,
|
||||
output: truncateMiddle(
|
||||
result.output,
|
||||
AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
|
||||
),
|
||||
settledAt: result.settledAt ?? new Date(envelope.event.occurredAt),
|
||||
},
|
||||
})) === true;
|
||||
} finally {
|
||||
await releaseProjection();
|
||||
}
|
||||
if (restored) return resolve(envelope, context);
|
||||
throw executionError('The background result receipt is being reconstructed.', {
|
||||
code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
|
||||
retryable: true,
|
||||
deferWithoutAttempt: true,
|
||||
retryAfter: '1',
|
||||
});
|
||||
}
|
||||
if (
|
||||
projection.status === 'claimed' &&
|
||||
projection.claim?.kind === 'manual' &&
|
||||
projection.claim.generationId != null
|
||||
) {
|
||||
await recoverDeadClaim?.({
|
||||
userId,
|
||||
conversationId: envelope.target.conversationId,
|
||||
messageId: envelope.target.parentMessageId,
|
||||
claimId: projection.claim.claimId,
|
||||
kind: 'manual',
|
||||
generationId: projection.claim.generationId,
|
||||
onlyIfUnreconciled: true,
|
||||
});
|
||||
}
|
||||
if (projection.status === 'claimed')
|
||||
throw executionError('The background projection is being reconciled.', {
|
||||
code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
|
||||
retryable: true,
|
||||
deferWithoutAttempt: true,
|
||||
retryAfter: '1',
|
||||
});
|
||||
|
||||
const producer = await methods.getAgentTriggerDeliveryProducerLease({
|
||||
deliveryKey: context.idempotencyKey,
|
||||
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
now: new Date(),
|
||||
});
|
||||
if (producer.status === 'expired') {
|
||||
throw executionError('The process-local background tool executor was lost.', {
|
||||
code: 'BACKGROUND_TOOL_PRODUCER_LOST',
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
throw executionError('The background result batch is not durable yet.', {
|
||||
code: 'BACKGROUND_TOOL_RESULT_NOT_READY',
|
||||
retryable: true,
|
||||
deferWithoutAttempt: true,
|
||||
retryAfter: waitingRetry(envelope.receivedAt),
|
||||
});
|
||||
}
|
||||
if (batch?.status === 'acquired') {
|
||||
const owned = { ...batchScope, batchId: batch.batchId };
|
||||
const dispatchId = randomUUID();
|
||||
let dispatchBegan = false;
|
||||
const release = async (): Promise<boolean> => {
|
||||
const admitted = await getGenerationAdmissionEvidence?.(
|
||||
userId,
|
||||
context.idempotencyKey,
|
||||
envelope.target.conversationId,
|
||||
envelope.target.conversationId,
|
||||
);
|
||||
if (admitted != null) {
|
||||
await methods.confirmAgentBackgroundToolResultBatch?.(owned);
|
||||
return false;
|
||||
}
|
||||
// Absence is not proof. The storage CAS permits only a first definite
|
||||
// failure or a never-dispatched plan, never a retry of an ambiguous POST.
|
||||
const released = await methods.releaseAgentBackgroundToolResultClaims?.({
|
||||
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
userId,
|
||||
conversationId: envelope.target.conversationId,
|
||||
parentMessageId: envelope.target.parentMessageId,
|
||||
claimId: context.idempotencyKey,
|
||||
batchId: batch.batchId,
|
||||
...(dispatchBegan && { dispatchId }),
|
||||
...(context.deliveryClaimToken != null && {
|
||||
deliveryClaimToken: context.deliveryClaimToken,
|
||||
}),
|
||||
});
|
||||
return released === true;
|
||||
};
|
||||
// A receipt may precede its projection. Reconcile every member, not just
|
||||
// the root, against a concurrent manual poll or late message persistence.
|
||||
const projections = await Promise.all(
|
||||
batch.results.map((result) =>
|
||||
methods.claimBackgroundToolResults({
|
||||
userId,
|
||||
conversationId: envelope.target.conversationId,
|
||||
messageId: envelope.target.parentMessageId,
|
||||
taskId: result.taskId,
|
||||
agentId: envelope.target.agentId,
|
||||
kind: 'wakeup',
|
||||
claimId: context.idempotencyKey,
|
||||
batchId: batch.batchId,
|
||||
limit: 1,
|
||||
}),
|
||||
),
|
||||
);
|
||||
if (
|
||||
projections.some(
|
||||
(projection) =>
|
||||
projection.status === 'claimed' || projection.status === 'outcome_unknown',
|
||||
)
|
||||
) {
|
||||
for (let index = 0; index < projections.length; index++) {
|
||||
const stale = projections[index];
|
||||
if (stale?.status !== 'claimed' || stale.claim?.kind !== 'wakeup') continue;
|
||||
const taskId = batch.results[index]!.taskId;
|
||||
const receipt = await methods.getAgentBackgroundToolResultClaim?.({
|
||||
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
userId,
|
||||
conversationId: envelope.target.conversationId,
|
||||
parentMessageId: envelope.target.parentMessageId,
|
||||
taskId,
|
||||
});
|
||||
if (receipt?.claimId !== context.idempotencyKey || receipt.batchId !== batch.batchId)
|
||||
continue;
|
||||
await methods.releaseBackgroundToolResultClaims({
|
||||
userId,
|
||||
conversationId: envelope.target.conversationId,
|
||||
messageId: envelope.target.parentMessageId,
|
||||
taskIds: [taskId],
|
||||
kind: 'wakeup',
|
||||
claimId: stale.claim.claimId,
|
||||
...(stale.claim.batchId != null && { batchId: stale.claim.batchId }),
|
||||
allowMissingMessage: true,
|
||||
});
|
||||
throw executionError('A stale background projection is being reconciled.', {
|
||||
code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
|
||||
retryable: true,
|
||||
deferWithoutAttempt: true,
|
||||
retryAfter: '1',
|
||||
});
|
||||
}
|
||||
const released = await release();
|
||||
const rootProjection =
|
||||
projections[batch.results.findIndex((result) => result.taskId === registration.taskId)];
|
||||
if (
|
||||
released &&
|
||||
rootProjection?.status === 'claimed' &&
|
||||
rootProjection.claim?.kind === 'manual' &&
|
||||
rootProjection.claim.receiptReconciled === true
|
||||
) {
|
||||
return { status: 'settled' };
|
||||
}
|
||||
if (released && recoverDeadClaim != null) {
|
||||
await Promise.all(
|
||||
projections.map(async (projection) => {
|
||||
if (
|
||||
projection.status !== 'claimed' ||
|
||||
projection.claim?.kind !== 'manual' ||
|
||||
projection.claim.receiptReconciled === true ||
|
||||
projection.claim.generationId == null
|
||||
)
|
||||
return;
|
||||
await recoverDeadClaim({
|
||||
userId,
|
||||
conversationId: envelope.target.conversationId,
|
||||
messageId: envelope.target.parentMessageId,
|
||||
claimId: projection.claim.claimId,
|
||||
kind: 'manual',
|
||||
generationId: projection.claim.generationId,
|
||||
onlyIfUnreconciled: true,
|
||||
});
|
||||
}),
|
||||
);
|
||||
}
|
||||
throw executionError('Background result ownership is being reconciled.', {
|
||||
code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
|
||||
retryable: true,
|
||||
deferWithoutAttempt: true,
|
||||
retryAfter: '1',
|
||||
});
|
||||
}
|
||||
return {
|
||||
status: 'ready',
|
||||
parentMessageId,
|
||||
...(parent.codeApprovalMode != null && { codeApprovalMode: parent.codeApprovalMode }),
|
||||
input: buildWakeupInput(batch.results),
|
||||
releaseOnDefiniteFailure: async () => {
|
||||
await release();
|
||||
},
|
||||
beginDispatch: async () => {
|
||||
dispatchBegan = true;
|
||||
if (
|
||||
!(await methods.beginAgentBackgroundToolResultBatchDispatch?.({ ...owned, dispatchId }))
|
||||
) {
|
||||
throw executionError('The background batch preparation was superseded.', {
|
||||
code: 'BACKGROUND_TOOL_BATCH_SUPERSEDED',
|
||||
retryable: true,
|
||||
deferWithoutAttempt: true,
|
||||
});
|
||||
}
|
||||
},
|
||||
settleOnAdmission: async (result) => {
|
||||
if (
|
||||
result.status === 'settled' &&
|
||||
getGenerationAdmissionEvidence != null &&
|
||||
(await getGenerationAdmissionEvidence(
|
||||
userId,
|
||||
context.idempotencyKey,
|
||||
envelope.target.conversationId,
|
||||
envelope.target.conversationId,
|
||||
)) == null
|
||||
) {
|
||||
throw new Error('A recovery fence does not prove background admission');
|
||||
}
|
||||
if (!(await methods.confirmAgentBackgroundToolResultBatch?.(owned))) {
|
||||
throw new Error('Background result batch admission was not confirmed');
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
const claim = await methods.claimBackgroundToolResults({
|
||||
userId,
|
||||
conversationId: envelope.target.conversationId,
|
||||
|
|
@ -545,6 +869,7 @@ export function createBackgroundToolCompletionWakeupResolver({
|
|||
deferWithoutAttempt: true,
|
||||
});
|
||||
};
|
||||
return resolve;
|
||||
}
|
||||
|
||||
/** Lists and discards a conversation's undelivered background completions from the
|
||||
|
|
@ -652,6 +977,7 @@ export function createBackgroundToolCompletionWakeupHandler(
|
|||
renewProducerLease: RenewBackgroundToolCompletionProducerLease,
|
||||
persistResult?: PersistBackgroundToolCompletionResult,
|
||||
expedite?: (deliveryKey: string) => void,
|
||||
getReceiptBatchingEnabled?: () => boolean,
|
||||
): (
|
||||
registration: BackgroundToolWakeupRegistration,
|
||||
) => Promise<BackgroundToolWakeupAdmission | false> {
|
||||
|
|
@ -695,7 +1021,10 @@ export function createBackgroundToolCompletionWakeupHandler(
|
|||
availableAt: new Date(
|
||||
Math.max(Date.now(), registration.createdAt) + WAKEUP_ADMISSION_DELAY_MS,
|
||||
),
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
requiredWorkerCapability:
|
||||
getReceiptBatchingEnabled?.() === true
|
||||
? AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3
|
||||
: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
producerLeaseUntil: new Date(Date.now() + BACKGROUND_TOOL_PRODUCER_LEASE_MS),
|
||||
});
|
||||
return {
|
||||
|
|
@ -735,8 +1064,22 @@ export function createBackgroundToolDeadClaimRecovery(
|
|||
claimId: string;
|
||||
}) => Promise<'fenced' | 'started' | 'unavailable'>,
|
||||
releaseReceiptClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'],
|
||||
batchRecovery?: Pick<
|
||||
AgentTriggerDeliveryMethods,
|
||||
'getAgentBackgroundToolResultBatch' | 'confirmAgentBackgroundToolResultBatch'
|
||||
>,
|
||||
getAdmissionEvidence?: BackgroundToolCompletionWakeupResolverDeps['getGenerationAdmissionEvidence'],
|
||||
): BackgroundToolDeadClaimRecovery {
|
||||
return async ({ userId, conversationId, messageId, claimId, kind, generationId }) => {
|
||||
return async ({
|
||||
userId,
|
||||
conversationId,
|
||||
messageId,
|
||||
claimId,
|
||||
kind,
|
||||
generationId,
|
||||
batchId,
|
||||
onlyIfUnreconciled,
|
||||
}) => {
|
||||
if (kind === 'manual') {
|
||||
if (generationId == null || generationId.length === 0) {
|
||||
return false;
|
||||
|
|
@ -753,8 +1096,41 @@ export function createBackgroundToolDeadClaimRecovery(
|
|||
messageId,
|
||||
kind: 'manual',
|
||||
claimId,
|
||||
...(onlyIfUnreconciled === true && { onlyIfUnreconciled }),
|
||||
});
|
||||
}
|
||||
const owner = {
|
||||
deliveryKey: claimId,
|
||||
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
userId,
|
||||
conversationId,
|
||||
parentMessageId: messageId,
|
||||
};
|
||||
const batch = await batchRecovery?.getAgentBackgroundToolResultBatch(owner);
|
||||
if (batchId != null && batch != null && batch.batchId !== batchId) return false;
|
||||
if (batch?.appliedAt != null) {
|
||||
await batchRecovery?.confirmAgentBackgroundToolResultBatch({
|
||||
...owner,
|
||||
batchId: batch.batchId,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
const confirmAdmission = async (): Promise<boolean> => {
|
||||
const evidence = await getAdmissionEvidence?.(
|
||||
userId,
|
||||
claimId,
|
||||
conversationId,
|
||||
conversationId,
|
||||
);
|
||||
if (evidence == null) return false;
|
||||
if (batch != null)
|
||||
await batchRecovery?.confirmAgentBackgroundToolResultBatch({
|
||||
...owner,
|
||||
batchId: batch.batchId,
|
||||
});
|
||||
return true;
|
||||
};
|
||||
if (batch != null && (await confirmAdmission())) return false;
|
||||
const claimGenerationIsActive = async (): Promise<boolean> => {
|
||||
const generation = await getGenerationJob(conversationId);
|
||||
return (
|
||||
|
|
@ -778,30 +1154,39 @@ export function createBackgroundToolDeadClaimRecovery(
|
|||
* installs a started tombstone that invalidates a delayed creator's token,
|
||||
* or observes that job creation already won. */
|
||||
const generationFence = await fenceGenerationClaim({ userId, conversationId, claimId });
|
||||
if (generationFence === 'unavailable') {
|
||||
if (generationFence === 'unavailable') return false;
|
||||
if (generationFence === 'started' && batch != null) {
|
||||
// The native claim CAS proves admission even when a custom store cannot
|
||||
// read historical claims or the live generation has been cleaned up.
|
||||
await batchRecovery?.confirmAgentBackgroundToolResultBatch({
|
||||
...owner,
|
||||
batchId: batch.batchId,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
if (await claimGenerationIsActive()) {
|
||||
return false;
|
||||
}
|
||||
const released = await releaseClaims({
|
||||
const receiptBatchId = batch?.batchId ?? batchId;
|
||||
const receiptReleased = await releaseReceiptClaims?.({
|
||||
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
userId,
|
||||
conversationId,
|
||||
parentMessageId: messageId,
|
||||
claimId,
|
||||
...(receiptBatchId != null && { batchId: receiptBatchId }),
|
||||
recoveryFenced: true,
|
||||
});
|
||||
if (receiptReleased === false) return false;
|
||||
if (batch != null) return receiptReleased === true;
|
||||
return releaseClaims({
|
||||
userId,
|
||||
conversationId,
|
||||
messageId,
|
||||
allowMissingMessage: true,
|
||||
kind: 'wakeup',
|
||||
claimId,
|
||||
...(receiptBatchId != null && { batchId: receiptBatchId }),
|
||||
});
|
||||
if (!released) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
(await releaseReceiptClaims?.({
|
||||
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
userId,
|
||||
conversationId,
|
||||
parentMessageId: messageId,
|
||||
claimId,
|
||||
})) ?? true
|
||||
);
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -457,6 +457,59 @@ describe('createToolExecuteHandler — background tool calls', () => {
|
|||
expect(retire).toHaveBeenCalledWith('background tool result was not persisted', undefined);
|
||||
});
|
||||
|
||||
it('copies committed manual reconciliation into the final background projection retry', async () => {
|
||||
let finishPersistence: (persisted: boolean) => void = () => undefined;
|
||||
let resolveProjection: (() => { resultClaim?: { receiptReconciled?: true } }) | undefined;
|
||||
const tool = makeSearchTool({ calls: 0 });
|
||||
const handler = createToolExecuteHandler({
|
||||
loadTools: async () => ({ loadedTools: [tool] }),
|
||||
backgroundToolCompletion: {
|
||||
preregister: async () => ({
|
||||
renew: async () => true,
|
||||
retire: async () => true,
|
||||
persistResult: async () => true,
|
||||
}),
|
||||
claim: async () => ({ status: 'not_ready' }),
|
||||
persist: (params) => {
|
||||
resolveProjection = params.resolveBackgroundTask;
|
||||
return new Promise<boolean>((resolve) => {
|
||||
finishPersistence = resolve;
|
||||
});
|
||||
},
|
||||
},
|
||||
});
|
||||
const [dispatch] = await runBatch(handler, {
|
||||
toolCalls: [
|
||||
{
|
||||
id: 'manual-marker-dispatch',
|
||||
stepId: 'manual-marker-step',
|
||||
name: tool.name,
|
||||
args: { q: 'marker', run_in_background: true },
|
||||
},
|
||||
],
|
||||
agentId: 'agent_parent_1',
|
||||
configurable: buildConfig([tool.name]),
|
||||
metadata: { thread_id: 'manual-marker-convo', run_id: 'manual-marker-parent' },
|
||||
});
|
||||
await flushMicrotasks();
|
||||
const taskId = JSON.parse(dispatch.content).background_task_id as string;
|
||||
try {
|
||||
expect(resolveProjection?.().resultClaim).toBeUndefined();
|
||||
expect(
|
||||
backgroundTaskRegistry.claimResult('exec_user', 'manual-marker-convo', taskId, {
|
||||
kind: 'manual',
|
||||
claimId: 'manual-marker-poll',
|
||||
receiptReconciled: true,
|
||||
generationId: 'manual-marker-generation',
|
||||
}),
|
||||
).toBe('acquired');
|
||||
expect(resolveProjection?.().resultClaim).toMatchObject({ receiptReconciled: true });
|
||||
} finally {
|
||||
finishPersistence(true);
|
||||
await flushMicrotasks();
|
||||
}
|
||||
});
|
||||
|
||||
it('falls back to the settled local result when a poll retired delivery before persistence failed', async () => {
|
||||
let finishPersistence: ((persisted: boolean) => void) | undefined;
|
||||
const persist = jest.fn(
|
||||
|
|
|
|||
|
|
@ -6446,6 +6446,9 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
|
|||
kind: current.resultClaim.kind,
|
||||
claimId: current.resultClaim.claimId,
|
||||
claimedAt: new Date(current.resultClaim.claimedAt),
|
||||
...(current.resultClaim.receiptReconciled === true && {
|
||||
receiptReconciled: true,
|
||||
}),
|
||||
...(current.resultClaim.generationId == null
|
||||
? {}
|
||||
: { generationId: current.resultClaim.generationId }),
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ export interface BackgroundToolResultState {
|
|||
claimId: string;
|
||||
claimedAt: Date;
|
||||
generationId?: string;
|
||||
receiptReconciled?: true;
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -269,3 +269,75 @@ event whose individual timing or acknowledgment is actionable. `fire`, `steer`,
|
|||
`continue` deliveries reject the option instead of silently weakening their semantics. Deliveries
|
||||
with `expectedAction` also reject coalescing because one generation cannot prove several distinct
|
||||
action fences.
|
||||
|
||||
## Background receipt batches
|
||||
|
||||
Ordinary background tool and code completions use `background_tool_completion_batch_v3`.
|
||||
Older workers cannot claim these rows; v2 receipts retain task-local delivery. Subagent
|
||||
completions do not join receipt batches. The existing `completionResultBatchSize` bounds
|
||||
selection, with no collection window. Scope is owner, tenant, conversation, launching
|
||||
parent message, and target agent. Results launched on different parent messages remain
|
||||
separate even when their branches later converge.
|
||||
|
||||
The root persists candidate keys before per-result claims, then freezes only successful
|
||||
claims in `backgroundToolResultBatch.members`. Retries use that same list. Each member's
|
||||
message projection is reconciled before dispatch. A first definite rejection releases the unadmitted plan. Once a handoff may have been
|
||||
admitted, later transport failures retain frozen ownership and the same idempotency key. Admission proof is stored before followers settle, and copied to each
|
||||
receipt so root retention cannot strand a follower.
|
||||
|
||||
Followers defer without consuming attempts. A dead or retired root is recovered through
|
||||
the generation-admission fence before claims are released. Losing collectors release
|
||||
claims they cannot dispatch. Concurrent roots can still split the ready set; batching is
|
||||
opportunistic, not one-turn-per-conversation election.
|
||||
|
||||
`triggerDelivery.spec.ts` includes a real-Mongo collecting barrier, crash/lost-reply
|
||||
injection, rolling-upgrade isolation, and an 8-conversation × 4-result storage stress
|
||||
harness. Its latency measures receipt admission, not model turns or deployment latency.
|
||||
|
||||
Receipt batching defaults off. Deploy compatible receipt and manual-poll consumers on
|
||||
**every replica**, then enable:
|
||||
|
||||
```yaml
|
||||
endpoints:
|
||||
agents:
|
||||
backgroundTasks:
|
||||
completionReceiptBatching: true
|
||||
```
|
||||
|
||||
Older manual-poll workers cannot read v3 ownership. Queue capability fencing alone does
|
||||
not protect that path. Disabling batching stops new v3 production, not existing
|
||||
ownership. Do not downgrade poll consumers while v3 receipts remain, including delivered
|
||||
receipts. A successor lease resumes interrupted cleanup before dispatch.
|
||||
|
||||
Every plan, receipt claim, and message claim carries a physical batch identity. Cleanup
|
||||
matches that identity; its final plan deletion also matches a release identity and the
|
||||
queue lease. A durable dispatch counter prevents a definite retry failure from releasing
|
||||
an earlier ambiguous handoff. Native recovery fences distinguish unadmitted tombstones
|
||||
from actual generation admission. Proven admissions settle followers, never re-present
|
||||
results.
|
||||
|
||||
Retired owners do not expire while a batch still needs claim release or per-receipt
|
||||
admission proof. Cleanup restores the ordinary retention deadline only after every
|
||||
member is safe without the owner. Retirement also closes publication and dispatch.
|
||||
|
||||
A failed receipt write is repaired from its durable terminal message projection.
|
||||
Manual polling marks receipt reconciliation before automatic delivery can settle that
|
||||
handoff; speculative manual ownership only defers the automatic delivery.
|
||||
|
||||
Manual confirmation resolves a lost reply by reading the exact committed claim.
|
||||
Failed confirmation rolls back only unreconciled ownership. If rollback is unavailable,
|
||||
automatic delivery invokes manual-generation recovery after that owner is no longer active.
|
||||
Its release CAS excludes committed manual handoffs, including confirmation after a stale
|
||||
snapshot. Explicit manual recovery keeps its existing generation-fenced contract.
|
||||
|
||||
Manual reconciliation survives registry restoration and final response persistence.
|
||||
Collecting plans acquire the root before siblings, so empty retirement can clear an
|
||||
unowned plan without orphaning claims. Native `started` recovery fences confirm admission
|
||||
even when a custom store cannot read historical claims after job cleanup.
|
||||
|
||||
Exact durable replays read the current claim without rewriting its handoff proof.
|
||||
Applied followers finish interrupted owner proof copying before they settle, including
|
||||
retired roots removed from the delivery queue.
|
||||
|
||||
Manual polls carry the observed physical batch ID through recovery. A released owner’s
|
||||
late projection is cleared by that exact ID; successor epochs and manual claims remain owned.
|
||||
|
|
|
|||
|
|
@ -11,6 +11,9 @@ export interface AgentTriggerDispatchContext {
|
|||
/** Durable delivery attempt metadata, when dispatched by the queue engine. */
|
||||
attempt?: number;
|
||||
maxAttempts?: number;
|
||||
/** Exact queue lease that owns this preparation. */
|
||||
deliveryClaimToken?: string;
|
||||
requiredWorkerCapability?: string;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
|
|
@ -44,7 +47,13 @@ export interface AgentTriggerDispatchHandlers<FireResult, ContinueResult, SteerR
|
|||
export function dispatchAgentTrigger<FireResult, ContinueResult, SteerResult>(
|
||||
envelope: unknown,
|
||||
handlers: AgentTriggerDispatchHandlers<FireResult, ContinueResult, SteerResult>,
|
||||
options?: { signal?: AbortSignal; attempt?: number; maxAttempts?: number },
|
||||
options?: {
|
||||
signal?: AbortSignal;
|
||||
attempt?: number;
|
||||
maxAttempts?: number;
|
||||
deliveryClaimToken?: string;
|
||||
requiredWorkerCapability?: string;
|
||||
},
|
||||
): Promise<ContinueResult | FireResult | SteerResult> {
|
||||
let normalized: AgentTriggerEnvelope;
|
||||
try {
|
||||
|
|
@ -56,6 +65,10 @@ export function dispatchAgentTrigger<FireResult, ContinueResult, SteerResult>(
|
|||
idempotencyKey: getAgentTriggerIdempotencyKey(normalized),
|
||||
...(options?.attempt != null && { attempt: options.attempt }),
|
||||
...(options?.maxAttempts != null && { maxAttempts: options.maxAttempts }),
|
||||
...(options?.deliveryClaimToken != null && { deliveryClaimToken: options.deliveryClaimToken }),
|
||||
...(options?.requiredWorkerCapability != null && {
|
||||
requiredWorkerCapability: options.requiredWorkerCapability,
|
||||
}),
|
||||
...(options?.signal != null && { signal: options.signal }),
|
||||
};
|
||||
if (normalized.mode === 'fire') {
|
||||
|
|
|
|||
|
|
@ -74,7 +74,12 @@ describe('createAgentTriggerDeliveryEngine', () => {
|
|||
|
||||
expect(dispatch).toHaveBeenCalledWith(
|
||||
{ version: 1 },
|
||||
{ signal: expect.any(AbortSignal), attempt: 1, maxAttempts: 8 },
|
||||
{
|
||||
signal: expect.any(AbortSignal),
|
||||
attempt: 1,
|
||||
maxAttempts: 8,
|
||||
deliveryClaimToken: 'claim-1',
|
||||
},
|
||||
);
|
||||
expect(store.beginAttempt).toHaveBeenCalledWith({
|
||||
id: 'delivery-row-1',
|
||||
|
|
|
|||
|
|
@ -92,6 +92,7 @@ export interface AgentTriggerDeliveryRecord {
|
|||
orderingKey: string;
|
||||
laneSequence: number;
|
||||
envelope: unknown;
|
||||
requiredWorkerCapability?: string;
|
||||
status: AgentTriggerDeliveryStatus;
|
||||
attempts: number;
|
||||
availableAt: Date;
|
||||
|
|
@ -204,7 +205,13 @@ export interface AgentTriggerDeliveryEngineDeps {
|
|||
store: AgentTriggerDeliveryStore;
|
||||
dispatch: (
|
||||
envelope: unknown,
|
||||
options?: { signal?: AbortSignal; attempt?: number; maxAttempts?: number },
|
||||
options?: {
|
||||
signal?: AbortSignal;
|
||||
attempt?: number;
|
||||
maxAttempts?: number;
|
||||
deliveryClaimToken?: string;
|
||||
requiredWorkerCapability?: string;
|
||||
},
|
||||
) => Promise<AgentTriggerExecutionResult>;
|
||||
/** Source-owned terminalization must commit before its delivery can become
|
||||
* dead, including recovery after a crash that exhausted the attempt budget. */
|
||||
|
|
@ -489,6 +496,10 @@ export function createAgentTriggerDeliveryEngine(
|
|||
signal: controller.signal,
|
||||
attempt,
|
||||
maxAttempts,
|
||||
deliveryClaimToken: delivery.claimToken,
|
||||
...(delivery.requiredWorkerCapability != null && {
|
||||
requiredWorkerCapability: delivery.requiredWorkerCapability,
|
||||
}),
|
||||
});
|
||||
} catch (error) {
|
||||
const attemptedAt = now();
|
||||
|
|
|
|||
|
|
@ -70,6 +70,8 @@ export type AgentTriggerContinuePreparation =
|
|||
releaseOnDefiniteFailure?: (error?: AgentTriggerExecutionError) => MaybePromise<void>;
|
||||
/** Commits the source handoff after generation admission. Failure is
|
||||
* outcome-ambiguous: the same delivery retries with the same request id. */
|
||||
/** Durable possible-handoff marker written immediately before transport. */
|
||||
beginDispatch?: () => MaybePromise<void>;
|
||||
settleOnAdmission?: (result: AgentTriggerContinueResult) => MaybePromise<void>;
|
||||
}
|
||||
| { status: 'settled' };
|
||||
|
|
@ -172,7 +174,13 @@ export interface AgentTriggerExecutionHostDeps {
|
|||
export interface AgentTriggerExecutionHost {
|
||||
dispatch: (
|
||||
envelope: unknown,
|
||||
options?: { signal?: AbortSignal; attempt?: number; maxAttempts?: number },
|
||||
options?: {
|
||||
signal?: AbortSignal;
|
||||
attempt?: number;
|
||||
maxAttempts?: number;
|
||||
deliveryClaimToken?: string;
|
||||
requiredWorkerCapability?: string;
|
||||
},
|
||||
) => Promise<AgentTriggerExecutionResult>;
|
||||
}
|
||||
|
||||
|
|
@ -636,6 +644,7 @@ async function startRun(
|
|||
const url = mode === 'fire' ? fireUrl(baseUrl) : continueUrl(baseUrl);
|
||||
const fetcher: AgentTriggerFetch = deps.fetch ?? globalThis.fetch;
|
||||
let response: Response;
|
||||
await readyPreparation?.beginDispatch?.();
|
||||
try {
|
||||
response = await fetcher(url, {
|
||||
method: 'POST',
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import {
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
|
||||
|
|
@ -185,6 +186,7 @@ describe('durable agent trigger service', () => {
|
|||
expect.objectContaining({
|
||||
workerCapabilities: [
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2,
|
||||
|
|
@ -211,6 +213,14 @@ describe('durable agent trigger service', () => {
|
|||
await service.stop();
|
||||
});
|
||||
|
||||
it('keeps receipt batching disabled until the deployment gate is explicitly enabled', async () => {
|
||||
const service = createAgentTriggerService({ mintToken: () => 'test' });
|
||||
expect(service.getBackgroundCompletionReceiptBatching()).toBe(false);
|
||||
await service.initialize({ address: '127.0.0.1', completionReceiptBatching: true });
|
||||
expect(service.getBackgroundCompletionReceiptBatching()).toBe(true);
|
||||
await service.stop();
|
||||
});
|
||||
|
||||
describe('waiting completion deliveries', () => {
|
||||
const address = { address: '127.0.0.1', family: 'IPv4' as const, port: 3080 };
|
||||
const flush = () => new Promise((resolve) => setImmediate(resolve));
|
||||
|
|
@ -407,6 +417,7 @@ describe('durable agent trigger service', () => {
|
|||
expect.objectContaining({
|
||||
workerCapabilities: [
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import {
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
|
||||
|
|
@ -50,6 +51,7 @@ const DEFAULT_PURGE_RECOVERY_LIMIT = 25;
|
|||
|
||||
export interface AgentTriggerServiceOptions {
|
||||
completionResultBatchSize?: number;
|
||||
completionReceiptBatching?: boolean;
|
||||
address?: BoundAddress | string | null;
|
||||
idlePolling?: {
|
||||
queuedTurnMaxIntervalMs?: number;
|
||||
|
|
@ -230,6 +232,7 @@ export interface AgentTriggerService {
|
|||
input: Parameters<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>[0],
|
||||
) => ReturnType<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>;
|
||||
getBackgroundCompletionResultBatchSize: () => number;
|
||||
getBackgroundCompletionReceiptBatching: () => boolean;
|
||||
/** Longest a waiting completion delivery re-checks readiness. */
|
||||
getCompletionWaitMaxIntervalMs: () => number;
|
||||
/** Best effort: moves waiting completion deliveries forward after what they wait on changed. */
|
||||
|
|
@ -271,6 +274,7 @@ function createDeliveryStore(
|
|||
...input,
|
||||
workerCapabilities: [
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2,
|
||||
|
|
@ -352,6 +356,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
|
|||
}
|
||||
let boundOrigin: string | undefined;
|
||||
let backgroundCompletionResultBatchSize = 8;
|
||||
let backgroundCompletionReceiptBatching = false;
|
||||
let completionWaitMaxIntervalMs = WAITING_RETRY_CAP_MS;
|
||||
let deliveryEngine: AgentTriggerDeliveryEngine | undefined;
|
||||
let initializePromise: Promise<void> | undefined;
|
||||
|
|
@ -410,7 +415,13 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
|
|||
|
||||
const dispatchForActivePrincipal = async (
|
||||
envelope: unknown,
|
||||
options?: { signal?: AbortSignal; attempt?: number; maxAttempts?: number },
|
||||
options?: {
|
||||
signal?: AbortSignal;
|
||||
attempt?: number;
|
||||
maxAttempts?: number;
|
||||
deliveryClaimToken?: string;
|
||||
requiredWorkerCapability?: string;
|
||||
},
|
||||
): Promise<AgentTriggerExecutionResult> => {
|
||||
const parsed = parseAgentTriggerEnvelope(envelope);
|
||||
await requireActivePrincipal(parsed.principal.userId);
|
||||
|
|
@ -619,6 +630,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
|
|||
return {
|
||||
initialize: (options = {}) => {
|
||||
backgroundCompletionResultBatchSize = options.completionResultBatchSize ?? 8;
|
||||
backgroundCompletionReceiptBatching = options.completionReceiptBatching === true;
|
||||
completionWaitMaxIntervalMs =
|
||||
options.idlePolling?.completionWaitMaxIntervalMs ?? WAITING_RETRY_CAP_MS;
|
||||
boundOrigin = selfOriginFromAddress(options.address) ?? boundOrigin;
|
||||
|
|
@ -809,6 +821,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
|
|||
return getClaim == null ? null : getClaim(input);
|
||||
}),
|
||||
getBackgroundCompletionResultBatchSize: () => backgroundCompletionResultBatchSize,
|
||||
getBackgroundCompletionReceiptBatching: () => backgroundCompletionReceiptBatching,
|
||||
getCompletionWaitMaxIntervalMs: () => completionWaitMaxIntervalMs,
|
||||
expediteCompletionWakeups: (input) => expediteCompletions(input),
|
||||
releaseBackgroundToolResultClaims: (input) =>
|
||||
|
|
|
|||
|
|
@ -270,7 +270,8 @@ function claimsMirrorExactly(left: TokenIdempotencyClaim, right: TokenIdempotenc
|
|||
left.claimToken === right.claimToken &&
|
||||
left.previousClaimToken === right.previousClaimToken &&
|
||||
left.generationProtocolVersion === right.generationProtocolVersion &&
|
||||
left.startedAt === right.startedAt
|
||||
left.startedAt === right.startedAt &&
|
||||
left.recoveryFence === right.recoveryFence
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -3339,7 +3340,7 @@ class GenerationJobManagerClass {
|
|||
}
|
||||
const normalized = normalizeTokenClaim(claim, 'admission evidence');
|
||||
assertClaimMatchesRequest(normalized, streamId, conversationId);
|
||||
if (normalized.startedAt == null) {
|
||||
if (normalized.startedAt == null || normalized.recoveryFence === true) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
|
|
@ -3380,14 +3381,14 @@ class GenerationJobManagerClass {
|
|||
return 'unavailable';
|
||||
}
|
||||
if (observed.existing.startedAt != null) {
|
||||
return 'started';
|
||||
return observed.existing.recoveryFence === true ? 'fenced' : 'started';
|
||||
}
|
||||
|
||||
let owned = observed;
|
||||
if (!observed.claimed) {
|
||||
owned = await this.takeoverGeneration(userId, clientRequestId, streamId, observed.existing);
|
||||
if (owned.existing?.startedAt != null) {
|
||||
return 'started';
|
||||
return owned.existing.recoveryFence === true ? 'fenced' : 'started';
|
||||
}
|
||||
if (!owned.claimed || owned.existing == null) {
|
||||
return 'unavailable';
|
||||
|
|
@ -3396,7 +3397,7 @@ class GenerationJobManagerClass {
|
|||
|
||||
const claim = normalizeTokenClaim(owned.existing, 'background completion recovery fence');
|
||||
if (claim.startedAt != null) {
|
||||
return 'started';
|
||||
return claim.recoveryFence === true ? 'fenced' : 'started';
|
||||
}
|
||||
await this.tombstoneObservedGenerationClaim(
|
||||
userId,
|
||||
|
|
@ -3405,6 +3406,7 @@ class GenerationJobManagerClass {
|
|||
claim,
|
||||
Date.now(),
|
||||
claim.generationProtocolVersion === 2 ? 2 : 1,
|
||||
true,
|
||||
);
|
||||
return 'fenced';
|
||||
}
|
||||
|
|
@ -3477,11 +3479,13 @@ class GenerationJobManagerClass {
|
|||
claim: TokenIdempotencyClaim,
|
||||
createdAt: number,
|
||||
generationProtocolVersion: 1 | 2,
|
||||
recoveryFence?: true,
|
||||
): Promise<TokenIdempotencyClaim> {
|
||||
const tombstone: TokenIdempotencyClaim = {
|
||||
...claim,
|
||||
startedAt: createdAt,
|
||||
generationProtocolVersion,
|
||||
...(recoveryFence === true && { recoveryFence }),
|
||||
};
|
||||
const primaryKey = this.generationClaimKey(userId, clientRequestId, streamId);
|
||||
let markError: unknown;
|
||||
|
|
|
|||
|
|
@ -394,6 +394,13 @@ describe('GenerationJobManager start-generation claim', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('does not mistake a replayed recovery tombstone for actual admission proof', async () => {
|
||||
const args = ['user-1', 'req-recovery-proof', 'stream-proof', 'stream-proof'] as const;
|
||||
expect(await manager.fenceGenerationClaimForRecovery(...args)).toBe('fenced');
|
||||
expect(await manager.fenceGenerationClaimForRecovery(...args)).toBe('fenced');
|
||||
expect(await manager.getGenerationAdmissionEvidence(...args)).toBeNull();
|
||||
});
|
||||
|
||||
it('invalidates an unpublished continuation creator before manual recovery', async () => {
|
||||
const original = await manager.claimGeneration(
|
||||
'user-1',
|
||||
|
|
|
|||
|
|
@ -714,6 +714,8 @@ export interface IdempotencyClaimValue {
|
|||
* already-started/cleaned generation and can never be taken over as an
|
||||
* abandoned pre-create lease. */
|
||||
startedAt?: number;
|
||||
/** Recovery closed an unpublished claim without admitting a generation. */
|
||||
recoveryFence?: true;
|
||||
}
|
||||
|
||||
/** Result of an atomic {@link IJobStore.claimIdempotencyKey} attempt. */
|
||||
|
|
|
|||
|
|
@ -1063,6 +1063,13 @@ describe('agent background completion batch config', () => {
|
|||
endpoints: { agents: { backgroundTasks: {} } },
|
||||
});
|
||||
expect(defaults.endpoints?.agents?.backgroundTasks?.completionResultBatchSize).toBe(8);
|
||||
expect(defaults.endpoints?.agents?.backgroundTasks?.completionReceiptBatching).toBe(false);
|
||||
expect(
|
||||
configSchema.parse({
|
||||
version: '1.0',
|
||||
endpoints: { agents: { backgroundTasks: { completionReceiptBatching: true } } },
|
||||
}).endpoints?.agents?.backgroundTasks?.completionReceiptBatching,
|
||||
).toBe(true);
|
||||
|
||||
for (const completionResultBatchSize of [0, 17, 1.5]) {
|
||||
expect(
|
||||
|
|
@ -1171,6 +1178,7 @@ describe('agent background task config', () => {
|
|||
}
|
||||
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
|
||||
completionResultBatchSize: 8,
|
||||
completionReceiptBatching: false,
|
||||
completionWakeups: true,
|
||||
completionResultMaxChars: 24 * 1024,
|
||||
ordinaryToolCancellation: false,
|
||||
|
|
@ -1190,6 +1198,7 @@ describe('agent background task config', () => {
|
|||
}
|
||||
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
|
||||
completionResultBatchSize: 8,
|
||||
completionReceiptBatching: false,
|
||||
completionWakeups: false,
|
||||
completionResultMaxChars: 24 * 1024,
|
||||
ordinaryToolCancellation: false,
|
||||
|
|
@ -1209,6 +1218,7 @@ describe('agent background task config', () => {
|
|||
}
|
||||
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
|
||||
completionResultBatchSize: 8,
|
||||
completionReceiptBatching: false,
|
||||
completionWakeups: true,
|
||||
completionResultMaxChars: 24 * 1024,
|
||||
ordinaryToolCancellation: true,
|
||||
|
|
|
|||
|
|
@ -1736,8 +1736,9 @@ export const agentsEndpointSchema = baseEndpointSchema
|
|||
.max(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX)
|
||||
.optional()
|
||||
.default(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT),
|
||||
/** Maximum message-backed sibling results in one continuation.
|
||||
* Independent receipts retain task-local delivery ownership. */
|
||||
/** Enable only after every replica has compatible receipt/poll consumers. */
|
||||
completionReceiptBatching: z.boolean().optional().default(false),
|
||||
/** Maximum compatible sibling results in one continuation. */
|
||||
completionResultBatchSize: z.number().int().min(1).max(16).optional().default(8),
|
||||
/** Cooperative cancellation for process-local ordinary tools. Off
|
||||
* by default so existing deployments opt into the new control. */
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ export {
|
|||
export {
|
||||
AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
|
||||
|
|
|
|||
|
|
@ -347,6 +347,7 @@ export function createMethods(
|
|||
|
||||
const agentQueuedTurnMethods = createAgentQueuedTurnMethods(mongoose);
|
||||
const agentTriggerDeliveryMethods = createAgentTriggerDeliveryMethods(mongoose, {
|
||||
releaseBatchProjections: messageMethods.releaseBackgroundToolResultClaims,
|
||||
purgeQueuedTurnsForUser: (user) =>
|
||||
agentQueuedTurnMethods.deleteAllAgentQueuedTurnsForUser({
|
||||
user: typeof user === 'string' ? new mongoose.Types.ObjectId(user) : user,
|
||||
|
|
|
|||
|
|
@ -5991,3 +5991,94 @@ describe('Message Operations', () => {
|
|||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('receipt-backed claim release before projection', () => {
|
||||
it('allows a missing message only when the receipt owner explicitly opts in', async () => {
|
||||
const input = {
|
||||
userId: new mongoose.Types.ObjectId().toString(),
|
||||
conversationId: 'missing-conversation',
|
||||
messageId: 'missing-parent',
|
||||
kind: 'wakeup' as const,
|
||||
claimId: 'batch-root',
|
||||
};
|
||||
await expect(releaseBackgroundToolResultClaims(input)).resolves.toBe(false);
|
||||
await expect(
|
||||
releaseBackgroundToolResultClaims({ ...input, allowMissingMessage: true }),
|
||||
).resolves.toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('receipt batch projection epochs', () => {
|
||||
it('does not let delayed cleanup remove a successor claim on the same logical request', async () => {
|
||||
await Message.create({
|
||||
messageId: 'epoch-parent',
|
||||
conversationId: 'epoch-conversation',
|
||||
user: 'epoch-owner',
|
||||
isCreatedByUser: false,
|
||||
unfinished: false,
|
||||
content: [
|
||||
{
|
||||
type: 'tool_call',
|
||||
tool_call: {
|
||||
id: 'epoch-call',
|
||||
output: 'done',
|
||||
backgroundTask: {
|
||||
taskId: 'epoch-task',
|
||||
toolName: 'tool',
|
||||
status: 'completed',
|
||||
completionWakeup: true,
|
||||
completionReceipt: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
const input = {
|
||||
userId: 'epoch-owner',
|
||||
conversationId: 'epoch-conversation',
|
||||
messageId: 'epoch-parent',
|
||||
taskId: 'epoch-task',
|
||||
kind: 'wakeup' as const,
|
||||
claimId: 'logical-root',
|
||||
limit: 1,
|
||||
};
|
||||
expect((await claimBackgroundToolResults({ ...input, batchId: 'old' })).status).toBe(
|
||||
'acquired',
|
||||
);
|
||||
let pause: () => void = () => undefined;
|
||||
let resume: () => void = () => undefined;
|
||||
const paused = new Promise<void>((resolve) => {
|
||||
pause = resolve;
|
||||
});
|
||||
const barrier = new Promise<void>((resolve) => {
|
||||
resume = resolve;
|
||||
});
|
||||
const update = Message.findOneAndUpdate.bind(Message);
|
||||
const oldCleanup = jest.spyOn(Message, 'findOneAndUpdate').mockImplementationOnce((...args) => {
|
||||
const query = update(...args);
|
||||
const execute = query.exec.bind(query);
|
||||
jest.spyOn(query, 'exec').mockImplementationOnce(async () => {
|
||||
pause();
|
||||
await barrier;
|
||||
return execute();
|
||||
});
|
||||
return query;
|
||||
});
|
||||
const predecessor = releaseBackgroundToolResultClaims({ ...input, batchId: 'old' });
|
||||
await paused;
|
||||
expect(await releaseBackgroundToolResultClaims({ ...input, batchId: 'old' })).toBe(true);
|
||||
expect((await claimBackgroundToolResults({ ...input, batchId: 'new' })).status).toBe(
|
||||
'acquired',
|
||||
);
|
||||
resume();
|
||||
expect(await predecessor).toBe(true);
|
||||
oldCleanup.mockRestore();
|
||||
expect((await claimBackgroundToolResults({ ...input, batchId: 'new' })).status).toBe(
|
||||
'acquired',
|
||||
);
|
||||
expect(await claimBackgroundToolResults({ ...input, batchId: 'old' })).toMatchObject({
|
||||
status: 'claimed',
|
||||
claim: { batchId: 'new' },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -567,7 +567,13 @@ export type BackgroundToolResultClaim =
|
|||
| { status: 'outcome_unknown'; toolName: string }
|
||||
| {
|
||||
status: 'claimed';
|
||||
claim?: { kind: 'manual' | 'wakeup'; claimId: string; generationId?: string };
|
||||
claim?: {
|
||||
kind: 'manual' | 'wakeup';
|
||||
claimId: string;
|
||||
generationId?: string;
|
||||
batchId?: string;
|
||||
receiptReconciled?: true;
|
||||
};
|
||||
messageId?: string;
|
||||
}
|
||||
| { status: 'acquired'; results: BackgroundToolResultRecord[]; messageId?: string };
|
||||
|
|
@ -846,6 +852,7 @@ export interface MessageMethods {
|
|||
claimId: string;
|
||||
claimedAt: Date;
|
||||
generationId?: string;
|
||||
receiptReconciled?: true;
|
||||
};
|
||||
};
|
||||
}): Promise<{ matched: boolean; unfinished: boolean }>;
|
||||
|
|
@ -858,6 +865,8 @@ export interface MessageMethods {
|
|||
agentId?: string;
|
||||
kind: 'manual' | 'wakeup';
|
||||
claimId: string;
|
||||
/** Physical receipt-batch identity, independent of its logical request. */
|
||||
batchId?: string;
|
||||
/** Response generation that owns this manual result delivery. */
|
||||
generationId?: string;
|
||||
/** Manual owner-process takeover after automatic delivery was retired. */
|
||||
|
|
@ -866,12 +875,24 @@ export interface MessageMethods {
|
|||
/** Includes JSON escaping, delimiters, and empty result fields. */
|
||||
maxMetadataChars?: number;
|
||||
}): Promise<BackgroundToolResultClaim>;
|
||||
confirmBackgroundToolResultClaim(params: {
|
||||
userId: string;
|
||||
conversationId: string;
|
||||
messageId: string;
|
||||
taskId: string;
|
||||
claimId: string;
|
||||
}): Promise<boolean>;
|
||||
releaseBackgroundToolResultClaims(params: {
|
||||
userId: string;
|
||||
conversationId: string;
|
||||
messageId: string;
|
||||
/** Omit to release every sibling owned by this exact batch claim. */
|
||||
taskIds?: string[];
|
||||
/** Receipt-backed results can precede the message projection. */
|
||||
allowMissingMessage?: true;
|
||||
/** Manual rollback must not erase a committed receipt handoff. */
|
||||
onlyIfUnreconciled?: true;
|
||||
batchId?: string;
|
||||
kind: 'manual' | 'wakeup';
|
||||
claimId: string;
|
||||
}): Promise<boolean>;
|
||||
|
|
@ -1983,7 +2004,15 @@ export function createMessageMethods(
|
|||
function readBackgroundToolResultClaim(
|
||||
row: Pick<IMessage, 'content'>,
|
||||
taskId: string,
|
||||
): { kind: 'manual' | 'wakeup'; claimId: string; generationId?: string } | undefined {
|
||||
):
|
||||
| {
|
||||
kind: 'manual' | 'wakeup';
|
||||
claimId: string;
|
||||
generationId?: string;
|
||||
batchId?: string;
|
||||
receiptReconciled?: true;
|
||||
}
|
||||
| undefined {
|
||||
for (const part of row.content ?? []) {
|
||||
if (part == null || typeof part !== 'object' || Array.isArray(part)) {
|
||||
continue;
|
||||
|
|
@ -1993,7 +2022,13 @@ export function createMessageMethods(
|
|||
tool_call?: {
|
||||
backgroundTask?: {
|
||||
taskId?: unknown;
|
||||
resultClaim?: { kind?: unknown; claimId?: unknown; generationId?: unknown };
|
||||
resultClaim?: {
|
||||
kind?: unknown;
|
||||
claimId?: unknown;
|
||||
generationId?: unknown;
|
||||
batchId?: unknown;
|
||||
receiptReconciled?: unknown;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -2010,6 +2045,8 @@ export function createMessageMethods(
|
|||
return {
|
||||
kind: claim.kind,
|
||||
claimId: claim.claimId,
|
||||
...(typeof claim.batchId === 'string' && { batchId: claim.batchId }),
|
||||
...(claim.receiptReconciled === true && { receiptReconciled: true }),
|
||||
...(typeof claim.generationId === 'string' && claim.generationId.length > 0
|
||||
? { generationId: claim.generationId }
|
||||
: {}),
|
||||
|
|
@ -2021,7 +2058,7 @@ export function createMessageMethods(
|
|||
|
||||
function parseBackgroundToolResults(
|
||||
message: IMessage,
|
||||
claim: { kind: 'manual' | 'wakeup'; claimId: string },
|
||||
claim: { kind: 'manual' | 'wakeup'; claimId: string; batchId?: string },
|
||||
): BackgroundToolResultRecord[] {
|
||||
const results: BackgroundToolResultRecord[] = [];
|
||||
for (const part of message.content ?? []) {
|
||||
|
|
@ -2040,7 +2077,7 @@ export function createMessageMethods(
|
|||
status?: unknown;
|
||||
cancelled?: unknown;
|
||||
settledAt?: unknown;
|
||||
resultClaim?: { kind?: unknown; claimId?: unknown };
|
||||
resultClaim?: { kind?: unknown; claimId?: unknown; batchId?: unknown };
|
||||
};
|
||||
};
|
||||
};
|
||||
|
|
@ -2052,7 +2089,8 @@ export function createMessageMethods(
|
|||
typeof task.toolName !== 'string' ||
|
||||
(task.status !== 'completed' && task.status !== 'error') ||
|
||||
task.resultClaim?.kind !== claim.kind ||
|
||||
task.resultClaim.claimId !== claim.claimId
|
||||
task.resultClaim.claimId !== claim.claimId ||
|
||||
task.resultClaim.batchId !== claim.batchId
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
|
@ -2133,6 +2171,7 @@ export function createMessageMethods(
|
|||
kind,
|
||||
claimId,
|
||||
generationId,
|
||||
batchId,
|
||||
allowUnfinished = false,
|
||||
limit = kind === 'wakeup' ? 8 : 1,
|
||||
maxMetadataChars,
|
||||
|
|
@ -2144,6 +2183,7 @@ export function createMessageMethods(
|
|||
if (
|
||||
(requestedMessageId != null &&
|
||||
(requestedMessageId.length === 0 || requestedMessageId.length > 256)) ||
|
||||
(batchId != null && (batchId.length === 0 || batchId.length > 128 || kind !== 'wakeup')) ||
|
||||
taskId.length === 0 ||
|
||||
taskId.length > 256 ||
|
||||
claimId.length === 0 ||
|
||||
|
|
@ -2214,7 +2254,10 @@ export function createMessageMethods(
|
|||
}
|
||||
const recoveredSource = requestedMessageId == null ? { messageId: resolvedMessageId } : {};
|
||||
const requestedClaim = readBackgroundToolResultClaim(row, taskId);
|
||||
const replaying = requestedClaim?.kind === kind && requestedClaim.claimId === claimId;
|
||||
const replaying =
|
||||
requestedClaim?.kind === kind &&
|
||||
requestedClaim.claimId === claimId &&
|
||||
requestedClaim.batchId === batchId;
|
||||
const candidates: string[] = [];
|
||||
const costs = new Map<string, number>();
|
||||
let receiptBacked = false;
|
||||
|
|
@ -2233,8 +2276,13 @@ export function createMessageMethods(
|
|||
}
|
||||
const terminal = task?.status === 'completed' || task?.status === 'error';
|
||||
const wakeupEligible = kind !== 'wakeup' || task?.completionWakeup === true;
|
||||
const resultClaim = task?.resultClaim as { kind?: unknown; claimId?: unknown } | undefined;
|
||||
const replay = resultClaim?.kind === kind && resultClaim.claimId === claimId;
|
||||
const resultClaim = task?.resultClaim as
|
||||
| { kind?: unknown; claimId?: unknown; batchId?: unknown }
|
||||
| undefined;
|
||||
const replay =
|
||||
resultClaim?.kind === kind &&
|
||||
resultClaim.claimId === claimId &&
|
||||
resultClaim.batchId === batchId;
|
||||
const sameAgent =
|
||||
agentId == null ||
|
||||
partAgentId == null ||
|
||||
|
|
@ -2315,105 +2363,103 @@ export function createMessageMethods(
|
|||
kind,
|
||||
claimId,
|
||||
claimedAt,
|
||||
...(batchId != null && { batchId }),
|
||||
...(kind === 'manual' && requestedGenerationId != null
|
||||
? { generationId: requestedGenerationId }
|
||||
: {}),
|
||||
};
|
||||
const updated = await Message.findOneAndUpdate(
|
||||
{
|
||||
user: userId,
|
||||
conversationId,
|
||||
messageId: resolvedMessageId,
|
||||
...(kind === 'manual' && allowUnfinished ? {} : { unfinished: { $ne: true } }),
|
||||
content: {
|
||||
$elemMatch: {
|
||||
type: 'tool_call',
|
||||
'tool_call.backgroundTask.taskId': taskId,
|
||||
'tool_call.backgroundTask.status': { $in: ['completed', 'error'] },
|
||||
...(kind === 'wakeup' ? { 'tool_call.backgroundTask.completionWakeup': true } : {}),
|
||||
...(replaying
|
||||
? {
|
||||
'tool_call.backgroundTask.resultClaim.kind': kind,
|
||||
'tool_call.backgroundTask.resultClaim.claimId': claimId,
|
||||
}
|
||||
: /** Missing OR stored null: the in-memory claimable scan, the
|
||||
* claim arrayFilters, and the settle stamp all treat a null
|
||||
* claim as unclaimed, and the subfield-preserving settle write
|
||||
* keeps a persisted null a whole-object rewrite used to drop.
|
||||
* `$exists: false` here would strand such a part as terminal
|
||||
* but permanently unclaimable. */
|
||||
{ 'tool_call.backgroundTask.resultClaim': null }),
|
||||
},
|
||||
const claimFilter: FilterQuery<IMessage> = {
|
||||
user: userId,
|
||||
conversationId,
|
||||
messageId: resolvedMessageId,
|
||||
...(kind === 'manual' && allowUnfinished ? {} : { unfinished: { $ne: true } }),
|
||||
content: {
|
||||
$elemMatch: {
|
||||
type: 'tool_call',
|
||||
'tool_call.backgroundTask.taskId': taskId,
|
||||
'tool_call.backgroundTask.status': { $in: ['completed', 'error'] },
|
||||
...(kind === 'wakeup' ? { 'tool_call.backgroundTask.completionWakeup': true } : {}),
|
||||
...(replaying
|
||||
? {
|
||||
'tool_call.backgroundTask.resultClaim.kind': kind,
|
||||
'tool_call.backgroundTask.resultClaim.claimId': claimId,
|
||||
'tool_call.backgroundTask.resultClaim.batchId': batchId ?? null,
|
||||
}
|
||||
: /** Missing OR stored null: the in-memory claimable scan, the
|
||||
* claim arrayFilters, and the settle stamp all treat a null
|
||||
* claim as unclaimed, and the subfield-preserving settle write
|
||||
* keeps a persisted null a whole-object rewrite used to drop.
|
||||
* `$exists: false` here would strand such a part as terminal
|
||||
* but permanently unclaimable. */
|
||||
{ 'tool_call.backgroundTask.resultClaim': null }),
|
||||
},
|
||||
...(agentId != null
|
||||
? {
|
||||
$expr: {
|
||||
$anyElementTrue: {
|
||||
$map: {
|
||||
input: { $ifNull: ['$content', []] },
|
||||
as: 'candidate',
|
||||
in: {
|
||||
$and: [
|
||||
{ $eq: ['$$candidate.tool_call.backgroundTask.taskId', taskId] },
|
||||
{
|
||||
$in: [
|
||||
{
|
||||
$ifNull: [
|
||||
{
|
||||
$ifNull: ['$$candidate.agentId', '$$candidate.tool_call.agentId'],
|
||||
},
|
||||
null,
|
||||
],
|
||||
},
|
||||
[null, agentId],
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
...(agentId != null
|
||||
? {
|
||||
$expr: {
|
||||
$anyElementTrue: {
|
||||
$map: {
|
||||
input: { $ifNull: ['$content', []] },
|
||||
as: 'candidate',
|
||||
in: {
|
||||
$and: [
|
||||
{ $eq: ['$$candidate.tool_call.backgroundTask.taskId', taskId] },
|
||||
{
|
||||
$in: [
|
||||
{
|
||||
$ifNull: [
|
||||
{
|
||||
$ifNull: ['$$candidate.agentId', '$$candidate.tool_call.agentId'],
|
||||
},
|
||||
null,
|
||||
],
|
||||
},
|
||||
[null, agentId],
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
/** Stamps the claim onto every part this pass admitted. The filtered
|
||||
* positional operator selects those parts by predicate, so the write
|
||||
* touches only them instead of re-emitting the whole content array, and
|
||||
* needs no read-modify-write. Amazon DocumentDB rejects the
|
||||
* aggregation-pipeline form this replaces. */
|
||||
{ $set: { 'content.$[part].tool_call.backgroundTask.resultClaim': claimStamp } },
|
||||
{
|
||||
new: true,
|
||||
projection: { content: 1 },
|
||||
arrayFilters: [
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
const updated = replaying
|
||||
? await Message.findOne(claimFilter).select({ content: 1 }).lean<IMessage | null>()
|
||||
: await Message.findOneAndUpdate(
|
||||
claimFilter,
|
||||
/** Stamps the claim onto every part this pass admitted. The filtered
|
||||
* positional operator selects those parts by predicate, so the write
|
||||
* touches only them instead of re-emitting the whole content array, and
|
||||
* needs no read-modify-write. Amazon DocumentDB rejects the
|
||||
* aggregation-pipeline form this replaces. */
|
||||
{ $set: { 'content.$[part].tool_call.backgroundTask.resultClaim': claimStamp } },
|
||||
{
|
||||
'part.type': 'tool_call',
|
||||
'part.tool_call.backgroundTask.taskId': { $in: candidates },
|
||||
'part.tool_call.backgroundTask.status': { $in: ['completed', 'error'] },
|
||||
...(kind === 'wakeup'
|
||||
? { 'part.tool_call.backgroundTask.completionWakeup': true }
|
||||
: {}),
|
||||
$and: [
|
||||
new: true,
|
||||
projection: { content: 1 },
|
||||
arrayFilters: [
|
||||
{
|
||||
/** Unclaimed, or already held by this exact claimant (replay). */
|
||||
$or: [
|
||||
{ 'part.tool_call.backgroundTask.resultClaim': null },
|
||||
'part.type': 'tool_call',
|
||||
'part.tool_call.backgroundTask.taskId': { $in: candidates },
|
||||
'part.tool_call.backgroundTask.status': { $in: ['completed', 'error'] },
|
||||
...(kind === 'wakeup'
|
||||
? { 'part.tool_call.backgroundTask.completionWakeup': true }
|
||||
: {}),
|
||||
$and: [
|
||||
{
|
||||
'part.tool_call.backgroundTask.resultClaim.kind': kind,
|
||||
'part.tool_call.backgroundTask.resultClaim.claimId': claimId,
|
||||
'part.tool_call.backgroundTask.resultClaim': null,
|
||||
},
|
||||
...(agentId == null ? [] : [agentOwnershipFilter('part.', agentId)]),
|
||||
],
|
||||
},
|
||||
...(agentId == null ? [] : [agentOwnershipFilter('part.', agentId)]),
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
).lean<IMessage | null>();
|
||||
).lean<IMessage | null>();
|
||||
if (updated == null) {
|
||||
return { status: 'not_ready' };
|
||||
}
|
||||
const results = parseBackgroundToolResults(updated, { kind, claimId });
|
||||
const results = parseBackgroundToolResults(updated, { kind, claimId, batchId });
|
||||
const competingClaim = readBackgroundToolResultClaim(updated, taskId);
|
||||
return results.some((result) => result.taskId === taskId)
|
||||
? { status: 'acquired', results, ...recoveredSource }
|
||||
|
|
@ -2424,21 +2470,82 @@ export function createMessageMethods(
|
|||
};
|
||||
}
|
||||
|
||||
/** Manual ownership is committed only after receipt arbitration completes. */
|
||||
async function confirmBackgroundToolResultClaim(
|
||||
input: Parameters<MessageMethods['confirmBackgroundToolResultClaim']>[0],
|
||||
): Promise<boolean> {
|
||||
const Message = mongoose.models.Message as Model<IMessage>;
|
||||
const identity = {
|
||||
'tool_call.backgroundTask.taskId': input.taskId,
|
||||
'tool_call.backgroundTask.resultClaim.kind': 'manual',
|
||||
'tool_call.backgroundTask.resultClaim.claimId': input.claimId,
|
||||
};
|
||||
try {
|
||||
const confirmed = await Message.updateOne(
|
||||
{
|
||||
user: input.userId,
|
||||
conversationId: input.conversationId,
|
||||
messageId: input.messageId,
|
||||
content: { $elemMatch: identity },
|
||||
},
|
||||
{
|
||||
$set: { 'content.$[part].tool_call.backgroundTask.resultClaim.receiptReconciled': true },
|
||||
},
|
||||
{
|
||||
arrayFilters: [
|
||||
{
|
||||
'part.tool_call.backgroundTask.taskId': input.taskId,
|
||||
'part.tool_call.backgroundTask.resultClaim.kind': 'manual',
|
||||
'part.tool_call.backgroundTask.resultClaim.claimId': input.claimId,
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
return confirmed.matchedCount === 1;
|
||||
} catch (error) {
|
||||
// A lost write reply is not an uncommitted handoff. Read only the exact
|
||||
// manual claim, never manufacture success from another claimant.
|
||||
const committed = await Message.exists({
|
||||
user: input.userId,
|
||||
conversationId: input.conversationId,
|
||||
messageId: input.messageId,
|
||||
content: {
|
||||
$elemMatch: {
|
||||
...identity,
|
||||
'tool_call.backgroundTask.resultClaim.receiptReconciled': true,
|
||||
},
|
||||
},
|
||||
});
|
||||
if (committed != null) return true;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function releaseBackgroundToolResultClaims({
|
||||
userId,
|
||||
conversationId,
|
||||
messageId,
|
||||
taskIds,
|
||||
allowMissingMessage,
|
||||
onlyIfUnreconciled,
|
||||
kind,
|
||||
claimId,
|
||||
batchId,
|
||||
}: {
|
||||
userId: string;
|
||||
conversationId: string;
|
||||
messageId: string;
|
||||
taskIds?: string[];
|
||||
/** Receipt-backed results can precede the message projection. */
|
||||
allowMissingMessage?: true;
|
||||
/** Manual rollback must not erase a committed receipt handoff. */
|
||||
onlyIfUnreconciled?: true;
|
||||
batchId?: string;
|
||||
kind: 'manual' | 'wakeup';
|
||||
claimId: string;
|
||||
}): Promise<boolean> {
|
||||
if (onlyIfUnreconciled === true && kind !== 'manual')
|
||||
throw new TypeError('Unreconciled rollback requires a manual claim');
|
||||
if (taskIds?.length === 0) {
|
||||
return true;
|
||||
}
|
||||
|
|
@ -2460,6 +2567,10 @@ export function createMessageMethods(
|
|||
{
|
||||
'part.tool_call.backgroundTask.resultClaim.kind': kind,
|
||||
'part.tool_call.backgroundTask.resultClaim.claimId': claimId,
|
||||
'part.tool_call.backgroundTask.resultClaim.batchId': batchId ?? null,
|
||||
...(onlyIfUnreconciled === true && {
|
||||
'part.tool_call.backgroundTask.resultClaim.receiptReconciled': { $ne: true },
|
||||
}),
|
||||
...(taskIds == null
|
||||
? {}
|
||||
: { 'part.tool_call.backgroundTask.taskId': { $in: taskIds } }),
|
||||
|
|
@ -2474,9 +2585,13 @@ export function createMessageMethods(
|
|||
messageId,
|
||||
content: { $not: { $type: 'array' } },
|
||||
});
|
||||
return arraylessRow != null;
|
||||
if (arraylessRow != null) return true;
|
||||
return (
|
||||
allowMissingMessage === true &&
|
||||
(await Message.exists({ user: userId, conversationId, messageId })) == null
|
||||
);
|
||||
}
|
||||
const remaining = parseBackgroundToolResults(updated, { kind, claimId });
|
||||
const remaining = parseBackgroundToolResults(updated, { kind, claimId, batchId });
|
||||
return taskIds == null
|
||||
? remaining.length === 0
|
||||
: !remaining.some((result) => taskIds.includes(result.taskId));
|
||||
|
|
@ -4249,6 +4364,7 @@ export function createMessageMethods(
|
|||
updateMessageText,
|
||||
updateToolCallResult,
|
||||
claimBackgroundToolResults,
|
||||
confirmBackgroundToolResultClaim,
|
||||
releaseBackgroundToolResultClaims,
|
||||
updateMessage,
|
||||
recordSubagentTaskControlReceipt,
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,4 +1,5 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { backgroundResultMetadata } from 'librechat-data-provider';
|
||||
import type { FilterQuery, Model, Types } from 'mongoose';
|
||||
import type {
|
||||
AgentBackgroundToolResultReceipt,
|
||||
|
|
@ -17,8 +18,10 @@ import type {
|
|||
IAgentTriggerUserPurge,
|
||||
IAgentTriggerUserPurgeDocument,
|
||||
} from '~/types/triggerDelivery';
|
||||
import type { MessageMethods } from './message';
|
||||
import {
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
|
||||
AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
|
||||
|
|
@ -38,6 +41,10 @@ const DEFAULT_PURGE_RECOVERY_LIMIT = 25;
|
|||
const MAX_PURGE_RECOVERY_LIMIT = 200;
|
||||
const HISTORY_LIMIT = 64;
|
||||
const MAX_BATCH_SIZE = 8;
|
||||
const BACKGROUND_RECEIPT_CAPABILITIES = [
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
|
||||
];
|
||||
const MAX_BATCH_BYTES = 512 * 1024;
|
||||
/** Bounds the claim's compare-and-swap retries. Each lost round means another
|
||||
* worker claimed the row this one read, so the queue is making progress. */
|
||||
|
|
@ -167,6 +174,48 @@ export type AgentBackgroundToolResultClaim =
|
|||
}>;
|
||||
};
|
||||
|
||||
export interface AgentBackgroundToolResultBatchInput {
|
||||
deliveryKey: string;
|
||||
sourceId: string;
|
||||
userId: string;
|
||||
tenantId?: string;
|
||||
conversationId: string;
|
||||
parentMessageId: string;
|
||||
agentId: string;
|
||||
limit: number;
|
||||
maxMetadataChars: number;
|
||||
deliveryClaimToken?: string;
|
||||
}
|
||||
|
||||
export type AgentBackgroundToolResultBatchClaim =
|
||||
| { status: 'legacy' }
|
||||
| { status: 'not_ready'; waitingForResult?: true }
|
||||
| {
|
||||
status: 'claimed';
|
||||
claimId: string;
|
||||
batchId: string;
|
||||
ownerStatus: 'pending' | 'applied' | 'recoverable';
|
||||
}
|
||||
| (Extract<AgentBackgroundToolResultClaim, { status: 'acquired' }> & { batchId: string });
|
||||
|
||||
export type AgentBackgroundToolResultBatchOwnerInput = Omit<
|
||||
AgentBackgroundToolResultBatchInput,
|
||||
'limit' | 'maxMetadataChars' | 'agentId'
|
||||
> & { agentId?: string };
|
||||
|
||||
export type AgentBackgroundToolResultBatchReleaseInput = {
|
||||
sourceId: string;
|
||||
userId: string;
|
||||
conversationId: string;
|
||||
parentMessageId: string;
|
||||
claimId: string;
|
||||
deliveryClaimToken?: string;
|
||||
batchId?: string;
|
||||
dispatchId?: string;
|
||||
/** Retirement and generation-claim CAS proved that no admission can occur. */
|
||||
recoveryFenced?: true;
|
||||
};
|
||||
|
||||
export interface AgentTriggerDeliveryFence {
|
||||
id: string;
|
||||
workerId: string;
|
||||
|
|
@ -414,13 +463,21 @@ export interface AgentTriggerDeliveryMethods {
|
|||
claimId: string;
|
||||
limit?: number;
|
||||
}) => Promise<AgentBackgroundToolResultClaim>;
|
||||
releaseAgentBackgroundToolResultClaims: (input: {
|
||||
sourceId: string;
|
||||
userId: string;
|
||||
conversationId: string;
|
||||
parentMessageId: string;
|
||||
claimId: string;
|
||||
}) => Promise<boolean>;
|
||||
releaseAgentBackgroundToolResultClaims: (
|
||||
input: AgentBackgroundToolResultBatchReleaseInput,
|
||||
) => Promise<boolean>;
|
||||
claimAgentBackgroundToolResultBatch: (
|
||||
input: AgentBackgroundToolResultBatchInput,
|
||||
) => Promise<AgentBackgroundToolResultBatchClaim>;
|
||||
getAgentBackgroundToolResultBatch: (
|
||||
input: AgentBackgroundToolResultBatchOwnerInput,
|
||||
) => Promise<IAgentTriggerDelivery['backgroundToolResultBatch'] | null>;
|
||||
beginAgentBackgroundToolResultBatchDispatch: (
|
||||
input: AgentBackgroundToolResultBatchOwnerInput & { batchId: string; dispatchId: string },
|
||||
) => Promise<boolean>;
|
||||
confirmAgentBackgroundToolResultBatch: (
|
||||
input: AgentBackgroundToolResultBatchOwnerInput & { batchId: string },
|
||||
) => Promise<boolean>;
|
||||
settleAgentTriggerHandlingOutcome: (
|
||||
input: SettleAgentTriggerHandlingOutcomeInput,
|
||||
) => Promise<boolean>;
|
||||
|
|
@ -628,6 +685,7 @@ function requireClaim(delivery: IAgentTriggerDelivery | null): AgentTriggerDeliv
|
|||
export function createAgentTriggerDeliveryMethods(
|
||||
mongoose: typeof import('mongoose'),
|
||||
deps: {
|
||||
releaseBatchProjections?: MessageMethods['releaseBackgroundToolResultClaims'];
|
||||
purgeQueuedTurnsForUser?: (user: string | Types.ObjectId) => Promise<unknown>;
|
||||
} = {},
|
||||
): AgentTriggerDeliveryMethods {
|
||||
|
|
@ -1125,6 +1183,8 @@ export function createAgentTriggerDeliveryMethods(
|
|||
input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 &&
|
||||
input.requiredWorkerCapability !==
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 &&
|
||||
input.requiredWorkerCapability !==
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3 &&
|
||||
input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 &&
|
||||
input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1 &&
|
||||
input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2
|
||||
|
|
@ -2182,38 +2242,70 @@ export function createAgentTriggerDeliveryMethods(
|
|||
},
|
||||
};
|
||||
})();
|
||||
const retired = await Delivery()
|
||||
.findOneAndUpdate(
|
||||
{
|
||||
deliveryKey: input.deliveryKey,
|
||||
'envelope.event.source.type': 'internal',
|
||||
'envelope.event.source.id': input.sourceId,
|
||||
...statusFence,
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
status: 'succeeded',
|
||||
result,
|
||||
settledAt: input.settledAt,
|
||||
expiresAt: new Date(input.settledAt.getTime() + SUCCESS_RETENTION_MS),
|
||||
laneCleanupPendingAt: input.settledAt,
|
||||
const retire = (retainBatch: boolean, discardEmptyPlan = false) =>
|
||||
Delivery()
|
||||
.findOneAndUpdate(
|
||||
{
|
||||
deliveryKey: input.deliveryKey,
|
||||
'envelope.event.source.type': 'internal',
|
||||
'envelope.event.source.id': input.sourceId,
|
||||
...statusFence,
|
||||
...(discardEmptyPlan && {
|
||||
backgroundToolResultBatch: { $exists: true },
|
||||
'backgroundToolResultBatch.members.0': { $exists: false },
|
||||
'backgroundToolResult.resultClaim.claimId': { $ne: input.deliveryKey },
|
||||
}),
|
||||
...(!discardEmptyPlan &&
|
||||
retainBatch && {
|
||||
backgroundToolResultBatch: { $exists: true },
|
||||
'backgroundToolResultBatch.proofCopiedAt': { $exists: false },
|
||||
}),
|
||||
...(!discardEmptyPlan &&
|
||||
!retainBatch && {
|
||||
$and: [
|
||||
{
|
||||
$or: [
|
||||
{ backgroundToolResultBatch: { $exists: false } },
|
||||
{ 'backgroundToolResultBatch.proofCopiedAt': { $exists: true } },
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
...(input.onlyIfUnclaimed === true
|
||||
? { 'backgroundToolResult.resultClaim': { $exists: false } }
|
||||
: {}),
|
||||
},
|
||||
$unset: {
|
||||
leaseBy: 1,
|
||||
leaseUntil: 1,
|
||||
claimToken: 1,
|
||||
capabilityStatus: 1,
|
||||
claimAvailableAt: 1,
|
||||
capabilityLeaseBy: 1,
|
||||
capabilityLeaseUntil: 1,
|
||||
capabilityClaimToken: 1,
|
||||
lastError: 1,
|
||||
{
|
||||
$set: {
|
||||
status: 'succeeded',
|
||||
result,
|
||||
settledAt: input.settledAt,
|
||||
...(!retainBatch && {
|
||||
expiresAt: new Date(input.settledAt.getTime() + SUCCESS_RETENTION_MS),
|
||||
}),
|
||||
laneCleanupPendingAt: input.settledAt,
|
||||
},
|
||||
$unset: {
|
||||
leaseBy: 1,
|
||||
leaseUntil: 1,
|
||||
claimToken: 1,
|
||||
capabilityStatus: 1,
|
||||
claimAvailableAt: 1,
|
||||
capabilityLeaseBy: 1,
|
||||
capabilityLeaseUntil: 1,
|
||||
capabilityClaimToken: 1,
|
||||
lastError: 1,
|
||||
...(retainBatch && { expiresAt: 1 }),
|
||||
...(discardEmptyPlan && { backgroundToolResultBatch: 1 }),
|
||||
},
|
||||
},
|
||||
},
|
||||
{ new: true },
|
||||
)
|
||||
.select('_id orderingKey laneCleanupPendingAt')
|
||||
.lean<Pick<IAgentTriggerDelivery, '_id' | 'orderingKey' | 'laneCleanupPendingAt'>>();
|
||||
{ new: true },
|
||||
)
|
||||
.select('_id orderingKey laneCleanupPendingAt')
|
||||
.lean<Pick<IAgentTriggerDelivery, '_id' | 'orderingKey' | 'laneCleanupPendingAt'>>();
|
||||
// A retired owner is its followers' recovery and frozen-membership record.
|
||||
// Its TTL cannot start while release or receipt-proof copying is incomplete.
|
||||
const retired = (await retire(false)) ?? (await retire(false, true)) ?? (await retire(true));
|
||||
if (retired?._id != null) {
|
||||
try {
|
||||
await fulfillLaneCleanupRequest(retired);
|
||||
|
|
@ -2268,7 +2360,7 @@ export function createAgentTriggerDeliveryMethods(
|
|||
requiredWorkerCapability: {
|
||||
$in: [
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
...BACKGROUND_RECEIPT_CAPABILITIES,
|
||||
],
|
||||
},
|
||||
'envelope.event.source.type': 'internal',
|
||||
|
|
@ -2312,7 +2404,7 @@ export function createAgentTriggerDeliveryMethods(
|
|||
requiredWorkerCapability: {
|
||||
$in: [
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
...BACKGROUND_RECEIPT_CAPABILITIES,
|
||||
],
|
||||
},
|
||||
'envelope.event.source.type': 'internal',
|
||||
|
|
@ -2366,7 +2458,7 @@ export function createAgentTriggerDeliveryMethods(
|
|||
'envelope.target.conversationId': input.conversationId,
|
||||
...(input.taskId != null && { 'envelope.event.payload.taskId': input.taskId }),
|
||||
/** Legacy rows keep results only on the parent message. */
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },
|
||||
};
|
||||
const read = (filter: FilterQuery<IAgentTriggerDelivery>) =>
|
||||
Delivery()
|
||||
|
|
@ -2568,7 +2660,7 @@ export function createAgentTriggerDeliveryMethods(
|
|||
}
|
||||
const identity = {
|
||||
deliveryKey: input.deliveryKey,
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },
|
||||
'envelope.event.source.type': 'internal',
|
||||
'envelope.event.source.id': input.sourceId,
|
||||
backgroundToolResultErasedAt: { $exists: false },
|
||||
|
|
@ -2612,7 +2704,7 @@ export function createAgentTriggerDeliveryMethods(
|
|||
const delivery = await Delivery()
|
||||
.findOne({
|
||||
deliveryKey: input.deliveryKey,
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },
|
||||
'envelope.event.source.type': 'internal',
|
||||
'envelope.event.source.id': input.sourceId,
|
||||
})
|
||||
|
|
@ -2648,7 +2740,7 @@ export function createAgentTriggerDeliveryMethods(
|
|||
}) {
|
||||
return {
|
||||
user: input.userId,
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },
|
||||
'envelope.event.source.type': 'internal',
|
||||
'envelope.event.source.id': input.sourceId,
|
||||
'envelope.target.conversationId': input.conversationId,
|
||||
|
|
@ -2769,38 +2861,592 @@ export function createAgentTriggerDeliveryMethods(
|
|||
return result == null ? { status: 'not_ready' } : { status: 'acquired', results: [result] };
|
||||
}
|
||||
|
||||
async function releaseAgentBackgroundToolResultClaims(input: {
|
||||
sourceId: string;
|
||||
userId: string;
|
||||
conversationId: string;
|
||||
parentMessageId: string;
|
||||
claimId: string;
|
||||
}): Promise<boolean> {
|
||||
async function releaseAgentBackgroundToolResultClaims(
|
||||
input: AgentBackgroundToolResultBatchReleaseInput,
|
||||
): Promise<boolean> {
|
||||
if (
|
||||
input.sourceId.length === 0 ||
|
||||
input.sourceId.length > 256 ||
|
||||
input.userId.length === 0 ||
|
||||
input.conversationId.length === 0 ||
|
||||
input.conversationId.length > 256 ||
|
||||
input.parentMessageId.length === 0 ||
|
||||
input.parentMessageId.length > 256 ||
|
||||
input.claimId.length === 0 ||
|
||||
[
|
||||
input.sourceId,
|
||||
input.userId,
|
||||
input.conversationId,
|
||||
input.parentMessageId,
|
||||
input.claimId,
|
||||
].some((value) => value.length === 0 || value.length > 256) ||
|
||||
input.claimId.length > 128
|
||||
) {
|
||||
throw new TypeError('Invalid background tool result receipt claim release');
|
||||
}
|
||||
const scope = backgroundResultIdentity(input);
|
||||
const releaseId = randomUUID();
|
||||
const releasing = await Delivery()
|
||||
.findOneAndUpdate(
|
||||
{
|
||||
...scope,
|
||||
deliveryKey: input.claimId,
|
||||
backgroundToolResultBatch: { $exists: true },
|
||||
'backgroundToolResultBatch.appliedAt': { $exists: false },
|
||||
...(input.batchId == null ? {} : { 'backgroundToolResultBatch.batchId': input.batchId }),
|
||||
...(input.deliveryClaimToken == null
|
||||
? {}
|
||||
: { capabilityClaimToken: input.deliveryClaimToken }),
|
||||
...(input.recoveryFenced === true
|
||||
? {}
|
||||
: {
|
||||
$or: [
|
||||
{ 'backgroundToolResultBatch.releasing': true },
|
||||
{ 'backgroundToolResultBatch.dispatchCount': 0 },
|
||||
...(input.dispatchId == null
|
||||
? []
|
||||
: [
|
||||
{
|
||||
'backgroundToolResultBatch.dispatchCount': 1,
|
||||
'backgroundToolResultBatch.dispatchId': input.dispatchId,
|
||||
},
|
||||
]),
|
||||
],
|
||||
}),
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
'backgroundToolResultBatch.releasing': true,
|
||||
'backgroundToolResultBatch.releaseId': releaseId,
|
||||
},
|
||||
},
|
||||
{ new: true, timestamps: false },
|
||||
)
|
||||
.select('status +backgroundToolResultBatch')
|
||||
.lean<Pick<IAgentTriggerDelivery, 'status' | 'backgroundToolResultBatch'>>();
|
||||
if (
|
||||
releasing == null &&
|
||||
(await Delivery().exists({
|
||||
...scope,
|
||||
deliveryKey: input.claimId,
|
||||
backgroundToolResultBatch: { $exists: true },
|
||||
})) != null
|
||||
)
|
||||
return false;
|
||||
const batch = releasing?.backgroundToolResultBatch;
|
||||
const batchId = batch?.batchId ?? input.batchId;
|
||||
const receiptScope = {
|
||||
...scope,
|
||||
'backgroundToolResult.resultClaim.claimId': input.claimId,
|
||||
'backgroundToolResult.resultClaim.batchId': batchId ?? null,
|
||||
};
|
||||
await Delivery().updateMany(
|
||||
{ ...scope, 'backgroundToolResult.resultClaim.claimId': input.claimId },
|
||||
{ ...receiptScope, 'backgroundToolResult.resultClaim.appliedAt': { $exists: false } },
|
||||
{ $unset: { 'backgroundToolResult.resultClaim': 1 } },
|
||||
{ timestamps: false },
|
||||
);
|
||||
return (
|
||||
if ((await Delivery().exists(receiptScope)) != null) return false;
|
||||
if (batch != null) {
|
||||
if (deps.releaseBatchProjections == null)
|
||||
throw new Error('Background batch projection cleanup is unavailable');
|
||||
if (
|
||||
!(await deps.releaseBatchProjections({
|
||||
userId: input.userId,
|
||||
conversationId: input.conversationId,
|
||||
messageId: input.parentMessageId,
|
||||
kind: 'wakeup',
|
||||
claimId: input.claimId,
|
||||
batchId: batch.batchId,
|
||||
allowMissingMessage: true,
|
||||
}))
|
||||
)
|
||||
return false;
|
||||
const clear = (retired: boolean) =>
|
||||
Delivery().updateOne(
|
||||
{
|
||||
...scope,
|
||||
deliveryKey: input.claimId,
|
||||
'backgroundToolResultBatch.batchId': batch.batchId,
|
||||
'backgroundToolResultBatch.releaseId': releaseId,
|
||||
'backgroundToolResultBatch.appliedAt': { $exists: false },
|
||||
status: retired ? 'succeeded' : { $ne: 'succeeded' },
|
||||
...(input.deliveryClaimToken == null
|
||||
? {}
|
||||
: { capabilityClaimToken: input.deliveryClaimToken }),
|
||||
},
|
||||
{
|
||||
$unset: { backgroundToolResultBatch: 1 },
|
||||
...(retired && { $max: { expiresAt: new Date(Date.now() + SUCCESS_RETENTION_MS) } }),
|
||||
},
|
||||
{ timestamps: false },
|
||||
);
|
||||
const cleared = await clear(releasing?.status === 'succeeded');
|
||||
if (cleared.matchedCount === 1) return true;
|
||||
return releasing?.status !== 'succeeded' && (await clear(true)).matchedCount === 1;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function backgroundBatchScope(
|
||||
input:
|
||||
| AgentBackgroundToolResultBatchInput
|
||||
| Omit<AgentBackgroundToolResultBatchInput, 'limit' | 'maxMetadataChars'>,
|
||||
) {
|
||||
return {
|
||||
...backgroundResultIdentity(input),
|
||||
tenantId: input.tenantId ?? null,
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
|
||||
'envelope.event.type': 'background-tool.completion',
|
||||
};
|
||||
}
|
||||
|
||||
/** Each result has its own CAS. Candidates survive a collecting crash; only
|
||||
* confirmed owners enter the frozen member list used by every retry. */
|
||||
async function claimAgentBackgroundToolResultBatch(
|
||||
input: AgentBackgroundToolResultBatchInput,
|
||||
): Promise<AgentBackgroundToolResultBatchClaim> {
|
||||
if (
|
||||
[
|
||||
input.deliveryKey,
|
||||
input.sourceId,
|
||||
input.userId,
|
||||
input.conversationId,
|
||||
input.parentMessageId,
|
||||
input.agentId,
|
||||
].some((value) => value.length === 0 || value.length > 256) ||
|
||||
input.deliveryKey.length > 128
|
||||
) {
|
||||
throw new TypeError('Invalid background result batch identity');
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(input.limit) ||
|
||||
input.limit < 1 ||
|
||||
!Number.isSafeInteger(input.maxMetadataChars) ||
|
||||
input.maxMetadataChars < 1
|
||||
) {
|
||||
throw new TypeError('Invalid background result batch bounds');
|
||||
}
|
||||
const scope = backgroundBatchScope(input);
|
||||
const projection =
|
||||
'deliveryKey requiredWorkerCapability status capabilityStatus result +backgroundToolResult +backgroundToolResultBatch';
|
||||
type Row = Pick<
|
||||
IAgentTriggerDelivery,
|
||||
| 'deliveryKey'
|
||||
| 'requiredWorkerCapability'
|
||||
| 'status'
|
||||
| 'capabilityStatus'
|
||||
| 'result'
|
||||
| 'backgroundToolResult'
|
||||
| 'backgroundToolResultBatch'
|
||||
> & { envelope?: unknown };
|
||||
const readRoot = () =>
|
||||
Delivery()
|
||||
.findOne({ ...scope, deliveryKey: input.deliveryKey })
|
||||
.select(projection + ' envelope')
|
||||
.lean<Row>();
|
||||
let root = await readRoot();
|
||||
if (root == null) {
|
||||
const gated = await Delivery().exists({
|
||||
user: input.userId,
|
||||
deliveryKey: input.deliveryKey,
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
|
||||
});
|
||||
return { status: gated == null ? 'legacy' : 'not_ready' };
|
||||
}
|
||||
if (root.backgroundToolResultBatch?.appliedAt != null) {
|
||||
if (
|
||||
root.backgroundToolResultBatch.proofCopiedAt == null &&
|
||||
!(await confirmAgentBackgroundToolResultBatch({
|
||||
...input,
|
||||
batchId: root.backgroundToolResultBatch.batchId,
|
||||
}))
|
||||
)
|
||||
return { status: 'not_ready' };
|
||||
return {
|
||||
status: 'claimed',
|
||||
claimId: input.deliveryKey,
|
||||
batchId: root.backgroundToolResultBatch.batchId,
|
||||
ownerStatus: 'applied',
|
||||
};
|
||||
}
|
||||
if (root.backgroundToolResultBatch?.releasing === true) {
|
||||
const released = await releaseAgentBackgroundToolResultClaims({
|
||||
...input,
|
||||
claimId: input.deliveryKey,
|
||||
batchId: root.backgroundToolResultBatch.batchId,
|
||||
});
|
||||
return released ? claimAgentBackgroundToolResultBatch(input) : { status: 'not_ready' };
|
||||
}
|
||||
if (root.backgroundToolResult == null) return { status: 'not_ready', waitingForResult: true };
|
||||
const claim = root.backgroundToolResult.resultClaim;
|
||||
if (
|
||||
claim?.claimId === input.deliveryKey &&
|
||||
claim.batchId !== root.backgroundToolResultBatch?.batchId
|
||||
) {
|
||||
await Delivery().updateOne(
|
||||
{
|
||||
...scope,
|
||||
deliveryKey: input.deliveryKey,
|
||||
'backgroundToolResult.resultClaim.claimId': input.deliveryKey,
|
||||
'backgroundToolResult.resultClaim.batchId': claim.batchId ?? null,
|
||||
'backgroundToolResult.resultClaim.appliedAt': { $exists: false },
|
||||
},
|
||||
{ $unset: { 'backgroundToolResult.resultClaim': 1 } },
|
||||
{ timestamps: false },
|
||||
);
|
||||
return { status: 'not_ready' };
|
||||
}
|
||||
if (claim != null && claim.claimId !== input.deliveryKey) {
|
||||
if (
|
||||
root.backgroundToolResultBatch != null &&
|
||||
root.backgroundToolResultBatch.appliedAt == null
|
||||
) {
|
||||
await releaseAgentBackgroundToolResultClaims({
|
||||
...input,
|
||||
claimId: input.deliveryKey,
|
||||
batchId: root.backgroundToolResultBatch.batchId,
|
||||
});
|
||||
}
|
||||
const owner = await Delivery()
|
||||
.findOne({ ...scope, deliveryKey: claim.claimId })
|
||||
.select(projection)
|
||||
.lean<Row>();
|
||||
const batch = owner?.backgroundToolResultBatch;
|
||||
const member = batch?.members?.includes(input.deliveryKey);
|
||||
if (member && batch?.batchId === claim.batchId && batch?.appliedAt != null) {
|
||||
// Retirement removed this owner from the queue. Its followers must
|
||||
// finish the proof copy before they can all settle and abandon it.
|
||||
if (
|
||||
batch.proofCopiedAt == null &&
|
||||
!(await confirmAgentBackgroundToolResultBatch({
|
||||
...input,
|
||||
deliveryKey: claim.claimId,
|
||||
batchId: batch.batchId,
|
||||
}))
|
||||
)
|
||||
return { status: 'not_ready' };
|
||||
return {
|
||||
status: 'claimed',
|
||||
claimId: claim.claimId,
|
||||
batchId: claim.batchId!,
|
||||
ownerStatus: 'applied',
|
||||
};
|
||||
}
|
||||
if (claim.appliedAt != null)
|
||||
return {
|
||||
status: 'claimed',
|
||||
claimId: claim.claimId,
|
||||
batchId: claim.batchId!,
|
||||
ownerStatus: 'applied',
|
||||
};
|
||||
// A stale collector may finish a CAS after another collector froze a subset.
|
||||
if (
|
||||
(batch?.members != null && !member) ||
|
||||
(batch != null && batch.batchId !== claim.batchId)
|
||||
) {
|
||||
await Delivery().updateOne(
|
||||
{
|
||||
...scope,
|
||||
deliveryKey: input.deliveryKey,
|
||||
'backgroundToolResult.resultClaim.claimId': claim.claimId,
|
||||
'backgroundToolResult.resultClaim.batchId': claim.batchId ?? null,
|
||||
'backgroundToolResult.resultClaim.appliedAt': { $exists: false },
|
||||
},
|
||||
{ $unset: { 'backgroundToolResult.resultClaim': 1 } },
|
||||
{ timestamps: false },
|
||||
);
|
||||
return { status: 'not_ready' };
|
||||
}
|
||||
const retired =
|
||||
owner?.result != null &&
|
||||
typeof owner.result === 'object' &&
|
||||
'backgroundToolCompletionRetired' in owner.result;
|
||||
const recoverable =
|
||||
owner == null ||
|
||||
DEAD_STATUSES.includes(owner.status) ||
|
||||
owner.capabilityStatus === 'dead' ||
|
||||
retired;
|
||||
return {
|
||||
status: 'claimed',
|
||||
claimId: claim.claimId,
|
||||
batchId: claim.batchId!,
|
||||
ownerStatus: recoverable ? 'recoverable' : 'pending',
|
||||
};
|
||||
}
|
||||
if (root.backgroundToolResultBatch == null) {
|
||||
const candidates =
|
||||
input.limit === 1
|
||||
? []
|
||||
: await Delivery()
|
||||
.find({
|
||||
...scope,
|
||||
deliveryKey: { $ne: input.deliveryKey },
|
||||
status: { $in: UNDELIVERED_STATUSES },
|
||||
capabilityStatus: { $ne: 'dead' },
|
||||
backgroundToolResult: { $exists: true },
|
||||
'backgroundToolResult.resultClaim': { $exists: false },
|
||||
})
|
||||
.select('deliveryKey envelope +backgroundToolResult')
|
||||
.sort({ createdAt: 1, _id: 1 })
|
||||
.limit(Math.min(input.limit, 32) - 1)
|
||||
.lean<Row[]>();
|
||||
const requested = projectBackgroundResult(root);
|
||||
if (requested == null) return { status: 'not_ready' };
|
||||
const metadataCost = (result: NonNullable<ReturnType<typeof projectBackgroundResult>>) =>
|
||||
JSON.stringify(backgroundResultMetadata(result)).length + 1;
|
||||
let remaining = input.maxMetadataChars - metadataCost(requested) - 1;
|
||||
if (remaining < 0) throw new TypeError('Background result metadata exceeds its input budget');
|
||||
const batchId = randomUUID();
|
||||
const keys = [input.deliveryKey];
|
||||
for (const candidate of candidates) {
|
||||
const result = projectBackgroundResult(candidate);
|
||||
if (result == null) continue;
|
||||
const cost = metadataCost(result);
|
||||
if (cost > remaining) continue;
|
||||
remaining -= cost;
|
||||
keys.push(candidate.deliveryKey);
|
||||
}
|
||||
const planned = await Delivery().updateOne(
|
||||
{
|
||||
...scope,
|
||||
deliveryKey: input.deliveryKey,
|
||||
backgroundToolResultBatch: { $exists: false },
|
||||
status: { $in: UNDELIVERED_STATUSES },
|
||||
capabilityStatus: { $ne: 'dead' },
|
||||
...(input.deliveryClaimToken == null
|
||||
? {}
|
||||
: { capabilityClaimToken: input.deliveryClaimToken }),
|
||||
$or: [
|
||||
{ 'backgroundToolResult.resultClaim': { $exists: false } },
|
||||
{ 'backgroundToolResult.resultClaim.claimId': input.deliveryKey },
|
||||
],
|
||||
},
|
||||
{ $set: { backgroundToolResultBatch: { batchId, dispatchCount: 0, candidates: keys } } },
|
||||
{ timestamps: false },
|
||||
);
|
||||
root =
|
||||
planned.modifiedCount === 1
|
||||
? { ...root, backgroundToolResultBatch: { batchId, dispatchCount: 0, candidates: keys } }
|
||||
: await readRoot();
|
||||
if (root == null) return { status: 'not_ready' };
|
||||
}
|
||||
let batch = root.backgroundToolResultBatch;
|
||||
if (batch == null) return { status: 'not_ready' };
|
||||
if (batch.members == null) {
|
||||
// Retirement of a collecting-only plan contends with this root CAS.
|
||||
// No sibling can depend on the owner until its own receipt is claimed.
|
||||
if (root.backgroundToolResult?.resultClaim?.claimId !== input.deliveryKey) {
|
||||
const claimedRoot = await Delivery().updateOne(
|
||||
{
|
||||
...scope,
|
||||
deliveryKey: input.deliveryKey,
|
||||
status: { $in: UNDELIVERED_STATUSES },
|
||||
capabilityStatus: { $ne: 'dead' },
|
||||
'backgroundToolResultBatch.batchId': batch.batchId,
|
||||
'backgroundToolResultBatch.releasing': { $ne: true },
|
||||
'backgroundToolResult.resultClaim': { $exists: false },
|
||||
...(input.deliveryClaimToken == null
|
||||
? {}
|
||||
: { capabilityClaimToken: input.deliveryClaimToken }),
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
'backgroundToolResult.resultClaim': {
|
||||
kind: 'wakeup',
|
||||
claimId: input.deliveryKey,
|
||||
claimedAt: new Date(),
|
||||
batchId: batch.batchId,
|
||||
},
|
||||
},
|
||||
},
|
||||
{ timestamps: false },
|
||||
);
|
||||
if (claimedRoot.matchedCount !== 1) return { status: 'not_ready' };
|
||||
}
|
||||
await Delivery().updateMany(
|
||||
{
|
||||
...scope,
|
||||
deliveryKey: { $in: batch.candidates },
|
||||
status: { $in: UNDELIVERED_STATUSES },
|
||||
capabilityStatus: { $ne: 'dead' },
|
||||
backgroundToolResult: { $exists: true },
|
||||
'backgroundToolResult.resultClaim': { $exists: false },
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
'backgroundToolResult.resultClaim': {
|
||||
kind: 'wakeup',
|
||||
claimId: input.deliveryKey,
|
||||
claimedAt: new Date(),
|
||||
batchId: batch.batchId,
|
||||
},
|
||||
},
|
||||
},
|
||||
{ timestamps: false },
|
||||
);
|
||||
const owned = await Delivery()
|
||||
.find({
|
||||
...scope,
|
||||
deliveryKey: { $in: batch.candidates },
|
||||
'backgroundToolResult.resultClaim.claimId': input.deliveryKey,
|
||||
'backgroundToolResult.resultClaim.batchId': batch.batchId,
|
||||
})
|
||||
.select('deliveryKey')
|
||||
.lean<Pick<IAgentTriggerDelivery, 'deliveryKey'>[]>();
|
||||
const owners = new Set(owned.map((row) => row.deliveryKey));
|
||||
const members = batch.candidates.filter((key) => owners.has(key));
|
||||
if (!owners.has(input.deliveryKey)) {
|
||||
await releaseAgentBackgroundToolResultClaims({
|
||||
...input,
|
||||
claimId: input.deliveryKey,
|
||||
batchId: batch.batchId,
|
||||
});
|
||||
return { status: 'not_ready' };
|
||||
}
|
||||
const frozen = await Delivery().updateOne(
|
||||
{
|
||||
...scope,
|
||||
deliveryKey: input.deliveryKey,
|
||||
'backgroundToolResult.resultClaim.claimId': input.deliveryKey,
|
||||
'backgroundToolResultBatch.batchId': batch.batchId,
|
||||
'backgroundToolResultBatch.releasing': { $ne: true },
|
||||
...(input.deliveryClaimToken == null
|
||||
? {}
|
||||
: { capabilityClaimToken: input.deliveryClaimToken }),
|
||||
'backgroundToolResultBatch.members': { $exists: false },
|
||||
},
|
||||
{ $set: { 'backgroundToolResultBatch.members': members } },
|
||||
{ timestamps: false },
|
||||
);
|
||||
if (frozen.modifiedCount === 1) {
|
||||
batch = { ...batch, members };
|
||||
} else {
|
||||
root = await readRoot();
|
||||
batch = root?.backgroundToolResultBatch;
|
||||
}
|
||||
}
|
||||
if (batch?.members == null || !batch.members.includes(input.deliveryKey))
|
||||
return { status: 'not_ready' };
|
||||
const rows = await Delivery()
|
||||
.find({
|
||||
...scope,
|
||||
deliveryKey: { $in: batch.members },
|
||||
'backgroundToolResult.resultClaim.claimId': input.deliveryKey,
|
||||
'backgroundToolResult.resultClaim.batchId': batch.batchId,
|
||||
})
|
||||
.select('deliveryKey envelope +backgroundToolResult')
|
||||
.lean<Row[]>();
|
||||
const byKey = new Map(rows.map((row) => [row.deliveryKey, projectBackgroundResult(row)]));
|
||||
const results = batch.members.map((key) => byKey.get(key));
|
||||
if (results.some((result) => result == null)) return { status: 'not_ready' };
|
||||
if (
|
||||
input.deliveryClaimToken != null &&
|
||||
(await Delivery().exists({
|
||||
...scope,
|
||||
'backgroundToolResult.resultClaim.claimId': input.claimId,
|
||||
deliveryKey: input.deliveryKey,
|
||||
capabilityClaimToken: input.deliveryClaimToken,
|
||||
'backgroundToolResultBatch.batchId': batch.batchId,
|
||||
'backgroundToolResultBatch.releasing': { $ne: true },
|
||||
})) == null
|
||||
) {
|
||||
return { status: 'not_ready' };
|
||||
}
|
||||
return {
|
||||
status: 'acquired',
|
||||
batchId: batch.batchId,
|
||||
results: results.filter((result): result is NonNullable<typeof result> => result != null),
|
||||
};
|
||||
}
|
||||
|
||||
async function getAgentBackgroundToolResultBatch(
|
||||
input: AgentBackgroundToolResultBatchOwnerInput,
|
||||
): Promise<IAgentTriggerDelivery['backgroundToolResultBatch'] | null> {
|
||||
const row = await Delivery()
|
||||
.findOne({ ...backgroundResultIdentity(input), deliveryKey: input.deliveryKey })
|
||||
.select('+backgroundToolResultBatch')
|
||||
.lean<Pick<IAgentTriggerDelivery, 'backgroundToolResultBatch'>>();
|
||||
return row?.backgroundToolResultBatch ?? null;
|
||||
}
|
||||
|
||||
async function beginAgentBackgroundToolResultBatchDispatch(
|
||||
input: AgentBackgroundToolResultBatchOwnerInput & { batchId: string; dispatchId: string },
|
||||
): Promise<boolean> {
|
||||
const marked = await Delivery().updateOne(
|
||||
{
|
||||
...backgroundResultIdentity(input),
|
||||
deliveryKey: input.deliveryKey,
|
||||
status: { $in: UNDELIVERED_STATUSES },
|
||||
capabilityStatus: { $ne: 'dead' },
|
||||
'backgroundToolResultBatch.batchId': input.batchId,
|
||||
'backgroundToolResultBatch.members.0': { $exists: true },
|
||||
'backgroundToolResultBatch.releasing': { $ne: true },
|
||||
'backgroundToolResultBatch.appliedAt': { $exists: false },
|
||||
...(input.deliveryClaimToken == null
|
||||
? {}
|
||||
: { capabilityClaimToken: input.deliveryClaimToken }),
|
||||
},
|
||||
{
|
||||
$inc: { 'backgroundToolResultBatch.dispatchCount': 1 },
|
||||
$set: { 'backgroundToolResultBatch.dispatchId': input.dispatchId },
|
||||
},
|
||||
{ timestamps: false },
|
||||
);
|
||||
return marked.matchedCount === 1;
|
||||
}
|
||||
|
||||
/** Admission proof precedes follower settlement, including lost HTTP replies. */
|
||||
async function confirmAgentBackgroundToolResultBatch(
|
||||
input: AgentBackgroundToolResultBatchOwnerInput & { batchId: string },
|
||||
): Promise<boolean> {
|
||||
const scope = backgroundResultIdentity(input);
|
||||
const root = await Delivery()
|
||||
.findOneAndUpdate(
|
||||
{
|
||||
...scope,
|
||||
deliveryKey: input.deliveryKey,
|
||||
'backgroundToolResultBatch.members.0': { $exists: true },
|
||||
'backgroundToolResultBatch.batchId': input.batchId,
|
||||
'backgroundToolResultBatch.releasing': { $ne: true },
|
||||
},
|
||||
{ $min: { 'backgroundToolResultBatch.appliedAt': new Date() } },
|
||||
{ new: true, timestamps: false },
|
||||
)
|
||||
.select('status +backgroundToolResultBatch')
|
||||
.lean<Pick<IAgentTriggerDelivery, 'status' | 'backgroundToolResultBatch'>>();
|
||||
if (root?.backgroundToolResultBatch?.appliedAt == null) return false;
|
||||
await Delivery().updateMany(
|
||||
{
|
||||
...scope,
|
||||
deliveryKey: { $in: root.backgroundToolResultBatch.members },
|
||||
'backgroundToolResult.resultClaim.claimId': input.deliveryKey,
|
||||
'backgroundToolResult.resultClaim.batchId': input.batchId,
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
'backgroundToolResult.resultClaim.appliedAt': root.backgroundToolResultBatch.appliedAt,
|
||||
},
|
||||
},
|
||||
{ timestamps: false },
|
||||
);
|
||||
if (
|
||||
(await Delivery().exists({
|
||||
...scope,
|
||||
'backgroundToolResult.resultClaim.claimId': input.deliveryKey,
|
||||
'backgroundToolResult.resultClaim.batchId': input.batchId,
|
||||
'backgroundToolResult.resultClaim.appliedAt': { $exists: false },
|
||||
})) != null
|
||||
)
|
||||
return false;
|
||||
const finish = (retired: boolean) =>
|
||||
Delivery().updateOne(
|
||||
{
|
||||
...scope,
|
||||
deliveryKey: input.deliveryKey,
|
||||
'backgroundToolResultBatch.batchId': input.batchId,
|
||||
'backgroundToolResultBatch.appliedAt': { $exists: true },
|
||||
status: retired ? 'succeeded' : { $ne: 'succeeded' },
|
||||
},
|
||||
{
|
||||
$min: { 'backgroundToolResultBatch.proofCopiedAt': new Date() },
|
||||
...(retired && { $max: { expiresAt: new Date(Date.now() + SUCCESS_RETENTION_MS) } }),
|
||||
},
|
||||
{ timestamps: false },
|
||||
);
|
||||
const finished = await finish(root.status === 'succeeded');
|
||||
if (finished.matchedCount === 1) return true;
|
||||
return root.status !== 'succeeded' && (await finish(true)).matchedCount === 1;
|
||||
}
|
||||
|
||||
async function settleAgentTriggerHandlingOutcome(
|
||||
|
|
@ -3715,8 +4361,7 @@ export function createAgentTriggerDeliveryMethods(
|
|||
const error = normalizeFailure(input.error);
|
||||
const receiptFence = recoverReceipt
|
||||
? {
|
||||
requiredWorkerCapability:
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },
|
||||
backgroundToolResult: { $exists: true },
|
||||
}
|
||||
: {};
|
||||
|
|
@ -4273,7 +4918,11 @@ export function createAgentTriggerDeliveryMethods(
|
|||
},
|
||||
{
|
||||
$set: { backgroundToolResultErasedAt: erasedAt },
|
||||
$unset: { backgroundToolResult: 1, backgroundToolResultDeletionPendingAt: 1 },
|
||||
$unset: {
|
||||
backgroundToolResult: 1,
|
||||
backgroundToolResultBatch: 1,
|
||||
backgroundToolResultDeletionPendingAt: 1,
|
||||
},
|
||||
},
|
||||
{ timestamps: false },
|
||||
);
|
||||
|
|
@ -4287,7 +4936,7 @@ export function createAgentTriggerDeliveryMethods(
|
|||
await Delivery().updateMany(
|
||||
{
|
||||
user,
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },
|
||||
'envelope.target.conversationId': { $in: conversationIds },
|
||||
backgroundToolResultErasedAt: { $exists: false },
|
||||
},
|
||||
|
|
@ -4316,6 +4965,10 @@ export function createAgentTriggerDeliveryMethods(
|
|||
getAgentBackgroundToolResult,
|
||||
getAgentBackgroundToolResultClaim,
|
||||
claimAgentBackgroundToolResults,
|
||||
claimAgentBackgroundToolResultBatch,
|
||||
confirmAgentBackgroundToolResultBatch,
|
||||
beginAgentBackgroundToolResultBatchDispatch,
|
||||
getAgentBackgroundToolResultBatch,
|
||||
releaseAgentBackgroundToolResultClaims,
|
||||
settleAgentTriggerHandlingOutcome,
|
||||
admitAgentEventActorAction,
|
||||
|
|
|
|||
|
|
@ -168,6 +168,8 @@ const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(
|
|||
kind: { type: String, enum: ['wakeup'], required: true },
|
||||
claimId: { type: String, required: true, maxlength: 128 },
|
||||
claimedAt: { type: Date, required: true },
|
||||
appliedAt: { type: Date },
|
||||
batchId: { type: String, maxlength: 128 },
|
||||
},
|
||||
{ _id: false },
|
||||
),
|
||||
|
|
@ -178,6 +180,23 @@ const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(
|
|||
),
|
||||
select: false,
|
||||
},
|
||||
backgroundToolResultBatch: {
|
||||
type: new Schema(
|
||||
{
|
||||
batchId: { type: String, required: true, maxlength: 128 },
|
||||
dispatchCount: { type: Number, required: true, min: 0 },
|
||||
dispatchId: { type: String, maxlength: 128 },
|
||||
releaseId: { type: String, maxlength: 128 },
|
||||
proofCopiedAt: { type: Date },
|
||||
candidates: { type: [String], required: true },
|
||||
releasing: { type: Boolean },
|
||||
members: { type: [String], default: undefined },
|
||||
appliedAt: { type: Date },
|
||||
},
|
||||
{ _id: false },
|
||||
),
|
||||
select: false,
|
||||
},
|
||||
backgroundToolResultErasedAt: { type: Date, select: false },
|
||||
backgroundToolResultDeletionPendingAt: { type: Date, select: false },
|
||||
attempts: { type: Number, required: true, default: 0, min: 0 },
|
||||
|
|
|
|||
|
|
@ -17,6 +17,8 @@ export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 =
|
|||
'background_tool_completion_v1';
|
||||
export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 =
|
||||
'background_tool_completion_receipt_v2';
|
||||
export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3 =
|
||||
'background_tool_completion_batch_v3';
|
||||
export const AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1 = 'agent_queued_turn_v1';
|
||||
export const AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2 = 'agent_queued_turn_v2';
|
||||
export const AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS: number = 64 * 1024;
|
||||
|
|
@ -92,6 +94,8 @@ export interface AgentBackgroundToolResultReceipt {
|
|||
kind: 'wakeup';
|
||||
claimId: string;
|
||||
claimedAt: Date;
|
||||
appliedAt?: Date;
|
||||
batchId?: string;
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -128,6 +132,18 @@ export interface IAgentTriggerDelivery {
|
|||
producerLeaseUntil?: Date;
|
||||
/** Durable source of truth for a background completion. */
|
||||
backgroundToolResult?: AgentBackgroundToolResultReceipt;
|
||||
/** Receipt batching is separate from event-envelope coalescing. */
|
||||
backgroundToolResultBatch?: {
|
||||
batchId: string;
|
||||
dispatchCount: number;
|
||||
dispatchId?: string;
|
||||
releaseId?: string;
|
||||
proofCopiedAt?: Date;
|
||||
candidates: string[];
|
||||
members?: string[];
|
||||
appliedAt?: Date;
|
||||
releasing?: true;
|
||||
};
|
||||
/** Conversation-deletion fence preventing a late producer from restoring private output. */
|
||||
backgroundToolResultErasedAt?: Date;
|
||||
backgroundToolResultDeletionPendingAt?: Date;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue