From 7238be3e5ea1eceaa9cc59496625c8ea55eb60a4 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 24 Sep 2026 22:28:28 -0400 Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=20fix:=20Deliver=20Waiting=20Completi?= =?UTF-8?q?on=20Wake-ups=20the=20Moment=20They=20Are=20Ready=20(#16339)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ๐Ÿข fix: Back Off Waiting Completion Wake-ups and Deliver Them When Ready * fix: Signal every readiness path and make the completion wait cap configurable * fix: Mark held completion deliveries instead of pipeline expedite, and announce store-won approval expiry * fix: Scope settle expedites to the resumed conversation and close the remaining signal gaps * fix: Announce a won approval expiry once and release subagent wake-up registrations * ๐Ÿงช ci: Count System Instructions in Mock Model Usage (#16350) The e2e fake model reported input tokens for the chat messages alone, because the SDK hands a test override model the pruned messages without the systemRunnable pipe. Once #16301 gave every memory-enabled chat a system prompt, the calibrated context snapshot put used tokens below the instructions and the gauge dropped its Messages row. Count input over the complete prompt, as a real provider bills it. --------- Co-authored-by: Lia --- api/server/services/Agents/triggers.js | 2 + .../Endpoints/agents/backgroundCompletion.js | 2 + .../Endpoints/agents/subagentThreadStore.js | 4 +- .../agents/subagentThreadStore.spec.js | 11 + .../api/src/agents/backgroundCompletion.ts | 3 + .../agents/backgroundCompletionWakeup.spec.ts | 19 + .../src/agents/backgroundCompletionWakeup.ts | 2 + .../src/agents/handlers.background.spec.ts | 38 ++ packages/api/src/agents/handlers.ts | 2 + .../agents/subagentCompletionWakeup.spec.ts | 4 +- .../src/agents/subagentCompletionWakeup.ts | 5 +- .../api/src/agents/subagentThreads.spec.ts | 122 ++++++- packages/api/src/agents/subagentThreads.ts | 57 ++- packages/api/src/agents/triggers/README.md | 8 +- .../api/src/agents/triggers/engine.spec.ts | 86 +++++ packages/api/src/agents/triggers/engine.ts | 2 +- .../agents/triggers/service.delivery.spec.ts | 137 +++++++ packages/api/src/agents/triggers/service.ts | 71 +++- .../api/src/stream/GenerationJobManager.ts | 92 ++++- .../__tests__/generationSettled.spec.ts | 145 ++++++++ .../stream/__tests__/idempotencyClaim.spec.ts | 120 +++--- packages/api/src/stream/index.ts | 2 + .../src/methods/triggerDelivery.spec.ts | 344 +++++++++++++++++- .../src/methods/triggerDelivery.ts | 254 +++++++------ .../src/schema/triggerDelivery.ts | 3 +- .../data-schemas/src/types/triggerDelivery.ts | 2 + 26 files changed, 1374 insertions(+), 163 deletions(-) create mode 100644 packages/api/src/stream/__tests__/generationSettled.spec.ts diff --git a/api/server/services/Agents/triggers.js b/api/server/services/Agents/triggers.js index 329bfd4635..f8944c6ea8 100644 --- a/api/server/services/Agents/triggers.js +++ b/api/server/services/Agents/triggers.js @@ -53,6 +53,7 @@ service = createAgentTriggerService({ ), isPrincipalActive: methods.isAgentTriggerPrincipalActive, supportsDetachedActionCompletion: () => GenerationJobManager.supportsDetachedAgentEventActions, + subscribeGenerationSettled: (listener) => GenerationJobManager.onGenerationSettled(listener), settleSourceBeforeDeadLetter: queuedTurnLifecycle.settleBeforeDeadLetter, prepareContinue: createAgentContinuationResolver({ eventActor: eventActorAdapter, @@ -92,6 +93,7 @@ module.exports = { retireAgentTrigger: service.retire, renewAgentTriggerProducerLease: service.renewProducerLease, persistAgentBackgroundToolResult: service.persistBackgroundToolResult, + expediteCompletionWakeups: service.expediteCompletionWakeups, getAgentBackgroundToolResultClaim: service.getBackgroundToolResultClaim, releaseAgentBackgroundToolResultClaims: service.releaseBackgroundToolResultClaims, drainAgentTriggerDeliveriesForUser: service.drainUser, diff --git a/api/server/services/Endpoints/agents/backgroundCompletion.js b/api/server/services/Endpoints/agents/backgroundCompletion.js index 8ca1cae6ef..da93d98242 100644 --- a/api/server/services/Endpoints/agents/backgroundCompletion.js +++ b/api/server/services/Endpoints/agents/backgroundCompletion.js @@ -11,6 +11,7 @@ const { releaseAgentBackgroundToolResultClaims, renewAgentTriggerProducerLease, retireAgentTrigger, + expediteCompletionWakeups, } = require('../../Agents/triggers'); const preregisterBackgroundToolCompletion = createBackgroundToolCompletionWakeupHandler( @@ -19,6 +20,7 @@ const preregisterBackgroundToolCompletion = createBackgroundToolCompletionWakeup renewAgentTriggerProducerLease, (deliveryKey, sourceId, result) => persistAgentBackgroundToolResult({ deliveryKey, sourceId, result }), + (deliveryKey) => expediteCompletionWakeups({ deliveryKeys: [deliveryKey] }), ); function createBackgroundToolResultPersistence({ req, updateToolCallResult }) { diff --git a/api/server/services/Endpoints/agents/subagentThreadStore.js b/api/server/services/Endpoints/agents/subagentThreadStore.js index b33b7df541..d98f0f275f 100644 --- a/api/server/services/Endpoints/agents/subagentThreadStore.js +++ b/api/server/services/Endpoints/agents/subagentThreadStore.js @@ -12,7 +12,7 @@ const { SubagentActivityStream, } = require('@librechat/api'); const db = require('~/models'); -const { enqueueAgentTrigger } = require('../../Agents/triggers'); +const { enqueueAgentTrigger, expediteCompletionWakeups } = require('../../Agents/triggers'); const GENERATION_DRAIN_TIMEOUT_MS = 45_000; const GENERATION_DRAIN_POLL_MS = 100; @@ -78,6 +78,8 @@ const subagentThreadTaskStore = createSubagentThreadTaskStore( releaseOwnerAdmission: db.releaseSubagentAdmission, cancelUnroutedTask: cancelUnroutedGeneration, onTaskPrepared: completionWakeupHandler, + onTaskSettled: (userId, conversationId, taskIds) => + expediteCompletionWakeups({ user: userId, conversationId, taskIds }), }, ); diff --git a/api/server/services/Endpoints/agents/subagentThreadStore.spec.js b/api/server/services/Endpoints/agents/subagentThreadStore.spec.js index bae8a2cdeb..a7d2609468 100644 --- a/api/server/services/Endpoints/agents/subagentThreadStore.spec.js +++ b/api/server/services/Endpoints/agents/subagentThreadStore.spec.js @@ -45,6 +45,7 @@ jest.mock('~/models', () => ({ jest.mock('../../Agents/triggers', () => ({ enqueueAgentTrigger: jest.fn(), + expediteCompletionWakeups: jest.fn(), })); const { @@ -80,6 +81,16 @@ describe('subagent thread Redis lifecycle', () => { expect(mockCompletionWakeupHandler).toHaveBeenCalledWith({ taskId: 'task-1' }); }); + it('expedites only the settled task identities in their parent conversation', () => { + const { expediteCompletionWakeups } = require('../../Agents/triggers'); + taskStoreOptions.onTaskSettled('user-1', 'parent-1', ['task-1', 'recovered-task']); + expect(expediteCompletionWakeups).toHaveBeenCalledWith({ + user: 'user-1', + conversationId: 'parent-1', + taskIds: ['task-1', 'recovered-task'], + }); + }); + it('registers local task-store quiescence independently of optional Redis setup', () => { expect(taskStoreShutdownRegistration).toEqual([ 'subagent task store', diff --git a/packages/api/src/agents/backgroundCompletion.ts b/packages/api/src/agents/backgroundCompletion.ts index 8ef1d74cca..02d57cb7d8 100644 --- a/packages/api/src/agents/backgroundCompletion.ts +++ b/packages/api/src/agents/backgroundCompletion.ts @@ -42,6 +42,9 @@ export interface BackgroundToolWakeupAdmission { * Manual polling requires an atomic unclaimed-only transition: once a * resolver owns a lease, its prepared continuation cannot be cancelled. */ retire: (reason: string, options?: BackgroundToolWakeupRetireOptions) => Promise; + /** Best effort: makes a waiting delivery claimable now that a result it can + * consume exists, e.g. a parent-message projection written without a receipt. */ + expedite?: () => void; } /** Durable ownership repair used by a manual poll after an automatic diff --git a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts index ceb95fe540..a30d5a4b49 100644 --- a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts +++ b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts @@ -112,6 +112,25 @@ describe('background tool completion wakeups', () => { jest.useRealTimers(); }); + it('expedites its own delivery when a result it can consume appears', async () => { + const expedite = jest.fn(); + const notify = createBackgroundToolCompletionWakeupHandler( + async () => ({ deliveryKey: 'delivery-key-1' }), + async () => true, + async () => true, + undefined, + expedite, + ); + + const admission = await notify(registration()); + if (admission === false) { + throw new Error('Expected an admission'); + } + admission.expedite?.(); + + expect(expedite).toHaveBeenCalledWith('delivery-key-1'); + }); + it('pre-registers the exact task on the invoking response branch', async () => { const enqueue = jest.fn< ReturnType, diff --git a/packages/api/src/agents/backgroundCompletionWakeup.ts b/packages/api/src/agents/backgroundCompletionWakeup.ts index 1ca6fca26c..e5ef7ca631 100644 --- a/packages/api/src/agents/backgroundCompletionWakeup.ts +++ b/packages/api/src/agents/backgroundCompletionWakeup.ts @@ -550,6 +550,7 @@ export function createBackgroundToolCompletionWakeupHandler( retire: RetireBackgroundToolCompletion, renewProducerLease: RenewBackgroundToolCompletionProducerLease, persistResult?: PersistBackgroundToolCompletionResult, + expedite?: (deliveryKey: string) => void, ): ( registration: BackgroundToolWakeupRegistration, ) => Promise { @@ -613,6 +614,7 @@ export function createBackgroundToolCompletionWakeupHandler( options == null ? retire(admitted.deliveryKey, BACKGROUND_TOOL_COMPLETION_SOURCE, reason) : retire(admitted.deliveryKey, BACKGROUND_TOOL_COMPLETION_SOURCE, reason, options), + ...(expedite == null ? {} : { expedite: () => expedite(admitted.deliveryKey) }), }; }; } diff --git a/packages/api/src/agents/handlers.background.spec.ts b/packages/api/src/agents/handlers.background.spec.ts index 645a473cf9..37ad6535d6 100644 --- a/packages/api/src/agents/handlers.background.spec.ts +++ b/packages/api/src/agents/handlers.background.spec.ts @@ -388,6 +388,44 @@ describe('createToolExecuteHandler โ€” background tool calls', () => { expect(JSON.parse(dispatch.content).message).not.toContain('host will resume you'); }); + it('expedites the delivery when only the parent-message projection was persisted', async () => { + const tool = makeSearchTool({ calls: 0 }); + const expedite = jest.fn(); + const retire = jest.fn(async () => true); + const handler = createToolExecuteHandler({ + loadTools: async () => ({ loadedTools: [tool] }), + backgroundToolCompletion: { + preregister: jest.fn(async () => ({ + renew: jest.fn(async () => true), + persistResult: jest.fn(async () => false), + retire, + expedite, + })), + persist: jest.fn(async () => true), + claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })), + }, + }); + + await runBatch(handler, { + toolCalls: [ + { + id: 'call-projection-only', + name: tool.name, + args: { q: 'projection', run_in_background: true }, + stepId: 'step-projection-only', + }, + ], + agentId: 'agent_parent_1', + configurable: buildConfig([tool.name]), + metadata: { thread_id: 'exec_convo', run_id: 'response-1' }, + }); + await flushMicrotasks(); + await flushMicrotasks(); + + expect(expedite).toHaveBeenCalledTimes(1); + expect(retire).not.toHaveBeenCalled(); + }); + it('retires a preregistered delivery when terminal persistence fails', async () => { const tool = makeSearchTool({ calls: 0 }); const retire = jest.fn(async () => true); diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts index f991418273..edbab15caa 100644 --- a/packages/api/src/agents/handlers.ts +++ b/packages/api/src/agents/handlers.ts @@ -6130,6 +6130,8 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand 'background tool result was not persisted', 'definite', ); + } else if (deliveryReady && !durableReceiptReady) { + completionAdmission?.expedite?.(); } } catch (persistError) { if (!durableReceiptReady) { diff --git a/packages/api/src/agents/subagentCompletionWakeup.spec.ts b/packages/api/src/agents/subagentCompletionWakeup.spec.ts index 16c54710c9..0f09627557 100644 --- a/packages/api/src/agents/subagentCompletionWakeup.spec.ts +++ b/packages/api/src/agents/subagentCompletionWakeup.spec.ts @@ -60,7 +60,7 @@ describe('createSubagentCompletionWakeupHandler', () => { const enqueue = enqueueMock(); const notify = createSubagentCompletionWakeupHandler(enqueue); - await notify(registration()); + await expect(notify(registration())).resolves.toBe(true); expect(enqueue).toHaveBeenCalledTimes(1); const [envelopeValue, options] = enqueue.mock.calls[0]!; @@ -112,7 +112,7 @@ describe('createSubagentCompletionWakeupHandler', () => { const enqueue = enqueueMock(); const notify = createSubagentCompletionWakeupHandler(enqueue); - await notify(registration({ parentAgentId: undefined })); + await expect(notify(registration({ parentAgentId: undefined }))).resolves.toBe(false); expect(enqueue).not.toHaveBeenCalled(); }); diff --git a/packages/api/src/agents/subagentCompletionWakeup.ts b/packages/api/src/agents/subagentCompletionWakeup.ts index 93285187bd..0b70b205a2 100644 --- a/packages/api/src/agents/subagentCompletionWakeup.ts +++ b/packages/api/src/agents/subagentCompletionWakeup.ts @@ -810,11 +810,11 @@ export function createSubagentCompletionWakeupResolver({ * simply defers until the terminal child message exists. */ export function createSubagentCompletionWakeupHandler( enqueue: EnqueueAgentTrigger, -): (registration: SubagentTaskWakeupRegistration) => Promise { +): (registration: SubagentTaskWakeupRegistration) => Promise { return async (registration) => { const parentAgentId = registration.parentAgentId?.trim(); if (parentAgentId == null || parentAgentId === '' || isEphemeralAgentId(parentAgentId)) { - return; + return false; } const eventId = registration.taskId; const envelope = createAgentTriggerEnvelope({ @@ -850,5 +850,6 @@ export function createSubagentCompletionWakeupHandler( Math.max(Date.now(), registration.createdAt) + WAKEUP_ADMISSION_DELAY_MS, ), }); + return true; }; } diff --git a/packages/api/src/agents/subagentThreads.spec.ts b/packages/api/src/agents/subagentThreads.spec.ts index 32daa78c44..410261e711 100644 --- a/packages/api/src/agents/subagentThreads.spec.ts +++ b/packages/api/src/agents/subagentThreads.spec.ts @@ -431,6 +431,73 @@ describe('SubagentThreadTaskStore', () => { }); }); + it('announces a settled child only after its terminal message is durable', async () => { + const userId = 'settled-child-user'; + const parentConversationId = randomUUID(); + await saveParent(userId, parentConversationId); + const saveMessage = jest.spyOn(methods, 'saveMessage'); + const terminalSavedAtSettle: boolean[] = []; + const store = new SubagentThreadTaskStore(methods, { + onTaskPrepared: jest.fn(), + onTaskSettled: (settledUserId, settledConversationId, taskIds) => { + expect(taskIds).toHaveLength(1); + expect( + saveMessage.mock.calls.some( + ([, message]) => + (message as { messageId?: string }).messageId === `${taskIds[0]}:assistant`, + ), + ).toBe(true); + expect(settledUserId).toBe(userId); + expect(settledConversationId).toBe(parentConversationId); + terminalSavedAtSettle.push( + saveMessage.mock.calls.some(([, message]) => + String((message as { messageId?: string }).messageId).endsWith(':assistant'), + ), + ); + }, + }); + const config = buildSubagentThreadTaskConfig( + store, + { userId, parentConversationId }, + { completionWakeups: true }, + ); + const started = config.store.start( + taskRequest(config.scopeId, { + parentRunId: 'parent-response-1', + parentAgentId: 'agent_parent_1', + }), + ); + await waitForSettled(store, config.scopeId, started); + saveMessage.mockRestore(); + + expect(terminalSavedAtSettle).toEqual([true]); + }); + + it.each([false, true])( + 'does not announce a child without an admitted delivery (wakeups enabled: %s)', + async (completionWakeups) => { + const userId = 'poll-only-settled-user'; + const parentConversationId = randomUUID(); + await saveParent(userId, parentConversationId); + const onTaskSettled = jest.fn(); + const store = new SubagentThreadTaskStore(methods, { + onTaskPrepared: jest.fn(() => false), + onTaskSettled, + }); + const config = buildSubagentThreadTaskConfig( + store, + { userId, parentConversationId }, + { completionWakeups }, + ); + const started = config.store.start( + taskRequest(config.scopeId, { parentRunId: 'parent-response-1' }), + ); + await waitForSettled(store, config.scopeId, started); + + expect(onTaskSettled).not.toHaveBeenCalled(); + }, + ); + it('keeps subagent completion delivery poll-only when wakeups are disabled', async () => { const userId = 'poll-only-user'; const parentConversationId = randomUUID(); @@ -951,7 +1018,11 @@ describe('SubagentThreadTaskStore', () => { async (_registration: SubagentTaskWakeupRegistration) => undefined, ); const firstWorker = new SubagentThreadTaskStore(methods, { onTaskPrepared: firstWakeup }); - const secondWorker = new SubagentThreadTaskStore(methods, { onTaskPrepared: replayWakeup }); + const onReplaySettled = jest.fn(); + const secondWorker = new SubagentThreadTaskStore(methods, { + onTaskPrepared: replayWakeup, + onTaskSettled: onReplaySettled, + }); const config = buildSubagentThreadTaskConfig( firstWorker, { userId, parentConversationId }, @@ -999,6 +1070,10 @@ describe('SubagentThreadTaskStore', () => { expect(firstRun).toHaveBeenCalledTimes(1); expect(replayRun).not.toHaveBeenCalled(); + expect(onReplaySettled).toHaveBeenCalledTimes(1); + expect(onReplaySettled).toHaveBeenCalledWith(userId, parentConversationId, [ + requireAccepted(first).task.taskId, + ]); expect(firstWakeup).toHaveBeenCalledTimes(1); const firstRegistration = firstWakeup.mock.calls[0]?.[0]; const replayRegistration = replayWakeup.mock.calls[0]?.[0]; @@ -1056,8 +1131,16 @@ describe('SubagentThreadTaskStore', () => { const parentConversationId = randomUUID(); const threadId = randomUUID(); await saveParent(userId, parentConversationId); - const store = new SubagentThreadTaskStore(methods); - const config = buildSubagentThreadTaskConfig(store, { userId, parentConversationId }); + const onTaskSettled = jest.fn(); + const store = new SubagentThreadTaskStore(methods, { + onTaskPrepared: jest.fn(), + onTaskSettled, + }); + const config = buildSubagentThreadTaskConfig( + store, + { userId, parentConversationId }, + { completionWakeups: true }, + ); await methods.saveConvo( { userId }, { @@ -1095,7 +1178,7 @@ describe('SubagentThreadTaskStore', () => { }, ); const run = jest.fn(taskRequest(config.scopeId).run); - const retry = store.start( + const retry = config.store.start( taskRequest(config.scopeId, { threadId, idempotencyKey: 'abandoned-attempt', @@ -1106,6 +1189,11 @@ describe('SubagentThreadTaskStore', () => { await waitForSettled(store, config.scopeId, retry); expect(run).not.toHaveBeenCalled(); + expect(onTaskSettled).toHaveBeenCalledTimes(1); + expect(onTaskSettled).toHaveBeenCalledWith(userId, parentConversationId, [ + requireAccepted(retry).task.taskId, + 'abandoned', + ]); expect(store.claim(config.scopeId, requireAccepted(retry).task.taskId)).toMatchObject({ status: 'error', error: @@ -1116,6 +1204,32 @@ describe('SubagentThreadTaskStore', () => { '+subagentTask', ); expect(messages.map((message) => message.subagentTask?.status)).toEqual(['running', 'error']); + + const replaySettled = jest.fn(); + const replayWorker = new SubagentThreadTaskStore(methods, { + onTaskPrepared: jest.fn(), + onTaskSettled: replaySettled, + }); + const replayConfig = buildSubagentThreadTaskConfig( + replayWorker, + { userId, parentConversationId }, + { completionWakeups: true }, + ); + const replay = replayConfig.store.start( + taskRequest(replayConfig.scopeId, { + threadId, + idempotencyKey: 'abandoned-attempt', + requestFingerprint: 'same-inputs', + run, + }), + ); + await waitForSettled(replayWorker, replayConfig.scopeId, replay); + expect(replaySettled).toHaveBeenCalledTimes(1); + expect(replaySettled).toHaveBeenCalledWith(userId, parentConversationId, [ + requireAccepted(retry).task.taskId, + 'abandoned', + ]); + expect(run).not.toHaveBeenCalled(); }); it('holds one active lease per child and exposes provisional ownership safely', async () => { diff --git a/packages/api/src/agents/subagentThreads.ts b/packages/api/src/agents/subagentThreads.ts index 355ff40a0d..f3bae71a20 100644 --- a/packages/api/src/agents/subagentThreads.ts +++ b/packages/api/src/agents/subagentThreads.ts @@ -140,6 +140,8 @@ interface PreparedThread { content: string; taskId: string; parentRunId: string; + /** Original running task closed by a retry under a new terminal message id. */ + previousTaskId?: string; }; } @@ -262,7 +264,14 @@ export interface SubagentThreadTaskStoreOptions extends InMemorySubagentTaskStor taskId: string; tenantId?: string; }) => Promise; - onTaskPrepared?: (registration: SubagentTaskWakeupRegistration) => Promise | void; + onTaskPrepared?: ( + registration: SubagentTaskWakeupRegistration, + ) => Promise | boolean | void; + /** Called once a child's terminal message is durable, with the parent conversation its + * completion wake-up resumes. The child generation settles before that write, so the + * wake-up is ready only from here. Task ids include the original abandoned + * attempt when a retry persisted its result under a new id. */ + onTaskSettled?: (userId: string, parentConversationId: string, taskIds: string[]) => void; } export interface SubagentTaskWakeupRegistration { @@ -616,6 +625,9 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { private readonly releaseOwnerAdmission?: (userId: string, token: string) => Promise; private readonly cancelUnroutedTask?: SubagentThreadTaskStoreOptions['cancelUnroutedTask']; private readonly onTaskPrepared?: SubagentThreadTaskStoreOptions['onTaskPrepared']; + private readonly onTaskSettled?: SubagentThreadTaskStoreOptions['onTaskSettled']; + /** Tasks whose completion wake-up was registered; only they announce settlement. */ + private readonly wakeupTaskIds = new Set(); private taskControlTransport?: SubagentTaskControlTransport; private activityStream = new SubagentActivityStream(new InMemoryEventTransport()); @@ -658,6 +670,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { this.releaseOwnerAdmission = options.releaseOwnerAdmission; this.cancelUnroutedTask = options.cancelUnroutedTask; this.onTaskPrepared = options.onTaskPrepared; + this.onTaskSettled = options.onTaskSettled; } /** Receives payload-free authoritative transitions from the SDK task store. */ @@ -1320,6 +1333,13 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { parentRunId: prepared.replay?.parentRunId ?? request.parentRunId, createdAt: prepared.taskCreatedAt, }); + if (prepared.replay != null) { + this.notifyTaskSettled( + scope, + prepared.replay.taskId, + prepared.replay.previousTaskId, + ); + } if (runtime.signal.aborted) { throw runtime.signal.reason ?? new Error('Subagent task was cancelled.'); } @@ -1425,6 +1445,9 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { } throw new Error(publicFailureDetail(error)); } finally { + if (prepared != null) { + this.wakeupTaskIds.delete(prepared.replay?.taskId ?? runtime.taskId); + } if (prepared != null && prepared.replay == null) { this.completeActivity( lease, @@ -2902,6 +2925,10 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { replay: { status: terminal.subagentTask.status as 'completed' | 'error' | 'cancelled', taskId: canonicalTaskId, + ...(terminal.parentMessageId?.endsWith(':user') === true && + terminal.parentMessageId !== `${canonicalTaskId}:user` && { + previousTaskId: terminal.parentMessageId.slice(0, -':user'.length), + }), parentRunId: terminal.subagentTask.parentRunId ?? request.parentRunId, content: terminal.text ?? @@ -2956,6 +2983,9 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { status: 'error', content: abandonedMessage, taskId, + ...(abandoned.messageId.endsWith(':user') && { + previousTaskId: abandoned.messageId.slice(0, -':user'.length), + }), parentRunId: request.parentRunId, }, }; @@ -3192,7 +3222,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { ) { return; } - await this.onTaskPrepared({ + const admitted = await this.onTaskPrepared({ userId: scope.userId, parentConversationId: scope.parentConversationId, parentMessageId: task.parentRunId, @@ -3203,6 +3233,9 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { subagentType: request.subagentType, createdAt: task.createdAt, }); + if (admitted !== false) { + this.wakeupTaskIds.add(task.taskId); + } } private async persistCancellation( @@ -3375,6 +3408,26 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { } logger.error(`[subagentThreads] Failed to refresh ${outcome} child thread`, error); } + this.notifyTaskSettled(scope, taskId); + } + + private notifyTaskSettled( + scope: SubagentThreadScope, + taskId: string, + previousTaskId?: string, + ): void { + if (!this.wakeupTaskIds.delete(taskId)) { + return; + } + try { + this.onTaskSettled?.( + scope.userId, + scope.parentConversationId, + previousTaskId == null ? [taskId] : [taskId, previousTaskId], + ); + } catch (error) { + logger.warn('[subagentThreads] Settled-task listener failed', error); + } } private aggregateDetachedUsage(detachedUsage: UsageMetadata[]) { diff --git a/packages/api/src/agents/triggers/README.md b/packages/api/src/agents/triggers/README.md index 875d797218..5fe25e09aa 100644 --- a/packages/api/src/agents/triggers/README.md +++ b/packages/api/src/agents/triggers/README.md @@ -103,7 +103,13 @@ experimental clustered startup: `completionWaitMaxIntervalMs` caps how long a background or subagent completion waits between readiness checks while its result or parent turn is not ready. It backs off from 5 seconds by a tenth -of its age, so a long wait costs about one re-check per cap instead of one every five seconds. +of its age; a durable result and a settled generation expedite it, so the cap bounds missed signals. +Parent settlement expedites completion deliveries only in that conversation. A child's durable +terminal result expedites only its task, plus the original task when recovering an abandoned +attempt. Terminal replays signal readiness after registering their delivery, without executing the +child again. A held delivery consumes its wake marker in the same fenced write that releases it +for ordering or defers readiness. These writes use classic operators for DocumentDB compatibility; +older workers can ignore the optional marker and fall back to the configured polling interval. Setting a recovery cap to `30000` restores its original fixed recovery frequency. No stored-data migration is needed; optional activity reporting preserves the existing numeric/boolean results. diff --git a/packages/api/src/agents/triggers/engine.spec.ts b/packages/api/src/agents/triggers/engine.spec.ts index c5532179a6..697834d4f8 100644 --- a/packages/api/src/agents/triggers/engine.spec.ts +++ b/packages/api/src/agents/triggers/engine.spec.ts @@ -123,6 +123,92 @@ describe('createAgentTriggerDeliveryEngine', () => { expect(store.retry).not.toHaveBeenCalled(); }); + it('re-dispatches at once a delivery its deferral found expedited while held', async () => { + const dispatchesWithin = async (deferred: boolean | 'expedited'): Promise => { + jest.useFakeTimers(); + try { + /** Models the store: the deferred row is claimable again only when the + * deferral honored a wake marker and moved it to now. */ + let claimable = true; + const store = storeWith({ + claimNext: jest.fn(async () => { + if (!claimable) { + return null; + } + claimable = false; + return delivery(); + }), + defer: jest.fn(async () => { + claimable = deferred === 'expedited'; + return deferred; + }), + }); + let dispatches = 0; + const dispatch = jest.fn(async () => { + dispatches++; + if (dispatches > 1) { + return successResult(); + } + throw new AgentTriggerExecutionError('The parent generation has not settled yet.', { + mode: 'continue', + certainty: 'definite', + code: 'PARENT_NOT_READY', + retryable: true, + deferWithoutAttempt: true, + retryAfter: '30', + }); + }); + const engine = createAgentTriggerDeliveryEngine( + { store, dispatch, now: () => START, workerId: 'worker-1' }, + { concurrency: 1, tickMs: 60_000, maxIdleTickMs: 60_000 }, + ); + engine.start(); + await jest.advanceTimersByTimeAsync(100); + await engine.stop(); + return dispatches; + } finally { + jest.useRealTimers(); + } + }; + + await expect(dispatchesWithin(true)).resolves.toBe(1); + await expect(dispatchesWithin('expedited')).resolves.toBe(2); + }); + + it('reclaims an ordering release made due by a held wake signal without waiting for its stale deadline', async () => { + jest.useFakeTimers(); + try { + let claimable = true; + const store = storeWith({ + claimNext: jest.fn(async () => { + if (!claimable) return null; + claimable = false; + return delivery(); + }), + findEarlierUnsettled: jest + .fn() + .mockResolvedValueOnce({ availableAt: new Date(START.getTime() + 60_000) }) + .mockResolvedValue(null), + release: jest.fn(async () => { + claimable = true; + return true; + }), + }); + const dispatch = jest.fn(async () => successResult()); + const engine = createAgentTriggerDeliveryEngine( + { store, dispatch, now: () => START }, + { concurrency: 1, tickMs: 60_000, maxIdleTickMs: 60_000 }, + ); + engine.start(); + await jest.advanceTimersByTimeAsync(100); + await engine.stop(); + expect(store.release).toHaveBeenCalledTimes(1); + expect(dispatch).toHaveBeenCalledTimes(1); + } finally { + jest.useRealTimers(); + } + }); + it('runs each claim pass under the root trace context', async () => { const withContext = jest.spyOn(context, 'with'); const store = storeWith({ claimNext: jest.fn(async () => null) }); diff --git a/packages/api/src/agents/triggers/engine.ts b/packages/api/src/agents/triggers/engine.ts index ac5bf350e2..e6d6f9e8fc 100644 --- a/packages/api/src/agents/triggers/engine.ts +++ b/packages/api/src/agents/triggers/engine.ts @@ -158,7 +158,7 @@ export interface AgentTriggerDeliveryStore { claimToken: string; attempt: number; availableAt: Date; - }) => Promise; + }) => Promise; complete: (input: { id: string; workerId: string; diff --git a/packages/api/src/agents/triggers/service.delivery.spec.ts b/packages/api/src/agents/triggers/service.delivery.spec.ts index 1130b7f605..0b1278141a 100644 --- a/packages/api/src/agents/triggers/service.delivery.spec.ts +++ b/packages/api/src/agents/triggers/service.delivery.spec.ts @@ -209,6 +209,143 @@ describe('durable agent trigger service', () => { await service.stop(); }); + describe('waiting completion deliveries', () => { + const address = { address: '127.0.0.1', family: 'IPv4' as const, port: 3080 }; + const flush = () => new Promise((resolve) => setImmediate(resolve)); + const result = { status: 'completed' as const, output: 'done', settledAt: START }; + + it('expedites and claims the delivery whose background result just became durable', async () => { + const methods = deliveryMethods({ + persistAgentBackgroundToolResult: jest.fn(async () => true), + expediteAgentTriggerDeliveries: jest.fn(async () => ({ expedited: 1, held: 0 })), + }); + const service = createAgentTriggerService({ + methods, + deliveryOptions: { concurrency: 1, tickMs: 60_000 }, + }); + await service.initialize({ address }); + jest.mocked(methods.claimNextAgentTriggerDelivery).mockClear(); + + await expect( + service.persistBackgroundToolResult({ + deliveryKey: 'trigger_background', + sourceId: 'background-tool-completion', + result, + }), + ).resolves.toBe(true); + await flush(); + + expect(methods.expediteAgentTriggerDeliveries).toHaveBeenCalledWith({ + deliveryKeys: ['trigger_background'], + sourceIds: ['background-tool-completion', 'subagent-completion'], + now: expect.any(Date), + }); + expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalled(); + await service.stop(); + }); + + it('targets only named subagent completions when a child settles', async () => { + const methods = deliveryMethods({ + expediteAgentTriggerDeliveries: jest.fn(async () => ({ expedited: 1, held: 0 })), + }); + const service = createAgentTriggerService({ methods }); + await service.initialize({ address }); + service.expediteCompletionWakeups({ + user: 'user-1', + conversationId: 'parent-1', + taskIds: ['child-1'], + }); + await flush(); + expect(methods.expediteAgentTriggerDeliveries).toHaveBeenCalledWith({ + user: 'user-1', + conversationId: 'parent-1', + taskIds: ['child-1'], + sourceIds: ['subagent-completion'], + now: expect.any(Date), + }); + await service.stop(); + }); + + it('does not expedite a result the store refused to persist', async () => { + const methods = deliveryMethods({ + persistAgentBackgroundToolResult: jest.fn(async () => false), + expediteAgentTriggerDeliveries: jest.fn(async () => ({ expedited: 1, held: 0 })), + }); + const service = createAgentTriggerService({ + methods, + deliveryOptions: { concurrency: 1, tickMs: 60_000 }, + }); + await service.initialize({ address }); + + await expect( + service.persistBackgroundToolResult({ + deliveryKey: 'trigger_background', + sourceId: 'background-tool-completion', + result, + }), + ).resolves.toBe(false); + await flush(); + + expect(methods.expediteAgentTriggerDeliveries).not.toHaveBeenCalled(); + await service.stop(); + }); + + it("expedites a principal's completion deliveries when one of its generations settles", async () => { + const methods = deliveryMethods({ + expediteAgentTriggerDeliveries: jest.fn(async () => ({ expedited: 2, held: 0 })), + }); + let settled: ((event: { userId: string; conversationId: string }) => void) | undefined; + const unsubscribe = jest.fn(); + const service = createAgentTriggerService({ + methods, + deliveryOptions: { concurrency: 1, tickMs: 60_000 }, + subscribeGenerationSettled: (listener) => { + settled = listener; + return unsubscribe; + }, + }); + await service.initialize({ address }); + jest.mocked(methods.claimNextAgentTriggerDelivery).mockClear(); + + settled?.({ userId: '507f1f77bcf86cd799439011', conversationId: 'conversation-1' }); + await flush(); + + expect(methods.expediteAgentTriggerDeliveries).toHaveBeenCalledWith({ + user: '507f1f77bcf86cd799439011', + conversationId: 'conversation-1', + sourceIds: ['background-tool-completion', 'subagent-completion'], + now: expect.any(Date), + }); + expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalled(); + await service.stop(); + expect(unsubscribe).toHaveBeenCalledTimes(1); + }); + + it('runs a claim pass even when nothing moved, since a match may already be due', async () => { + const methods = deliveryMethods({ + expediteAgentTriggerDeliveries: jest.fn(async () => ({ expedited: 0, held: 0 })), + }); + let settled: ((event: { userId: string; conversationId: string }) => void) | undefined; + const service = createAgentTriggerService({ + methods, + deliveryOptions: { concurrency: 1, tickMs: 60_000 }, + subscribeGenerationSettled: (listener) => { + settled = listener; + return () => undefined; + }, + }); + await service.initialize({ address }); + jest.mocked(methods.claimNextAgentTriggerDelivery).mockClear(); + + settled?.({ userId: '507f1f77bcf86cd799439011', conversationId: 'conversation-1' }); + await flush(); + + expect(methods.expediteAgentTriggerDeliveries).toHaveBeenCalledTimes(1); + expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalled(); + await service.stop(); + }); + }); + describe('completion wait configuration', () => { const address = { address: '127.0.0.1', family: 'IPv4' as const, port: 3080 }; diff --git a/packages/api/src/agents/triggers/service.ts b/packages/api/src/agents/triggers/service.ts index f208336e57..fc2125bef3 100644 --- a/packages/api/src/agents/triggers/service.ts +++ b/packages/api/src/agents/triggers/service.ts @@ -26,7 +26,9 @@ import type { import type { AgentTriggerEnqueueOptions, PreparedAgentTriggerDelivery } from './delivery'; import type { BoundAddress } from '../../app/origin'; import { AgentTriggerDeliveryDeferredError, createAgentTriggerDeliveryEngine } from './engine'; +import { BACKGROUND_TOOL_COMPLETION_SOURCE } from '../backgroundCompletionWakeup'; import { isShutdownInProgress, registerShutdownTask } from '../../app/shutdown'; +import { SUBAGENT_COMPLETION_SOURCE } from '../subagentCompletionWakeup'; import { generateAgentTriggerToken } from '../../crypto/jwt'; import { prepareAgentTriggerDelivery } from './delivery'; import { selfOriginFromAddress } from '../../app/origin'; @@ -35,6 +37,9 @@ import { parseAgentTriggerEnvelope } from './envelope'; import { createIdleRecoveryLoop } from '../recovery'; import { WAITING_RETRY_CAP_MS } from './backoff'; +/** Internal sources whose deliveries wait on a result or on their parent generation. */ +const COMPLETION_WAKEUP_SOURCES = [BACKGROUND_TOOL_COMPLETION_SOURCE, SUBAGENT_COMPLETION_SOURCE]; + export const AGENT_TRIGGER_TOKEN_TTL = '60s'; const DEFAULT_USER_DRAIN_TIMEOUT_MS = 35_000; const DEFAULT_USER_DRAIN_POLL_MS = 100; @@ -53,6 +58,11 @@ export interface AgentTriggerServiceOptions { }; } +/** A principal's deliveries resuming one conversation, or exact deliveries. */ +export type AgentTriggerCompletionExpedite = + | { user: string; conversationId: string; taskIds?: string[] } + | { deliveryKeys: string[] }; + export interface AgentTriggerServiceDeps { fetch?: AgentTriggerExecutionHostDeps['fetch']; getTimezone?: AgentTriggerExecutionHostDeps['getTimezone']; @@ -69,6 +79,16 @@ export interface AgentTriggerServiceDeps { reclaimCheckpointDeletions?: (limit: number, activity?: { found: boolean }) => Promise; supportsDetachedActionCompletion?: () => boolean; settleSourceBeforeDeadLetter?: AgentTriggerDeliveryEngineDeps['settleSourceBeforeDeadLetter']; + /** Subscribes to generations reaching a terminal state; returns an unsubscribe. */ + subscribeGenerationSettled?: ( + listener: (event: AgentTriggerGenerationSettledEvent) => void, + ) => () => void; +} + +/** The part of a settled generation that decides which waiting deliveries it may unblock. */ +export interface AgentTriggerGenerationSettledEvent { + userId: string; + conversationId: string; } export interface AgentTriggerDeliveryReceipt { @@ -124,6 +144,7 @@ export interface AgentTriggerDeliveryPersistence { retireAgentTriggerDelivery: AgentTriggerDeliveryMethods['retireAgentTriggerDelivery']; renewAgentTriggerDeliveryProducerLease: AgentTriggerDeliveryMethods['renewAgentTriggerDeliveryProducerLease']; persistAgentBackgroundToolResult?: AgentTriggerDeliveryMethods['persistAgentBackgroundToolResult']; + expediteAgentTriggerDeliveries?: AgentTriggerDeliveryMethods['expediteAgentTriggerDeliveries']; getAgentBackgroundToolResultClaim?: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']; releaseAgentBackgroundToolResultClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims']; retryAgentTriggerDelivery: AgentTriggerDeliveryStore['retry']; @@ -210,6 +231,8 @@ export interface AgentTriggerService { getBackgroundCompletionResultBatchSize: () => number; /** Longest a waiting completion delivery re-checks readiness. */ getCompletionWaitMaxIntervalMs: () => number; + /** Best effort: moves waiting completion deliveries forward after what they wait on changed. */ + expediteCompletionWakeups: (input: AgentTriggerCompletionExpedite) => void; releaseBackgroundToolResultClaims: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims']; drainUser: (userId: string) => Promise; prepareUserPurge: (userId: string, fenceStartedAt: Date, tenantId?: string) => Promise; @@ -539,9 +562,45 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A void purgeRecoveryLoop?.start(); }; + /** Moves waiting completion deliveries forward when what they wait on has + * changed, and marks held ones so their next deferral re-checks at once. The + * claim pass always runs: a matching delivery may already be due here without + * having moved. Best effort: a missed expedite only means the delivery + * re-checks at its backoff instead of immediately. */ + const expediteCompletions = (input: AgentTriggerCompletionExpedite): void => { + const expedite = deps.methods?.expediteAgentTriggerDeliveries; + if (expedite == null || !deliveryReady || stopping) { + return; + } + void runAsSystem(() => + expedite({ + ...('user' in input + ? { + user: input.user, + conversationId: input.conversationId, + ...(input.taskIds != null && { taskIds: input.taskIds }), + } + : { deliveryKeys: input.deliveryKeys }), + sourceIds: + 'user' in input && input.taskIds != null + ? [SUBAGENT_COMPLETION_SOURCE] + : COMPLETION_WAKEUP_SOURCES, + now: new Date(), + }), + ) + .then(() => deliveryEngine?.wake()) + .catch((error) => + logger.warn('[agent-triggers] failed to expedite waiting completion deliveries:', error), + ); + }; + + let unsubscribeGenerationSettled: (() => void) | undefined; + const stop = async (): Promise => { stopping = true; deliveryReady = false; + unsubscribeGenerationSettled?.(); + unsubscribeGenerationSettled = undefined; await purgeRecoveryLoop?.stop(); await initializePromise?.catch(() => undefined); await deliveryEngine?.stop(); @@ -611,6 +670,11 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A ); deliveryReady = true; deliveryEngine.start(); + /** Deliveries waiting on a parent resume that parent's conversation, so a + * settled generation wakes only those, not every waiting task of the user. */ + unsubscribeGenerationSettled ??= deps.subscribeGenerationSettled?.( + ({ userId, conversationId }) => expediteCompletions({ user: userId, conversationId }), + ); startPurgeRecovery(); logger.info('[agent-triggers] durable delivery engine started'); }).finally(() => { @@ -727,7 +791,11 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A persistBackgroundToolResult: (input) => runAsSystem(async () => { const persist = requireMethods().persistAgentBackgroundToolResult; - return persist == null ? false : persist(input); + const persisted = persist == null ? false : await persist(input); + if (persisted) { + expediteCompletions({ deliveryKeys: [input.deliveryKey] }); + } + return persisted; }), getBackgroundToolResultClaim: (input) => runAsSystem(async () => { @@ -736,6 +804,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A }), getBackgroundCompletionResultBatchSize: () => backgroundCompletionResultBatchSize, getCompletionWaitMaxIntervalMs: () => completionWaitMaxIntervalMs, + expediteCompletionWakeups: (input) => expediteCompletions(input), releaseBackgroundToolResultClaims: (input) => runAsSystem(async () => { const release = requireMethods().releaseAgentBackgroundToolResultClaims; diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts index 991ee1b882..a92318e29f 100644 --- a/packages/api/src/stream/GenerationJobManager.ts +++ b/packages/api/src/stream/GenerationJobManager.ts @@ -568,10 +568,22 @@ export interface CreateGenerationJobOptions { * receiving this claim, then pass the same object to {@link finishTerminalJob} * from a `finally` block. */ +/** A generation reached a terminal state and released its runtime. */ +export interface GenerationSettledEvent { + streamId: string; + conversationId: string; + userId: string; + status: TerminalJobClaim['status']; +} + +export type GenerationSettledListener = (event: GenerationSettledEvent) => void; + export interface TerminalJobClaim { readonly streamId: string; readonly createdAt: number; readonly conversationId?: string; + /** The generation's owner, so settlement can be announced to that principal's waiters. */ + readonly userId?: string; readonly status: 'complete' | 'error' | 'aborted'; readonly error?: string; /** The winner must durably publish either its normal FINAL or a @@ -786,6 +798,7 @@ class GenerationJobManagerClass { /** Makes terminal cleanup idempotent while keeping claims opaque to callers. */ private terminalFinishPromises = new WeakMap>(); + private generationSettledListeners = new Set(); /** Exact local runtime observed when a claim won; never clean a later runtime. */ private terminalClaimRuntimes = new WeakMap(); @@ -1078,6 +1091,7 @@ class GenerationJobManagerClass { createdAt, ...(job?.createdAt === createdAt && job.conversationId != null && { conversationId: job.conversationId }), + ...(job?.createdAt === createdAt && job.userId != null && { userId: job.userId }), status: 'error' as const, error, drainedSteers: Object.freeze([...drainedSteers]), @@ -2256,7 +2270,7 @@ class GenerationJobManagerClass { streamId: string, job: Pick< SerializableJobData, - 'createdAt' | 'conversationId' | 'providerExecutionId' | 'agentEventDeliveryKey' + 'createdAt' | 'conversationId' | 'providerExecutionId' | 'agentEventDeliveryKey' | 'userId' >, message: string, ): Promise { @@ -2288,6 +2302,13 @@ class GenerationJobManagerClass { expectCreatedAt: job.createdAt, }) ) { + /** A direct terminal transition builds no claim, so it announces itself. */ + this.notifyGenerationSettled({ + streamId, + conversationId: job.conversationId, + userId: job.userId, + status: 'error', + }); return true; } } catch (error) { @@ -2298,6 +2319,15 @@ class GenerationJobManagerClass { try { const current = await this.jobStore.getJob(streamId); + if (current?.createdAt === job.createdAt && current.status === 'error') { + this.notifyGenerationSettled({ + streamId, + conversationId: current.conversationId, + userId: current.userId, + status: 'error', + }); + return true; + } if ( current == null || current.createdAt !== job.createdAt || @@ -3901,6 +3931,7 @@ class GenerationJobManagerClass { ...(jobData.conversationId != null && { conversationId: jobData.conversationId, }), + ...(jobData.userId != null && { userId: jobData.userId }), status, ...(terminalError != null && { error: terminalError }), ...(options.persistencePending === true && { @@ -4071,6 +4102,40 @@ class GenerationJobManagerClass { * claim is idempotent, and every local mutation is pinned to the runtime * object and generation epoch captured when the CAS won. */ + /** + * Calls `listener` after each generation owned by this process reaches a + * terminal state and its runtime is released โ€” by completion, error, or + * abort. Listeners run synchronously and must not throw; failures are logged + * and never affect terminal cleanup. Returns an unsubscribe function. + */ + onGenerationSettled(listener: GenerationSettledListener): () => void { + this.generationSettledListeners.add(listener); + return () => { + this.generationSettledListeners.delete(listener); + }; + } + + private notifyGenerationSettled( + target: Pick, + ): void { + if (target.userId == null || this.generationSettledListeners.size === 0) { + return; + } + const event: GenerationSettledEvent = { + streamId: target.streamId, + conversationId: target.conversationId ?? target.streamId, + userId: target.userId, + status: target.status, + }; + for (const listener of this.generationSettledListeners) { + try { + listener(event); + } catch (listenerError) { + logger.error('[GenerationJobManager] Generation settled listener failed', listenerError); + } + } + } + finishTerminalJob(claim: TerminalJobClaim): Promise { const inFlight = this.terminalFinishPromises.get(claim); if (inFlight) { @@ -4260,6 +4325,7 @@ class GenerationJobManagerClass { metricStatus = 'error'; } recordGenerationJob(this.storeLabel, metricStatus); + this.notifyGenerationSettled(claim); } if (cleanupError != null) { @@ -4616,6 +4682,7 @@ class GenerationJobManagerClass { ...(jobData.conversationId != null && { conversationId: jobData.conversationId, }), + ...(jobData.userId != null && { userId: jobData.userId }), status: 'aborted', persistencePending: true, drainedSteers: Object.freeze([...drainedSteers]), @@ -8865,6 +8932,17 @@ class GenerationJobManagerClass { await this.runApprovalExpiredHandler(streamId, expiredJob); await this.notifyApprovalExpiredRuntime(streamId, expiredJob.createdAt, observedRuntime); + /** Expiry is a direct `requires_action -> aborted` transition that never builds a + * terminal claim, so it announces settlement itself. */ + this.notifyGenerationSettled({ + streamId, + conversationId: expiredJob.conversationId, + userId: expiredJob.userId, + status: 'aborted', + }); + /** Terminal now; releasing ownership keeps the sweep's relay branch from + * announcing this generation a second time. */ + this.releaseJobOwnership(streamId, expiredJob.createdAt); return true; } @@ -9055,7 +9133,17 @@ class GenerationJobManagerClass { await this.runApprovalExpiredHandler(streamId, job); } await this.notifyApprovalExpiredRuntime(streamId, job.createdAt, runtime); - changed = this.releaseJobOwnership(streamId, job.createdAt) || changed; + const released = this.releaseJobOwnership(streamId, job.createdAt); + if (released) { + /** The store won the expiry CAS, so no local claim announced it. */ + this.notifyGenerationSettled({ + streamId, + conversationId: job.conversationId, + userId: job.userId, + status: 'aborted', + }); + } + changed = released || changed; continue; } if ( diff --git a/packages/api/src/stream/__tests__/generationSettled.spec.ts b/packages/api/src/stream/__tests__/generationSettled.spec.ts new file mode 100644 index 0000000000..473b548dce --- /dev/null +++ b/packages/api/src/stream/__tests__/generationSettled.spec.ts @@ -0,0 +1,145 @@ +/** + * Waiting completion deliveries are expedited when a generation settles, so the + * settled notification must fire exactly once for every terminal path and name + * the generation's owner โ€” and a failing listener must never disturb cleanup. + */ +import type { GenerationSettledEvent } from '../GenerationJobManager'; +import { buildPendingAction, buildToolApprovalPayload } from '~/agents/hitl/policy'; + +/** Suppress winston Console transport output (survives jest.resetModules) */ +jest.spyOn(console, 'log').mockImplementation(); + +async function configureManager() { + const { GenerationJobManager } = await import('../GenerationJobManager'); + const { InMemoryJobStore } = await import('../implementations/InMemoryJobStore'); + const { InMemoryEventTransport } = await import('../implementations/InMemoryEventTransport'); + + GenerationJobManager.configure({ + jobStore: new InMemoryJobStore(), + eventTransport: new InMemoryEventTransport(), + isRedis: false, + cleanupOnComplete: false, + }); + GenerationJobManager.initialize(); + return GenerationJobManager; +} + +describe('generation settled notifications', () => { + beforeEach(() => { + jest.resetModules(); + }); + + it('announces a completed generation once, with its owner', async () => { + const manager = await configureManager(); + const events: GenerationSettledEvent[] = []; + manager.onGenerationSettled((event) => events.push(event)); + const job = await manager.createJob('settled-complete', 'user-1', 'conversation-1'); + + await manager.completeJob('settled-complete', undefined, job.createdAt); + await manager.completeJob('settled-complete', undefined, job.createdAt); + + expect(events).toEqual([ + { + streamId: 'settled-complete', + conversationId: 'conversation-1', + userId: 'user-1', + status: 'complete', + }, + ]); + await manager.destroy(); + }); + + it('announces a failed generation', async () => { + const manager = await configureManager(); + const events: GenerationSettledEvent[] = []; + manager.onGenerationSettled((event) => events.push(event)); + const job = await manager.createJob('settled-error', 'user-2'); + + await manager.completeJob('settled-error', 'provider failed', job.createdAt); + + expect(events).toEqual([ + expect.objectContaining({ streamId: 'settled-error', userId: 'user-2', status: 'error' }), + ]); + await manager.destroy(); + }); + + it('announces an aborted generation', async () => { + const manager = await configureManager(); + const events: GenerationSettledEvent[] = []; + manager.onGenerationSettled((event) => events.push(event)); + await manager.createJob('settled-abort', 'user-3'); + + const result = await manager.abortJob('settled-abort'); + + expect(result.success).toBe(true); + expect(events).toEqual([ + expect.objectContaining({ streamId: 'settled-abort', userId: 'user-3', status: 'aborted' }), + ]); + await manager.destroy(); + }); + + it('announces a generation whose approval expired', async () => { + const manager = await configureManager(); + const events: GenerationSettledEvent[] = []; + manager.onGenerationSettled((event) => events.push(event)); + await manager.createJob('settled-approval', 'user-6', 'conversation-6'); + const action = buildPendingAction( + buildToolApprovalPayload([ + { name: 'shell', arguments: { command: 'ls' }, tool_call_id: 'call_expire' }, + ]), + { + streamId: 'settled-approval', + conversationId: 'conversation-6', + runId: 'run-1', + responseMessageId: 'msg-1', + }, + ); + await manager.approvals.pause('settled-approval', action); + expect(events).toEqual([]); + + await expect(manager.expireApproval('settled-approval')).resolves.toBe(true); + await expect(manager.expireApproval('settled-approval')).resolves.toBe(false); + /** The next expiry sweep relays the terminal job without announcing it again. */ + await (manager as unknown as { cleanup(): Promise }).cleanup(); + + expect(events).toEqual([ + { + streamId: 'settled-approval', + conversationId: 'conversation-6', + userId: 'user-6', + status: 'aborted', + }, + ]); + await manager.destroy(); + }); + + it('keeps cleanup and later listeners running when one listener throws', async () => { + const manager = await configureManager(); + const events: GenerationSettledEvent[] = []; + manager.onGenerationSettled(() => { + throw new Error('listener failed'); + }); + manager.onGenerationSettled((event) => events.push(event)); + const job = await manager.createJob('settled-throwing-listener', 'user-4'); + + await expect( + manager.completeJob('settled-throwing-listener', undefined, job.createdAt), + ).resolves.toBe(true); + + expect(events).toHaveLength(1); + await manager.destroy(); + }); + + it('stops notifying after unsubscribe', async () => { + const manager = await configureManager(); + const listener = jest.fn(); + const unsubscribe = manager.onGenerationSettled(listener); + unsubscribe(); + const job = await manager.createJob('settled-unsubscribed', 'user-5'); + + await manager.completeJob('settled-unsubscribed', undefined, job.createdAt); + + expect(listener).not.toHaveBeenCalled(); + await manager.destroy(); + }); +}); diff --git a/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts b/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts index 3b5d964d9d..1c8578d86d 100644 --- a/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts +++ b/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts @@ -1070,48 +1070,86 @@ describe('GenerationJobManager start-generation claim', () => { expect(Object.getOwnPropertyDescriptor(durable!, 'replacedJobs')).toBeUndefined(); }); - it('terminalizes the exact committed epoch when legacy verification fails during recovery', async () => { - const streamId = 'stream-lost-create-legacy-failure'; - const clientRequestId = 'req-lost-create-legacy-failure'; - const claim = await manager.claimGeneration('user-1', clientRequestId, streamId, streamId, 2); - const actualCreate = store.createJob.bind(store); - jest.spyOn(store, 'createJob').mockImplementationOnce(async (...args) => { - await actualCreate(...args); - throw new Error('simulated lost atomic create reply'); - }); - const actualClaim = store.claimIdempotencyKey.bind(store); - let failLegacyProbe = true; - jest.spyOn(store, 'claimIdempotencyKey').mockImplementation((key, value, ttlSeconds) => { - if (failLegacyProbe && key === `{user-1:${clientRequestId}}`) { - failLegacyProbe = false; - return Promise.reject(new Error('simulated legacy probe outage')); + it.each([ + { loseTerminalReply: false, replaced: false }, + { loseTerminalReply: true, replaced: false }, + { loseTerminalReply: true, replaced: true }, + ])( + 'announces only the exact recovered epoch (lost reply: $loseTerminalReply, replaced: $replaced)', + async ({ loseTerminalReply, replaced }) => { + const settled = jest.fn(); + manager.onGenerationSettled(settled); + const transition = store.transitionStatus.bind(store); + jest.spyOn(store, 'transitionStatus').mockImplementation(async (id, input) => { + const committed = await transition(id, input); + if (loseTerminalReply && input.to === 'error' && committed) { + if (replaced) { + const previous = await store.getJob(id); + const replacement = await actualCreate(id, 'user-1', id); + await transition(id, { + from: 'running', + to: 'error', + expectCreatedAt: replacement.createdAt, + patch: { error: previous?.error, finalEvent: previous?.finalEvent }, + }); + } + throw new Error('lost terminal CAS reply'); + } + return committed; + }); + const streamId = 'stream-lost-create-legacy-failure'; + const clientRequestId = 'req-lost-create-legacy-failure'; + const claim = await manager.claimGeneration('user-1', clientRequestId, streamId, streamId, 2); + const actualCreate = store.createJob.bind(store); + jest.spyOn(store, 'createJob').mockImplementationOnce(async (...args) => { + await actualCreate(...args); + throw new Error('simulated lost atomic create reply'); + }); + const actualClaim = store.claimIdempotencyKey.bind(store); + let failLegacyProbe = true; + jest.spyOn(store, 'claimIdempotencyKey').mockImplementation((key, value, ttlSeconds) => { + if (failLegacyProbe && key === `{user-1:${clientRequestId}}`) { + failLegacyProbe = false; + return Promise.reject(new Error('simulated legacy probe outage')); + } + return actualClaim(key, value, ttlSeconds); + }); + + await expect( + manager.createJob(streamId, 'user-1', streamId, { + idempotencyClientRequestId: clientRequestId, + idempotencyClaimToken: claim.existing!.claimToken, + initialMetadata: { generationProtocolVersion: 2 }, + }), + ).rejects.toThrow('simulated lost atomic create reply'); + + const durable = await store.getJob(streamId); + expect(durable).toMatchObject({ + status: 'error', + error: 'Generation idempotency rollout fence could not be recovered', + finalEvent: expect.stringContaining('terminal_payload_missing'), + }); + if (replaced) { + expect(settled).not.toHaveBeenCalled(); + return; } - return actualClaim(key, value, ttlSeconds); - }); - - await expect( - manager.createJob(streamId, 'user-1', streamId, { - idempotencyClientRequestId: clientRequestId, - idempotencyClaimToken: claim.existing!.claimToken, - initialMetadata: { generationProtocolVersion: 2 }, - }), - ).rejects.toThrow('simulated lost atomic create reply'); - - const durable = await store.getJob(streamId); - expect(durable).toMatchObject({ - status: 'error', - error: 'Generation idempotency rollout fence could not be recovered', - finalEvent: expect.stringContaining('terminal_payload_missing'), - }); - const retry = await manager.claimGeneration('user-1', clientRequestId, streamId, streamId, 2); - expect(retry).toMatchObject({ - claimed: false, - existing: { startedAt: durable!.createdAt }, - }); - await expect( - manager.takeoverGeneration('user-1', clientRequestId, streamId, retry.existing!), - ).resolves.toMatchObject({ claimed: false }); - }); + expect(settled).toHaveBeenCalledTimes(1); + expect(settled).toHaveBeenCalledWith({ + streamId, + conversationId: streamId, + userId: 'user-1', + status: 'error', + }); + const retry = await manager.claimGeneration('user-1', clientRequestId, streamId, streamId, 2); + expect(retry).toMatchObject({ + claimed: false, + existing: { startedAt: durable!.createdAt }, + }); + await expect( + manager.takeoverGeneration('user-1', clientRequestId, streamId, retry.existing!), + ).resolves.toMatchObject({ claimed: false }); + }, + ); it('terminalizes and preserves the primary fence when the legacy started mark fails', async () => { const claim = await manager.claimGeneration( diff --git a/packages/api/src/stream/index.ts b/packages/api/src/stream/index.ts index b6c97e6098..ef72b19ee5 100644 --- a/packages/api/src/stream/index.ts +++ b/packages/api/src/stream/index.ts @@ -4,6 +4,8 @@ export { type CreateGenerationJobOptions, type GenerationJobManagerOptions, type TerminalJobClaim, + type GenerationSettledEvent, + type GenerationSettledListener, GENERATION_RECOVERY_FAILED_ERROR, TERMINAL_PUBLICATION_RECONNECT_ERROR, } from './GenerationJobManager'; diff --git a/packages/data-schemas/src/methods/triggerDelivery.spec.ts b/packages/data-schemas/src/methods/triggerDelivery.spec.ts index 3c6850d7b3..7498b932c4 100644 --- a/packages/data-schemas/src/methods/triggerDelivery.spec.ts +++ b/packages/data-schemas/src/methods/triggerDelivery.spec.ts @@ -453,6 +453,348 @@ describe('agent trigger delivery methods', () => { ).resolves.toEqual({ status: 'live', leaseUntil: renewedUntil }); }); + describe('expediteAgentTriggerDeliveries', () => { + const background = { id: 'background-tool-completion', type: 'internal' }; + const later = new Date(START.getTime() + 30_000); + const capable = { + workerId: 'background-capable-worker', + claimToken: 'background-capable-claim', + now: START, + leaseUntil: new Date(START.getTime() + 60_000), + workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2], + }; + const waiting = (overrides: Partial[0]> = {}) => + methods.enqueueAgentTriggerDelivery( + enqueueInput({ + orderingKey: `background-lane-${counter + 1}`, + envelope: { event: { source: background } }, + requiredWorkerCapability: + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + availableAt: later, + ...overrides, + }), + ); + + it('makes a deferred delivery claimable now once its result is durable', async () => { + const target = await waiting(); + const sibling = await waiting(); + await expect(methods.claimNextAgentTriggerDelivery(capable)).resolves.toBeNull(); + + await expect( + methods.expediteAgentTriggerDeliveries({ + deliveryKeys: [target.delivery.deliveryKey], + sourceIds: [background.id], + now: START, + }), + ).resolves.toEqual({ expedited: 1, held: 0 }); + + await expect(methods.claimNextAgentTriggerDelivery(capable)).resolves.toMatchObject({ + id: target.delivery.id, + }); + await expect( + methods.claimNextAgentTriggerDelivery({ ...capable, claimToken: 'second-claim' }), + ).resolves.toBeNull(); + const untouched = await Delivery.findById(sibling.delivery.id).lean(); + expect(untouched?.availableAt).toEqual(later); + }); + + it("moves only the principal's waiting rows from the named sources", async () => { + const user = new mongoose.Types.ObjectId(); + const mine = await waiting({ user }); + const otherUser = await waiting(); + const otherSource = await waiting({ + user, + envelope: { event: { source: { id: 'agent-queued-turn', type: 'internal' } } }, + }); + const external = await waiting({ + user, + envelope: { event: { source: { id: background.id, type: 'webhook' } } }, + }); + const due = await waiting({ user, availableAt: START }); + + await expect( + methods.expediteAgentTriggerDeliveries({ user, sourceIds: [background.id], now: START }), + ).resolves.toEqual({ expedited: 1, held: 0 }); + + const rows = await Delivery.find({ + _id: { + $in: [mine, otherUser, otherSource, external, due].map((row) => row.delivery.id), + }, + }).lean(); + const availableAt = new Map(rows.map((row) => [String(row._id), row.availableAt])); + expect(availableAt.get(mine.delivery.id)).toEqual(START); + expect(availableAt.get(otherUser.delivery.id)).toEqual(later); + expect(availableAt.get(otherSource.delivery.id)).toEqual(later); + expect(availableAt.get(external.delivery.id)).toEqual(later); + expect(availableAt.get(due.delivery.id)).toEqual(START); + }); + + it('marks a delivery a worker currently holds instead of moving it', async () => { + const user = new mongoose.Types.ObjectId(); + const held = await waiting({ user, availableAt: START }); + const claim = await methods.claimNextAgentTriggerDelivery(capable); + expect(claim).toMatchObject({ id: held.delivery.id }); + const before = await Delivery.findById(held.delivery.id).lean(); + + await expect( + methods.expediteAgentTriggerDeliveries({ + user, + sourceIds: [background.id], + now: new Date(START.getTime() - 60_000), + }), + ).resolves.toEqual({ expedited: 0, held: 1 }); + + const after = await Delivery.findById(held.delivery.id).lean(); + expect(after?.status).toBe(before?.status); + expect(after?.availableAt).toEqual(before?.availableAt); + expect(after?.wakeRequestedAt).toEqual(new Date(START.getTime() - 60_000)); + }); + + it('re-checks at once when readiness changed while the delivery was held', async () => { + const user = new mongoose.Types.ObjectId(); + const held = await waiting({ user, availableAt: START }); + const claim = await methods.claimNextAgentTriggerDelivery(capable); + expect(claim).toMatchObject({ id: held.delivery.id }); + const fence = { + id: held.delivery.id, + workerId: capable.workerId, + claimToken: capable.claimToken, + }; + const attempt = await methods.beginAgentTriggerDeliveryAttempt({ ...fence, now: START }); + await expect( + methods.expediteAgentTriggerDeliveries({ user, sourceIds: [background.id], now: START }), + ).resolves.toEqual({ expedited: 0, held: 1 }); + + const beforeDefer = Date.now(); + await expect( + methods.deferAgentTriggerDeliveryAttempt({ + ...fence, + attempt: attempt!, + availableAt: later, + }), + ).resolves.toBe('expedited'); + + const deferred = await Delivery.findById(held.delivery.id).lean(); + expect(deferred?.wakeRequestedAt).toBeUndefined(); + expect(deferred?.availableAt.getTime()).toBeGreaterThanOrEqual(beforeDefer); + expect(deferred?.availableAt).not.toEqual(later); + + const reclaimed = await methods.claimNextAgentTriggerDelivery({ + ...capable, + claimToken: 'second-claim', + now: new Date(), + leaseUntil: new Date(Date.now() + 60_000), + }); + expect(reclaimed).toMatchObject({ id: held.delivery.id }); + const secondFence = { ...fence, claimToken: 'second-claim' }; + const secondAttempt = await methods.beginAgentTriggerDeliveryAttempt({ + ...secondFence, + now: new Date(), + }); + await expect( + methods.deferAgentTriggerDeliveryAttempt({ + ...secondFence, + attempt: secondAttempt!, + availableAt: later, + }), + ).resolves.toBe(true); + expect((await Delivery.findById(held.delivery.id).lean())?.availableAt).toEqual(later); + }); + + it.each(['ordinary', 'legacy', 'shielded'] as const)( + 'honors a held wake marker on ordering release for %s leases', + async (profile) => { + const row = await waiting({ + availableAt: START, + ...(profile === 'ordinary' && { requiredWorkerCapability: undefined }), + }); + if (profile === 'legacy') { + await Delivery.updateOne( + { _id: row.delivery.id }, + { + $set: { status: 'capability_pending', availableAt: START }, + $unset: { capabilityStatus: 1, leaseUntil: 1 }, + }, + ); + } + const claimed = await methods.claimNextAgentTriggerDelivery(capable); + expect(claimed?.id).toBe(row.delivery.id); + const fence = { + id: row.delivery.id, + workerId: capable.workerId, + claimToken: capable.claimToken, + }; + await methods.expediteAgentTriggerDeliveries({ + deliveryKeys: [row.delivery.deliveryKey], + sourceIds: [background.id], + now: START, + }); + await expect( + methods.releaseAgentTriggerDelivery({ + ...fence, + claimToken: 'stale', + availableAt: later, + }), + ).resolves.toBe(false); + expect((await Delivery.findById(row.delivery.id).lean())?.wakeRequestedAt).toEqual(START); + await expect( + methods.releaseAgentTriggerDelivery({ ...fence, availableAt: later }), + ).resolves.toBe(true); + const released = await Delivery.findById(row.delivery.id).lean(); + expect(released?.wakeRequestedAt).toBeUndefined(); + expect(released?.availableAt.getTime()).toBeLessThanOrEqual(Date.now()); + expect(released?.claimAvailableAt).toEqual(released?.availableAt); + expect(released?.attempts).toBe(0); + const reclaimed = await methods.claimNextAgentTriggerDelivery({ + ...capable, + now: new Date(), + claimToken: 'next', + }); + expect(reclaimed?.id).toBe(row.delivery.id); + await methods.releaseAgentTriggerDelivery({ + ...fence, + claimToken: 'next', + availableAt: later, + }); + expect((await Delivery.findById(row.delivery.id).lean())?.availableAt).toEqual(later); + }, + ); + + it.each(['release', 'defer'] as const)( + 'does not lose an expedite between the unmarked and marked %s writes', + async (operation) => { + const row = await waiting({ availableAt: START }); + await methods.claimNextAgentTriggerDelivery(capable); + const fence = { + id: row.delivery.id, + workerId: capable.workerId, + claimToken: capable.claimToken, + }; + const attempt = + operation === 'defer' + ? await methods.beginAgentTriggerDeliveryAttempt({ ...fence, now: START }) + : undefined; + const updateOne = Delivery.collection.updateOne.bind(Delivery.collection); + let injected = false; + const spy = jest + .spyOn(Delivery.collection, 'updateOne') + .mockImplementation(async (filter, update, options) => { + if ( + !injected && + filter.capabilityClaimToken === capable.claimToken && + (filter.wakeRequestedAt as { $exists?: boolean } | undefined)?.$exists === false + ) { + injected = true; + await methods.expediteAgentTriggerDeliveries({ + deliveryKeys: [row.delivery.deliveryKey], + sourceIds: [background.id], + now: START, + }); + } + return updateOne(filter, update, options); + }); + try { + const input = { ...fence, availableAt: later }; + if (operation === 'defer') { + await expect( + methods.deferAgentTriggerDeliveryAttempt({ ...input, attempt: attempt! }), + ).resolves.toBe('expedited'); + } else { + await expect(methods.releaseAgentTriggerDelivery(input)).resolves.toBe(true); + } + } finally { + spy.mockRestore(); + } + expect(injected).toBe(true); + const released = await Delivery.findById(row.delivery.id).lean(); + expect(released?.wakeRequestedAt).toBeUndefined(); + expect(released?.availableAt.getTime()).toBeLessThanOrEqual(Date.now()); + expect(released?.attempts).toBe(0); + }, + ); + + it('leaves unfinished siblings backed off when only named child tasks settle', async () => { + const user = new mongoose.Types.ObjectId(); + const source = { id: 'subagent-completion', type: 'internal' }; + const task = (taskId: string) => + waiting({ + user, + envelope: { + event: { source, payload: { taskId } }, + target: { conversationId: 'parent' }, + }, + }); + const original = await task('original'); + const recovered = await task('recovered'); + const sibling = await task('sibling'); + await expect( + methods.expediteAgentTriggerDeliveries({ + user, + conversationId: 'parent', + taskIds: ['original', 'recovered'], + sourceIds: [source.id], + now: START, + }), + ).resolves.toEqual({ expedited: 2, held: 0 }); + expect((await Delivery.findById(original.delivery.id).lean())?.availableAt).toEqual(START); + expect((await Delivery.findById(recovered.delivery.id).lean())?.availableAt).toEqual(START); + expect((await Delivery.findById(sibling.delivery.id).lean())?.availableAt).toEqual(later); + await expect( + methods.expediteAgentTriggerDeliveries({ + user, + conversationId: 'parent', + taskIds: [], + sourceIds: [source.id], + now: START, + }), + ).rejects.toThrow(TypeError); + }); + + it("narrows a principal's selection to the conversation being resumed", async () => { + const user = new mongoose.Types.ObjectId(); + const target = await waiting({ + user, + envelope: { event: { source: background }, target: { conversationId: 'settled-convo' } }, + }); + const other = await waiting({ + user, + envelope: { event: { source: background }, target: { conversationId: 'other-convo' } }, + }); + + await expect( + methods.expediteAgentTriggerDeliveries({ + user, + conversationId: 'settled-convo', + sourceIds: [background.id], + now: START, + }), + ).resolves.toEqual({ expedited: 1, held: 0 }); + + expect((await Delivery.findById(target.delivery.id).lean())?.availableAt).toEqual(START); + expect((await Delivery.findById(other.delivery.id).lean())?.availableAt).toEqual(later); + }); + + it('refuses an unbounded or malformed selection', async () => { + await expect( + methods.expediteAgentTriggerDeliveries({ sourceIds: [background.id], now: START }), + ).rejects.toThrow(TypeError); + await expect( + methods.expediteAgentTriggerDeliveries({ + user: new mongoose.Types.ObjectId(), + sourceIds: [], + now: START, + }), + ).rejects.toThrow(TypeError); + await expect( + methods.expediteAgentTriggerDeliveries({ + deliveryKeys: ['trigger_x'], + sourceIds: [background.id], + now: new Date(Number.NaN), + }), + ).rejects.toThrow(TypeError); + }); + }); + it('persists one private background result receipt independently of message rows', async () => { const source = { id: 'background-tool-completion', type: 'internal' }; const queued = await methods.enqueueAgentTriggerDelivery( @@ -2468,7 +2810,7 @@ describe('agent trigger delivery methods', () => { ); }); - it("indexes a user's waiting deliveries for user-scoped readiness reads", async () => { + it("indexes a user's waiting deliveries for settle-time expediting", async () => { const deliveryIndexes = await Delivery.collection.indexes(); const userIndex = deliveryIndexes.find( (index) => diff --git a/packages/data-schemas/src/methods/triggerDelivery.ts b/packages/data-schemas/src/methods/triggerDelivery.ts index e3fe4c2d0f..1e2b6ead6f 100644 --- a/packages/data-schemas/src/methods/triggerDelivery.ts +++ b/packages/data-schemas/src/methods/triggerDelivery.ts @@ -246,6 +246,27 @@ export interface AgentEventActorReceiptStorageMetrics { deadDeliveries: number; } +/** Selects deferred internal deliveries whose readiness condition just changed. */ +export interface ExpediteAgentTriggerDeliveriesInput { + sourceIds: readonly string[]; + /** Exact deliveries, e.g. the one whose result just became durable. */ + deliveryKeys?: readonly string[]; + /** Every matching delivery of one principal, e.g. after one of its generations settled. */ + user?: string | Types.ObjectId; + /** Narrows a principal's selection to deliveries that resume this conversation. */ + conversationId?: string; + /** Exact tasks within a principal's conversation, including a repaired attempt's predecessor. */ + taskIds?: readonly string[]; + now: Date; +} + +export interface ExpediteAgentTriggerDeliveriesResult { + /** Deferred, unheld deliveries moved to `now`. */ + expedited: number; + /** Deliveries a worker held, marked so their next deferral re-checks at once. */ + held: number; +} + export interface AgentTriggerDeliveryMethods { ensureAgentTriggerDeliveryIndexes: () => Promise; enqueueAgentTriggerDelivery: ( @@ -270,9 +291,10 @@ export interface AgentTriggerDeliveryMethods { beginAgentTriggerDeliveryAttempt: ( input: AgentTriggerDeliveryFence & { now: Date }, ) => Promise; + /** `expedited` when readiness changed while the delivery was held: it is due now instead. */ deferAgentTriggerDeliveryAttempt: ( input: AgentTriggerDeliveryFence & { attempt: number; availableAt: Date }, - ) => Promise; + ) => Promise; completeAgentTriggerDelivery: ( input: AgentTriggerDeliveryFence & { attempt: number; @@ -307,6 +329,9 @@ export interface AgentTriggerDeliveryMethods { sourceId: string; now: Date; }) => Promise; + expediteAgentTriggerDeliveries: ( + input: ExpediteAgentTriggerDeliveriesInput, + ) => Promise; persistAgentBackgroundToolResult: ( input: PersistAgentBackgroundToolResultInput, ) => Promise; @@ -1846,57 +1871,61 @@ export function createAgentTriggerDeliveryMethods( $or: [ordinaryFence(input), legacyCapabilityFence(input), shieldCapabilityFence(input)], }); + /** Readiness signals and the lease release must meet in one fenced write. Try + * the unmarked state first, then the marked state: a concurrent expedite can + * only add a marker while this lease is held. After release, expedite's second + * update sees an unheld row. No pipeline updates or unfenced follow-up needed. */ + async function releaseWaitingDelivery( + input: AgentTriggerDeliveryFence & { availableAt: Date; attempt?: number }, + ): Promise { + const attemptFence = input.attempt == null ? {} : { attempts: input.attempt }; + const attemptChange = input.attempt == null ? {} : { $inc: { attempts: -1 } }; + const profiles = [ + { filter: shieldCapabilityFence(input), status: 'leased', capabilityStatus: 'pending' }, + { filter: legacyCapabilityFence(input), status: 'capability_pending' }, + { filter: ordinaryFence(input), status: 'pending' }, + ] as const; + for (const marked of [false, true]) { + for (const profile of profiles) { + const availableAt = marked ? new Date() : input.availableAt; + const result = await Delivery().updateOne( + { + _id: input.id, + ...profile.filter, + ...attemptFence, + wakeRequestedAt: { $exists: marked }, + }, + { + ...attemptChange, + $set: { + status: profile.status, + ...('capabilityStatus' in profile && { capabilityStatus: profile.capabilityStatus }), + availableAt, + claimAvailableAt: availableAt, + }, + $unset: { + leaseBy: 1, + leaseUntil: 1, + claimToken: 1, + capabilityLeaseBy: 1, + capabilityLeaseUntil: 1, + capabilityClaimToken: 1, + wakeRequestedAt: 1, + }, + }, + ); + if (result.modifiedCount === 1) { + return marked ? 'expedited' : true; + } + } + } + return false; + } + async function releaseAgentTriggerDelivery( input: AgentTriggerDeliveryFence & { availableAt: Date }, ): Promise { - const shieldResult = await Delivery().updateOne( - { _id: input.id, ...shieldCapabilityFence(input) }, - { - $set: { - status: 'leased', - availableAt: input.availableAt, - capabilityStatus: 'pending', - claimAvailableAt: input.availableAt, - }, - $unset: { - leaseBy: 1, - leaseUntil: 1, - claimToken: 1, - capabilityLeaseBy: 1, - capabilityLeaseUntil: 1, - capabilityClaimToken: 1, - }, - }, - ); - if (shieldResult.modifiedCount === 1) { - return true; - } - const capabilityResult = await Delivery().updateOne( - { _id: input.id, ...legacyCapabilityFence(input) }, - { - $set: { - status: 'capability_pending', - availableAt: input.availableAt, - claimAvailableAt: input.availableAt, - }, - $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1 }, - }, - ); - if (capabilityResult.modifiedCount === 1) { - return true; - } - const result = await Delivery().updateOne( - { _id: input.id, ...ordinaryFence(input) }, - { - $set: { - status: 'pending', - availableAt: input.availableAt, - claimAvailableAt: input.availableAt, - }, - $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1 }, - }, - ); - return result.modifiedCount === 1; + return (await releaseWaitingDelivery(input)) !== false; } async function beginAgentTriggerDeliveryAttempt( @@ -1926,64 +1955,11 @@ export function createAgentTriggerDeliveryMethods( /** Releases a pre-dispatch deferral and restores the attempt consumed by beginAttempt. */ async function deferAgentTriggerDeliveryAttempt( input: AgentTriggerDeliveryFence & { attempt: number; availableAt: Date }, - ): Promise { + ): Promise { if (!Number.isSafeInteger(input.attempt) || input.attempt <= 0) { throw new TypeError('attempt must be a positive integer'); } - const update = { - $inc: { attempts: -1 }, - $set: { - availableAt: input.availableAt, - claimAvailableAt: input.availableAt, - }, - $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1 }, - }; - const shieldResult = await Delivery().updateOne( - { - _id: input.id, - ...shieldCapabilityFence(input), - attempts: input.attempt, - }, - { - $inc: update.$inc, - $set: { - status: 'leased', - availableAt: input.availableAt, - capabilityStatus: 'pending', - claimAvailableAt: input.availableAt, - }, - $unset: { - leaseBy: 1, - leaseUntil: 1, - claimToken: 1, - capabilityLeaseBy: 1, - capabilityLeaseUntil: 1, - capabilityClaimToken: 1, - }, - }, - ); - if (shieldResult.modifiedCount === 1) { - return true; - } - const capabilityResult = await Delivery().updateOne( - { - _id: input.id, - ...legacyCapabilityFence(input), - attempts: input.attempt, - }, - { - ...update, - $set: { ...update.$set, status: 'capability_pending' }, - }, - ); - if (capabilityResult.modifiedCount === 1) { - return true; - } - const result = await Delivery().updateOne( - { _id: input.id, ...ordinaryFence(input), attempts: input.attempt }, - { ...update, $set: { ...update.$set, status: 'pending' } }, - ); - return result.modifiedCount === 1; + return releaseWaitingDelivery(input); } async function completeAgentTriggerDelivery( @@ -2287,6 +2263,75 @@ export function createAgentTriggerDeliveryMethods( : { status: 'expired', leaseUntil: delivery.producerLeaseUntil }; } + /** Pulls deferred deliveries back to `now` when the condition they were + * waiting on has changed, so a waiting delivery can back off without delaying + * the moment it becomes deliverable. Unclaimed rows move now; held rows retain + * a signal consumed atomically by readiness deferral or ordering release. */ + async function expediteAgentTriggerDeliveries( + input: ExpediteAgentTriggerDeliveriesInput, + ): Promise { + const deliveryKeys = input.deliveryKeys ?? []; + if ( + (input.taskIds != null && + (input.user == null || + input.conversationId == null || + input.taskIds.length === 0 || + input.taskIds.some((id) => id.length === 0 || id.length > 256))) || + input.sourceIds.length === 0 || + input.sourceIds.some((id) => id.length === 0 || id.length > 256) || + deliveryKeys.some((key) => key.length === 0 || key.length > 256) || + (deliveryKeys.length === 0 && input.user == null) || + (input.conversationId != null && + (input.conversationId.length === 0 || input.conversationId.length > 256)) || + !(input.now instanceof Date) || + !Number.isFinite(input.now.getTime()) + ) { + throw new TypeError('Invalid agent trigger delivery expedite'); + } + const selection = { + 'envelope.event.source.type': 'internal', + 'envelope.event.source.id': { $in: [...input.sourceIds] }, + ...(deliveryKeys.length > 0 && { deliveryKey: { $in: [...deliveryKeys] } }), + ...(input.taskIds != null && { + 'envelope.event.payload.taskId': { $in: [...input.taskIds] }, + }), + ...(input.user != null && { user: input.user }), + ...(input.conversationId != null && { + 'envelope.target.conversationId': input.conversationId, + }), + }; + /** Classic operators only: aggregation-pipeline updates are not portable. A + * held row may be deferred on readiness its worker read before the change, + * so it keeps a marker that its deferral honors instead of moving now. The + * marker is written first: a row released after it was read as held is then + * seen unheld by the move, so no release between the two escapes both. */ + const held = await Delivery().updateMany( + { + ...selection, + status: { $in: ['leased', 'capability_leased'] }, + $or: [{ leaseBy: { $exists: true } }, { capabilityLeaseBy: { $exists: true } }], + }, + { $set: { wakeRequestedAt: input.now } }, + ); + const moved = await Delivery().updateMany( + { + ...selection, + availableAt: { $gt: input.now }, + leaseBy: { $exists: false }, + $or: [ + { status: { $in: ['pending', 'capability_pending'] } }, + { + status: 'leased', + capabilityStatus: 'pending', + capabilityLeaseBy: { $exists: false }, + }, + ], + }, + { $set: { availableAt: input.now, claimAvailableAt: input.now } }, + ); + return { expedited: moved.modifiedCount, held: held.matchedCount }; + } + /** Stores terminal output on the pre-admitted delivery before attempting the * parent-message projection. The first terminal receipt wins; exact retries * are idempotent and conflicting rewrites fail closed. */ @@ -4049,6 +4094,7 @@ export function createAgentTriggerDeliveryMethods( retireAgentTriggerDelivery, renewAgentTriggerDeliveryProducerLease, getAgentTriggerDeliveryProducerLease, + expediteAgentTriggerDeliveries, persistAgentBackgroundToolResult, getAgentBackgroundToolResult, getAgentBackgroundToolResultClaim, diff --git a/packages/data-schemas/src/schema/triggerDelivery.ts b/packages/data-schemas/src/schema/triggerDelivery.ts index 2d08f6d14c..d9b503e6c5 100644 --- a/packages/data-schemas/src/schema/triggerDelivery.ts +++ b/packages/data-schemas/src/schema/triggerDelivery.ts @@ -218,6 +218,7 @@ const triggerDeliverySchema: Schema = new Schema( requeueCount: { type: Number, default: 0, min: 0 }, stagingRecoveryAt: { type: Date }, laneCleanupPendingAt: { type: Date }, + wakeRequestedAt: { type: Date }, }, { timestamps: true }, ); @@ -253,7 +254,7 @@ triggerDeliverySchema.index( { sparse: true }, ); triggerDeliverySchema.index({ status: 1, updatedAt: -1 }); -/** One user's waiting deliveries, read when their readiness changes. */ +/** One user's waiting deliveries, read when that user's generation settles. */ triggerDeliverySchema.index({ user: 1, status: 1, availableAt: 1 }); triggerDeliverySchema.index({ 'actorReceipt.resolution': 1 }, { sparse: true }); triggerDeliverySchema.index({ user: 1, actorActionAdmittedAt: 1 }, { sparse: true }); diff --git a/packages/data-schemas/src/types/triggerDelivery.ts b/packages/data-schemas/src/types/triggerDelivery.ts index be019d65a2..5865d88267 100644 --- a/packages/data-schemas/src/types/triggerDelivery.ts +++ b/packages/data-schemas/src/types/triggerDelivery.ts @@ -171,6 +171,8 @@ export interface IAgentTriggerDelivery { stagingRecoveryAt?: Date; /** Durable proof that successful settlement still owes lane cleanup publication. */ laneCleanupPendingAt?: Date; + /** Readiness changed while a worker held this delivery; its next deferral re-checks at once. */ + wakeRequestedAt?: Date; createdAt?: Date; updatedAt?: Date; }