diff --git a/api/server/controllers/agents/callbacks.js b/api/server/controllers/agents/callbacks.js index 4b27bde7fe..a42afb3625 100644 --- a/api/server/controllers/agents/callbacks.js +++ b/api/server/controllers/agents/callbacks.js @@ -20,6 +20,7 @@ const { sendEvent, computeUsageCostUSD, GenerationJobManager, + waitForGenerationSettled, writeAttachmentEvent, createToolExecuteHandler, createOwnedToolEndHandler, @@ -1286,14 +1287,20 @@ function createPtcProgressEmitter({ res, streamId = null, jobCreatedAt }) { * output?: string; * attachments?: Object[]; * }) => Promise} params.updateToolCallResult + * @param {number} [params.jobCreatedAt] - Immutable dispatch generation epoch. + * @param {string} [params.streamId] - The stream owning that epoch. */ -function createBackgroundCodeResultHandler({ req, updateToolCallResult }) { +function createBackgroundCodeResultHandler({ req, updateToolCallResult, jobCreatedAt, streamId }) { return createCodeHarvestHandler({ req, updateToolCallResult, preflightCodeOutputBatch, processCodeOutput, runPreviewFinalize, + generationCreatedAt: jobCreatedAt, + generationStreamId: streamId, + waitForGenerationSettled: (conversationId, options) => + waitForGenerationSettled(GenerationJobManager, conversationId, options), }); } diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js index 875d085951..f6f0f3ebd0 100644 --- a/api/server/services/Endpoints/agents/initialize.js +++ b/api/server/services/Endpoints/agents/initialize.js @@ -511,6 +511,8 @@ const initializeClientWithProvider = async ({ }, persistBackgroundCodeResult: createBackgroundCodeResultHandler({ req, + streamId, + jobCreatedAt, updateToolCallResult: db.updateToolCallResult, }), backgroundToolCompletion: { diff --git a/api/server/services/Endpoints/agents/initialize.spec.js b/api/server/services/Endpoints/agents/initialize.spec.js index 62b05309f0..b93442123f 100644 --- a/api/server/services/Endpoints/agents/initialize.spec.js +++ b/api/server/services/Endpoints/agents/initialize.spec.js @@ -304,6 +304,9 @@ describe('initializeClient — processAgent ACL gate', () => { jobCreatedAt: 1234, }), ); + expect( + require('~/server/controllers/agents/callbacks').createBackgroundCodeResultHandler, + ).toHaveBeenCalledWith(expect.objectContaining({ jobCreatedAt: 1234 })); expect(createToolEndCallback).toHaveBeenCalledWith( expect.objectContaining({ streamId: 'conv_1', diff --git a/packages/api/src/agents/background.spec.ts b/packages/api/src/agents/background.spec.ts index 5418fdeb3f..70dca23967 100644 --- a/packages/api/src/agents/background.spec.ts +++ b/packages/api/src/agents/background.spec.ts @@ -1369,6 +1369,54 @@ describe('BackgroundTaskRegistryClass', () => { expect(registry.get('u1', 'c1', created.task.id)).toBeUndefined(); }); + it.each(['completed', 'error', 'cancelled', 'blocked'] as const)( + 'preserves pending %s results through both TTLs and releases protection afterward', + async (status) => { + jest.useFakeTimers(); + try { + const registry = new BackgroundTaskRegistryClass(); + const created = registry.create({ + userId: 'u', + conversationId: 'c', + toolCallId: 'call', + toolName: 'execute_code', + harvestStarted: true, + }); + if ('atCapacity' in created) throw new Error('Unexpected capacity rejection'); + if (status === 'cancelled') registry.cancel('u', 'c', created.task.id, 'cancelled'); + else if (status === 'error') registry.fail('u', 'c', created.task.id, 'failed'); + else + registry.complete('u', 'c', created.task.id, { content: 'result', harvestStarted: true }); + registry.markCompletionPersistencePending('u', 'c', created.task.id); + registry.finishHarvest('u', 'c', created.task.id, [{ file_id: 'file' }]); + if (status === 'blocked') registry.blockArtifact('u', 'c', created.task.id, 'blocked'); + registry.claimResult('u', 'c', created.task.id, { kind: 'manual', claimId: 'poll' }); + await jest.advanceTimersByTimeAsync(7 * 60 * 60_000); + expect(registry.get('u', 'c', created.task.id)).toMatchObject({ + completionPersistencePending: true, + }); + registry.markCompletionPersistenceFinished('u', 'c', created.task.id); + expect( + registry.get('u', 'c', created.task.id)?.completionPersistencePending, + ).toBeUndefined(); + if (status === 'blocked') + expect(registry.get('u', 'c', created.task.id)).toMatchObject({ + artifactBlocked: true, + artifact: undefined, + }); + if (status === 'completed') + expect(registry.get('u', 'c', created.task.id)).toMatchObject({ + result: 'result', + resultClaim: { kind: 'manual', claimId: 'poll' }, + }); + await jest.advanceTimersByTimeAsync(61 * 60_000); + expect(registry.get('u', 'c', created.task.id)).toBeUndefined(); + } finally { + jest.useRealTimers(); + } + }, + ); + it('caps concurrent running tasks per conversation', () => { const registry = new BackgroundTaskRegistryClass(); let atCapacity = false; diff --git a/packages/api/src/agents/background.ts b/packages/api/src/agents/background.ts index 2c9b744ee4..4caf96f09a 100644 --- a/packages/api/src/agents/background.ts +++ b/packages/api/src/agents/background.ts @@ -834,7 +834,11 @@ export class BackgroundTaskRegistryClass { private sweepBucketTasks(bucket: TaskBucket, now: number): void { for (const [taskId, task] of bucket.tasks) { - if (task.status !== 'running' && now - task.updatedAt > COMPLETED_TASK_TTL_MS) { + if ( + task.status !== 'running' && + task.completionPersistencePending !== true && + now - task.updatedAt > COMPLETED_TASK_TTL_MS + ) { bucket.tasks.delete(taskId); } } @@ -859,7 +863,13 @@ export class BackgroundTaskRegistryClass { } this.lastGlobalSweepAt = now; for (const [bucketKey, bucket] of this.buckets) { - if (now - bucket.lastAccess > IDLE_BUCKET_TTL_MS && bucket.capacityPermits.size === 0) { + if ( + now - bucket.lastAccess > IDLE_BUCKET_TTL_MS && + bucket.capacityPermits.size === 0 && + ![...bucket.tasks.values()].some( + (task) => task.status === 'running' || task.completionPersistencePending === true, + ) + ) { this.buckets.delete(bucketKey); continue; } @@ -1615,7 +1625,14 @@ export class BackgroundTaskRegistryClass { } markCompletionPersistenceFinished(userId: string, conversationId: string, taskId: string): void { - this.update(userId, conversationId, taskId, { completionPersistencePending: undefined }); + const bucket = this.buckets.get(this.key(userId, conversationId)); + const task = bucket?.tasks.get(taskId); + if (bucket == null || task == null) return; + /** Even a policy-blocked task must release retention protection. This only + * clears lifecycle state; the immutable artifact block remains intact. */ + task.completionPersistencePending = undefined; + task.updatedAt = Date.now(); + bucket.lastAccess = task.updatedAt; } markCompletionPersistenceFailed(userId: string, conversationId: string, taskId: string): void { diff --git a/packages/api/src/agents/handlers.background.spec.ts b/packages/api/src/agents/handlers.background.spec.ts index 37ad6535d6..6eac3eedb1 100644 --- a/packages/api/src/agents/handlers.background.spec.ts +++ b/packages/api/src/agents/handlers.background.spec.ts @@ -2621,6 +2621,59 @@ describe('createToolExecuteHandler — backgrounded code execution', () => { expect(poll[0].artifact).toBeUndefined(); }); + it('releases stored files to same-turn polls while the row patch waits for the dispatch turn', async () => { + const state: CodeToolState = { calls: 0 }; + const emitted: unknown[] = []; + const handler = createToolExecuteHandler({ + loadTools: async () => ({ loadedTools: [makeCodeTool(state)] }), + /** Files are stored; the row patch then waits for the long dispatch turn. */ + persistBackgroundCodeResult: (params) => { + params.onFilesPersisted?.([{ file_id: 'f1', toolCallId: params.toolCallId }]); + return new Promise(() => undefined); + }, + emitAttachment: (attachment) => { + emitted.push(attachment); + }, + }); + const configurable = buildConfig(['execute_code']); + + const dispatch = await runBatch(handler, { + toolCalls: [codeCall({ id: 'call_code_long_turn' })], + agentId: 'a', + configurable, + metadata: { thread_id: 'exec_convo_code_long_turn', run_id: 'msg-long-turn' }, + }); + await flushMicrotasks(); + await flushMicrotasks(); + + const poll = (await runBatch(handler, { + toolCalls: [ + { + id: 'call_poll_long_turn', + name: CHECK_BACKGROUND_TASK_NAME, + args: { background_task_id: JSON.parse(dispatch[0].content).background_task_id }, + }, + ], + agentId: 'a', + configurable, + metadata: { thread_id: 'exec_convo_code_long_turn', run_id: 'msg-long-turn' }, + })) as Array<{ content: string; artifact?: unknown }>; + + const polled = JSON.parse(poll[0].content); + expect(polled.status).toBe('completed'); + expect(polled.note).toContain('attached to the tool call'); + expect(emitted[0]).toEqual({ file_id: 'f1', toolCallId: 'call_code_long_turn' }); + expect(poll[0].artifact).toEqual(CODE_ARTIFACT); + /** Still protected from retention eviction while the row patch waits. */ + expect( + backgroundTaskRegistry.get( + 'exec_user', + 'exec_convo_code_long_turn', + JSON.parse(dispatch[0].content).background_task_id, + )?.completionPersistencePending, + ).toBe(true); + }); + it('falls back to poll-turn delivery when the harvest fails (files not lost)', async () => { const state: CodeToolState = { calls: 0 }; const toolEndCalls: Array<{ name?: string; artifact?: unknown }> = []; diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts index edbab15caa..601d2dd077 100644 --- a/packages/api/src/agents/handlers.ts +++ b/packages/api/src/agents/handlers.ts @@ -315,6 +315,7 @@ export interface ToolExecuteOptions { reapply?: boolean; backgroundTask?: BackgroundToolResultState; resolveBackgroundTask?: () => BackgroundToolResultState; + onFilesPersisted?: (attachments: unknown[]) => void; }) => Promise<{ attachments?: unknown[]; deliveryReady?: boolean } | null>; /** Shared ordinary-tool completion lifecycle. The delivery is registered * before invoke; settlement is persisted onto the original response row. */ @@ -6169,6 +6170,13 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand : { backgroundTask, resolveBackgroundTask }), output: params.output ?? localTask?.result, artifact: params.artifact, + onFilesPersisted: (attachments) => + backgroundTaskRegistry.finishHarvest( + backgroundUserId, + backgroundConversationId, + task.id, + attachments, + ), }); if (persisted == null) { /** Harvest never persisted anything (missing anchor @@ -6257,10 +6265,8 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand artifact?: unknown; status: 'completed' | 'error' | 'cancelled'; }): Promise => { - if (harvestEnabled) { - await persistBackgroundResult(params); - return; - } + /** Held for the whole persist, including a code harvest that waits for + * a long dispatch turn, so retention pressure cannot evict the task. */ backgroundTaskRegistry.markCompletionPersistencePending( backgroundUserId, backgroundConversationId, diff --git a/packages/api/src/agents/harvest.lifecycle.spec.ts b/packages/api/src/agents/harvest.lifecycle.spec.ts new file mode 100644 index 0000000000..a41569f644 --- /dev/null +++ b/packages/api/src/agents/harvest.lifecycle.spec.ts @@ -0,0 +1,156 @@ +import type { GenerationSettlementState } from '../stream/interfaces/IJobStore'; +import type { GenerationSettledListener } from '../stream/GenerationJobManager'; +import type { GenerationSettledSource } from '../stream/settled'; +import type { GenerationJobStatus } from '../types/stream'; +import type { ServerRequest } from '~/types'; +import { InMemoryEventTransport } from '../stream/implementations/InMemoryEventTransport'; +import { InMemoryJobStore } from '../stream/implementations/InMemoryJobStore'; +import { GenerationJobManagerClass } from '../stream/GenerationJobManager'; +import { createBackgroundCodeResultHandler } from './harvest'; +import { waitForGenerationSettled } from '../stream/settled'; + +function sourceForRunningTurn() { + const listeners = new Set(); + const job: { + status: GenerationJobStatus; + createdAt: number; + terminalPersistencePending?: boolean; + } = { status: 'running', createdAt: 1 }; + return { + job, + listeners, + getGenerationSettlementState: jest.fn( + async (): Promise => ({ ...job }), + ), + onGenerationSettled: (listener: GenerationSettledListener) => { + listeners.add(listener); + return () => { + listeners.delete(listener); + }; + }, + settle: () => { + job.status = 'complete'; + for (const listener of [...listeners]) + listener({ + streamId: 'conversation', + conversationId: 'conversation', + userId: 'user', + status: 'complete', + }); + }, + }; +} + +function harvest(source: GenerationSettledSource, row: { unfinished: boolean }) { + const update = jest.fn(async () => ({ matched: true, unfinished: row.unfinished })); + const handler = createBackgroundCodeResultHandler({ + req: { user: { id: 'user' } } as ServerRequest, + preflightCodeOutputBatch: async () => [], + processCodeOutput: jest.fn(), + updateToolCallResult: update, + runPreviewFinalize: jest.fn(), + waitForGenerationSettled: (conversationId, options) => + waitForGenerationSettled(source, conversationId, options), + }); + let result: boolean | undefined; + const pending = handler({ + toolName: 'execute_code', + toolCallId: 'call', + stepId: 'step', + messageId: 'message', + conversationId: 'conversation', + output: 'finished code', + backgroundTask: { + taskId: 'task', + toolName: 'execute_code', + status: 'completed', + settledAt: new Date(), + completionWakeup: true, + }, + }).then((value) => { + result = value?.deliveryReady; + }); + return { update, pending, getResult: () => result }; +} + +describe('background code harvest generation lifecycle', () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + afterEach(() => { + jest.clearAllTimers(); + jest.useRealTimers(); + }); + + it('keeps harvesting a live turn after a transient first status-read failure', async () => { + const source = sourceForRunningTurn(); + source.getGenerationSettlementState.mockRejectedValueOnce(new Error('transient Redis outage')); + const row = { unfinished: true }; + const running = harvest(source, row); + await jest.advanceTimersByTimeAsync(1_100_000); + const beforeSettle = running.getResult(); + expect(source.getGenerationSettlementState.mock.calls.length).toBeGreaterThan(1); + row.unfinished = false; + source.settle(); + await jest.advanceTimersByTimeAsync(0); + expect(beforeSettle).toBeUndefined(); + await running.pending; + expect(running.getResult()).toBe(true); + }); + + it('anchors the original result when a remote successor replaces the observed generation', async () => { + const source = sourceForRunningTurn(); + const row = { unfinished: true }; + const running = harvest(source, row); + await jest.advanceTimersByTimeAsync(1_000_000); + expect(running.update.mock.calls.length).toBe(14); + row.unfinished = false; + source.job.createdAt = 2; + source.job.status = 'requires_action'; + await jest.advanceTimersByTimeAsync(120_000); + const beforeSuccessorSettles = running.getResult(); + expect(running.update.mock.calls.length).toBe(15); + source.settle(); + await jest.advanceTimersByTimeAsync(0); + await running.pending; + expect(beforeSuccessorSettles).toBe(true); + }); + + it('does not retire a result while the real manager is awaiting terminal message persistence', async () => { + const manager = new GenerationJobManagerClass(); + const store = new InMemoryJobStore({ ttlAfterComplete: 0 }); + manager.configure({ + jobStore: store, + eventTransport: new InMemoryEventTransport(), + isRedis: false, + cleanupOnComplete: false, + }); + manager.initialize(); + let finishSave!: () => void; + const save = new Promise((resolve) => { + finishSave = resolve; + }); + try { + const job = await manager.createJob('conversation', 'user', 'conversation'); + const row = { unfinished: true }; + const running = harvest(manager, row); + await jest.advanceTimersByTimeAsync(1_000_000); + const completing = manager.completeJob('conversation', 'provider error', job.createdAt, { + beforeErrorPublication: () => save, + }); + await jest.advanceTimersByTimeAsync(120_000); + const beforePersistence = running.getResult(); + expect((await store.getJob('conversation'))?.terminalPersistencePending).toBe(true); + row.unfinished = false; + finishSave(); + await completing; + await jest.advanceTimersByTimeAsync(0); + expect(beforePersistence).toBeUndefined(); + await running.pending; + expect(running.getResult()).toBe(true); + } finally { + finishSave(); + await manager.destroy(); + } + }); +}); diff --git a/packages/api/src/agents/harvest.spec.ts b/packages/api/src/agents/harvest.spec.ts index fe5c98ae0f..8994cfbb55 100644 --- a/packages/api/src/agents/harvest.spec.ts +++ b/packages/api/src/agents/harvest.spec.ts @@ -153,3 +153,218 @@ describe('createBackgroundToolResultHandler claim ownership', () => { ); }); }); + +describe('createBackgroundCodeResultHandler long dispatch turns', () => { + const RETRY_SCHEDULE_MS = 1_000_000; + const backgroundTask = { + taskId: 'task-1', + toolName: 'execute_code', + status: 'completed' as const, + settledAt: new Date('2026-09-24T00:00:00Z'), + completionWakeup: true as const, + }; + + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + function createHandler( + updateToolCallResult: jest.Mock, + waitForGenerationSettled?: (conversationId: string) => Promise, + ) { + return createBackgroundCodeResultHandler({ + req, + preflightCodeOutputBatch: jest + .fn() + .mockResolvedValue([{ file: params.artifact.files[0], sessionId: 'storage-session' }]), + processCodeOutput: jest.fn().mockResolvedValue({ file: { file_id: 'persisted-file' } }), + updateToolCallResult, + runPreviewFinalize: jest.fn(), + ...(waitForGenerationSettled != null && { waitForGenerationSettled }), + }); + } + + it('waits for a dispatch turn that outlives the retry schedule, then anchors', async () => { + let turnRunning = true; + const updateToolCallResult = jest.fn(async () => ({ + matched: true, + unfinished: turnRunning, + })); + let settle: (settled: boolean) => void = () => undefined; + const waitForGenerationSettled = jest.fn( + () => + new Promise((resolve) => { + settle = resolve; + }), + ); + const onFilesPersisted = jest.fn(); + const handler = createHandler(updateToolCallResult, waitForGenerationSettled); + + const result = handler({ ...params, backgroundTask, onFilesPersisted }); + await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS); + + expect(onFilesPersisted).toHaveBeenCalledWith([ + { file_id: 'persisted-file', stepId: 'step-1' }, + ]); + expect(onFilesPersisted.mock.invocationCallOrder[0]).toBeLessThan( + updateToolCallResult.mock.invocationCallOrder[0], + ); + expect(waitForGenerationSettled).toHaveBeenCalledWith('conversation-1', { + signal: expect.any(AbortSignal), + }); + const attemptsWhileRunning = updateToolCallResult.mock.calls.length; + await jest.advanceTimersByTimeAsync(60 * 60 * 1_000); + expect(updateToolCallResult).toHaveBeenCalledTimes(attemptsWhileRunning); + + turnRunning = false; + settle(true); + await jest.advanceTimersByTimeAsync(0); + + await expect(result).resolves.toEqual({ + attachments: [{ file_id: 'persisted-file', stepId: 'step-1' }], + deliveryReady: true, + }); + expect(updateToolCallResult).toHaveBeenCalledTimes(attemptsWhileRunning + 1); + }); + + it('waits for a running turn that has not saved its row yet', async () => { + let rowSaved = false; + const updateToolCallResult = jest.fn(async () => ({ matched: rowSaved, unfinished: false })); + const waitForGenerationSettled = jest.fn(async () => { + rowSaved = true; + return true; + }); + const handler = createHandler(updateToolCallResult, waitForGenerationSettled); + + const result = handler({ ...params, backgroundTask }); + await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS); + + await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: true })); + expect(waitForGenerationSettled).toHaveBeenCalledTimes(1); + }); + + it('gives the result up when no generation is running and the row never appears', async () => { + const updateToolCallResult = jest.fn(async () => ({ matched: false, unfinished: false })); + const waitForGenerationSettled = jest.fn(async () => false); + const handler = createHandler(updateToolCallResult, waitForGenerationSettled); + + const result = handler({ ...params, backgroundTask }); + await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS + 60_000); + + await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: false })); + expect(updateToolCallResult).toHaveBeenCalledTimes(14 + 5); + }); + + it('anchors a final save that lands well after the generation reads as settled', async () => { + let settledAt: number | undefined; + const updateToolCallResult = jest.fn(async () => ({ + matched: true, + unfinished: settledAt == null || Date.now() - settledAt < 40_000, + })); + const waitForGenerationSettled = jest.fn(async () => { + settledAt = Date.now(); + return true; + }); + const handler = createHandler(updateToolCallResult, waitForGenerationSettled); + + const result = handler({ ...params, backgroundTask }); + await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS + 60_000); + + await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: true })); + }); + + it('gives the result up when the turn ends without saving the tool call', async () => { + const updateToolCallResult = jest.fn(async () => ({ matched: true, unfinished: true })); + const waitForGenerationSettled = jest.fn(async () => true); + const handler = createHandler(updateToolCallResult, waitForGenerationSettled); + + const result = handler({ ...params, backgroundTask }); + await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS + 60_000); + + await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: false })); + expect(waitForGenerationSettled).toHaveBeenCalledTimes(1); + }); + + it('anchors as soon as the dispatch turn settles instead of at the next scheduled retry', async () => { + let turnRunning = true; + const updateToolCallResult = jest.fn(async () => ({ matched: true, unfinished: turnRunning })); + let settle: (settled: boolean) => void = () => undefined; + const waitForGenerationSettled = jest.fn( + () => + new Promise((resolve) => { + settle = resolve; + }), + ); + const handler = createHandler(updateToolCallResult, waitForGenerationSettled); + let finished = false; + + const result = handler({ ...params, backgroundTask }).then((value) => { + finished = true; + return value; + }); + /** 45 s in, the schedule is inside a 30 s gap (next step near 69 s). */ + await jest.advanceTimersByTimeAsync(45_000); + const attemptsBeforeSettle = updateToolCallResult.mock.calls.length; + turnRunning = false; + settle(true); + await jest.advanceTimersByTimeAsync(0); + + expect(finished).toBe(true); + await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: true })); + expect(updateToolCallResult).toHaveBeenCalledTimes(attemptsBeforeSettle + 1); + }); + + it.each([params.conversationId, 'another-conversation'])( + 'pins only the epoch owned by the host stream %s', + async (generationStreamId) => { + const wait = jest.fn(async () => true); + const handler = createBackgroundCodeResultHandler({ + req, + preflightCodeOutputBatch: async () => [], + processCodeOutput: jest.fn(), + runPreviewFinalize: jest.fn(), + updateToolCallResult: async () => ({ matched: true, unfinished: false }), + generationCreatedAt: 1234, + generationStreamId, + waitForGenerationSettled: wait, + }); + await handler({ ...params, backgroundTask }); + expect(wait).toHaveBeenCalledWith(params.conversationId, { + ...(generationStreamId === params.conversationId && { generationCreatedAt: 1234 }), + signal: expect.any(AbortSignal), + }); + }, + ); + + it('stops listening for the turn once the result is anchored', async () => { + const updateToolCallResult = jest.fn(async () => ({ matched: true, unfinished: false })); + let signal: AbortSignal | undefined; + const waitForGenerationSettled = jest.fn( + (_conversationId: string, options?: { signal?: AbortSignal }) => { + signal = options?.signal; + return new Promise(() => undefined); + }, + ); + const handler = createHandler(updateToolCallResult, waitForGenerationSettled); + + await expect(handler({ ...params, backgroundTask })).resolves.toEqual( + expect.objectContaining({ deliveryReady: true }), + ); + expect(signal?.aborted).toBe(true); + }); + + it('keeps the bounded schedule when no settlement signal is wired', async () => { + const updateToolCallResult = jest.fn(async () => ({ matched: true, unfinished: true })); + const handler = createHandler(updateToolCallResult); + + const result = handler({ ...params, backgroundTask }); + await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS); + + await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: false })); + expect(updateToolCallResult).toHaveBeenCalledTimes(14); + }); +}); diff --git a/packages/api/src/agents/harvest.ts b/packages/api/src/agents/harvest.ts index 84738597ad..63cd918861 100644 --- a/packages/api/src/agents/harvest.ts +++ b/packages/api/src/agents/harvest.ts @@ -1,5 +1,6 @@ import { logger } from '@librechat/data-schemas'; import type { PreparedCodeOutputEntry } from '~/files/code/preflight'; +import type { GenerationSettledWaitOptions } from '../stream/settled'; import type { CodeExecutionContext } from './execution'; import type { ServerRequest } from '~/types'; @@ -12,6 +13,9 @@ import type { ServerRequest } from '~/types'; const BACKGROUND_PATCH_RETRY_DELAYS_MS = [ 250, 500, 1_000, 2_000, 5_000, 10_000, 20_000, 30_000, 60_000, 120_000, 180_000, 240_000, 300_000, ]; +/** Bounded final attempts after the dispatch epoch has finished or disappeared. + * Live terminal persistence is waited for explicitly, not inferred from this delay. */ +const SETTLED_PATCH_RETRY_DELAYS_MS = [1_000, 5_000, 15_000, 30_000]; interface HarvestFileRef { id: string; name: string; @@ -51,6 +55,10 @@ export interface BackgroundToolResultState { } export interface CodeHarvestDeps { + /** Generation identity captured by the host at dispatch, not at harvest time. */ + generationCreatedAt?: number; + /** The stream owning that epoch; nested child conversations must observe their own. */ + generationStreamId?: string | null; req: ServerRequest; /** Data-schemas method: idempotent tool-call part patch + attachment append. */ updateToolCallResult: (params: { @@ -95,6 +103,12 @@ export interface CodeHarvestDeps { fileId: string; previewRevision?: number; }) => void; + /** Waits for the dispatch epoch, including terminal persistence. Read outages + * remain retryable; replacement ends the wait without following a successor. */ + waitForGenerationSettled?: ( + conversationId: string, + options?: GenerationSettledWaitOptions, + ) => Promise; } export interface CodeHarvestParams { @@ -119,31 +133,48 @@ export interface CodeHarvestParams { /** Re-reads local claim ownership on every retry so a same-generation * manual poll cannot be overwritten by a later automatic continuation. */ resolveBackgroundTask?: () => BackgroundToolResultState; + /** Called once generated files are stored, before the row patch, which can + * wait for the dispatch turn to end: a poll in that turn needs them now. */ + onFilesPersisted?: (attachments: unknown[]) => void; } export type CodeHarvestHandler = ( params: CodeHarvestParams, ) => Promise<{ attachments: unknown[]; deliveryReady?: boolean } | null>; -const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); +type BackgroundResultRowParams = { + userId: string; + messageId: string; + conversationId: string; + toolCallId: string; + stepId?: string; + agentId?: string; + output?: string; + attachments?: unknown[]; + backgroundTask?: BackgroundToolResultState; + resolveBackgroundTask?: () => BackgroundToolResultState; +}; -async function persistBackgroundToolResultRow( +/** Resolves `true` once `interrupt` fires, or `false` after `ms`; never both. */ +function sleepUnlessInterrupted(ms: number, interrupt?: Promise): Promise { + return new Promise((resolve) => { + const timer = setTimeout(() => resolve(false), ms); + void interrupt?.then(() => { + clearTimeout(timer); + resolve(true); + }); + }); +} + +/** `false` once `interrupt` cut the schedule short, so the caller can retry at once. */ +async function anchorBackgroundToolResultRow( updateToolCallResult: CodeHarvestDeps['updateToolCallResult'], - params: { - userId: string; - messageId: string; - conversationId: string; - toolCallId: string; - stepId?: string; - agentId?: string; - output?: string; - attachments?: unknown[]; - backgroundTask?: BackgroundToolResultState; - resolveBackgroundTask?: () => BackgroundToolResultState; - }, + params: BackgroundResultRowParams, + retryDelaysMs: readonly number[], + interrupt?: Promise, ): Promise { const { resolveBackgroundTask, ...persistedParams } = params; - for (let attempt = 0; attempt <= BACKGROUND_PATCH_RETRY_DELAYS_MS.length; attempt++) { + for (let attempt = 0; attempt <= retryDelaysMs.length; attempt++) { const currentBackgroundTask = resolveBackgroundTask?.() ?? persistedParams.backgroundTask; const result = await updateToolCallResult({ ...persistedParams, @@ -153,14 +184,74 @@ async function persistBackgroundToolResultRow( if (result.matched && !result.unfinished) { return true; } - if (attempt === BACKGROUND_PATCH_RETRY_DELAYS_MS.length) { + if (attempt === retryDelaysMs.length) { break; } - await sleep(BACKGROUND_PATCH_RETRY_DELAYS_MS[attempt]); + if (await sleepUnlessInterrupted(retryDelaysMs[attempt], interrupt)) { + return false; + } } return false; } +/** + * Patches the result onto the dispatch turn's row, which is absent or unfinished + * while that turn streams. The patch is retried on a schedule, but the turn + * settling is what actually makes the row patchable, so a settle observed + * mid-schedule retries at once instead of at the next scheduled step. A turn that + * outlives the schedule is waited for rather than given up. Only a positive + * settle cuts the schedule short: an unknown generation state keeps the schedule. + */ +async function persistBackgroundToolResultRow( + updateToolCallResult: CodeHarvestDeps['updateToolCallResult'], + params: BackgroundResultRowParams, + waitForGenerationSettled?: CodeHarvestDeps['waitForGenerationSettled'], + generationCreatedAt?: number, +): Promise { + if (waitForGenerationSettled == null) { + return anchorBackgroundToolResultRow( + updateToolCallResult, + params, + BACKGROUND_PATCH_RETRY_DELAYS_MS, + ); + } + const listening = new AbortController(); + const settlement = waitForGenerationSettled(params.conversationId, { + signal: listening.signal, + ...(generationCreatedAt == null ? {} : { generationCreatedAt }), + }).catch((error: unknown) => { + logger.warn( + `[background] Failed waiting for the dispatch turn of message ${params.messageId} to settle:`, + error, + ); + return false; + }); + const settledTurn = new Promise((resolve) => { + void settlement.then((settled) => { + if (settled) resolve(); + }); + }); + try { + const anchored = await anchorBackgroundToolResultRow( + updateToolCallResult, + params, + BACKGROUND_PATCH_RETRY_DELAYS_MS, + settledTurn, + ); + if (anchored) { + return true; + } + await settlement; + return anchorBackgroundToolResultRow( + updateToolCallResult, + params, + SETTLED_PATCH_RETRY_DELAYS_MS, + ); + } finally { + listening.abort(); + } +} + /** Persists an ordinary detached tool result without invoking code-artifact processing. */ export function createBackgroundToolResultHandler( deps: Pick, @@ -218,6 +309,9 @@ export function createBackgroundCodeResultHandler(deps: CodeHarvestDeps): CodeHa preflightCodeOutputBatch, processCodeOutput, runPreviewFinalize, + waitForGenerationSettled, + generationCreatedAt, + generationStreamId, } = deps; return async ({ toolCallId, @@ -233,6 +327,7 @@ export function createBackgroundCodeResultHandler(deps: CodeHarvestDeps): CodeHa reapply, backgroundTask, resolveBackgroundTask, + onFilesPersisted, }) => { const userId = req.user?.id; if (!userId || !messageId || !conversationId) { @@ -311,22 +406,30 @@ export function createBackgroundCodeResultHandler(deps: CodeHarvestDeps): CodeHa } } - const deliveryReady = await persistBackgroundToolResultRow(updateToolCallResult, { - userId, - messageId, - conversationId, - toolCallId, - stepId, - agentId, - output, - attachments, - ...(backgroundTask != null ? { backgroundTask } : {}), - ...(resolveBackgroundTask != null ? { resolveBackgroundTask } : {}), - }); + onFilesPersisted?.(attachments); + const deliveryReady = await persistBackgroundToolResultRow( + updateToolCallResult, + { + userId, + messageId, + conversationId, + toolCallId, + stepId, + agentId, + output, + attachments, + ...(backgroundTask != null ? { backgroundTask } : {}), + ...(resolveBackgroundTask != null ? { resolveBackgroundTask } : {}), + }, + waitForGenerationSettled, + generationStreamId == null || generationStreamId === conversationId + ? generationCreatedAt + : undefined, + ); if (!deliveryReady) { logger.warn( `[background] Could not anchor code result onto message ${messageId} (tool call ${toolCallId}); ` + - 'the dispatch turn never persisted. Poll delivery still returns the result.', + 'the dispatch turn ended without saving that tool call. Poll delivery still returns the result.', ); } return { attachments, ...(backgroundTask != null ? { deliveryReady } : {}) }; diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts index a92318e29f..7a2120965e 100644 --- a/packages/api/src/stream/GenerationJobManager.ts +++ b/packages/api/src/stream/GenerationJobManager.ts @@ -20,6 +20,7 @@ import type { import type { StandardGraph } from '@librechat/agents'; import type { SerializableJobData, + GenerationSettlementState, CreatedJobData, IEventTransport, UsageMetadata, @@ -3772,9 +3773,21 @@ class GenerationJobManagerClass { await this.jobStore.releaseIdempotencyKey(legacyKey, expectedClaim); } - /** - * Get job status. - */ + /** Observes identity and final-save ownership without attaching a runtime or + * promoting a slow terminal writer to stale-owner recovery. */ + async getGenerationSettlementState( + streamId: string, + ): Promise { + const job = await this.jobStore.getJob(streamId); + if (job == null) return undefined; + return { + createdAt: job.createdAt, + status: job.status, + terminalPersistencePending: job.terminalPersistencePending, + }; + } + + /** Get job status. */ async getJobStatus(streamId: string): Promise { const jobData = await this.jobStore.getJob(streamId); return jobData?.status as t.GenerationJobStatus | undefined; diff --git a/packages/api/src/stream/__tests__/settled.spec.ts b/packages/api/src/stream/__tests__/settled.spec.ts new file mode 100644 index 0000000000..ff760d45dd --- /dev/null +++ b/packages/api/src/stream/__tests__/settled.spec.ts @@ -0,0 +1,276 @@ +import type { GenerationSettledEvent, GenerationSettledListener } from '../GenerationJobManager'; +import type { GenerationJobStatus } from '../../types/stream'; +import { waitForGenerationSettled } from '../settled'; + +function createSource(initial: Record) { + const statuses = new Map(Object.entries(initial)); + const listeners = new Set(); + const epochs = new Map(); + const pending = new Set(); + return { + statuses, + epochs, + pending, + listeners, + getGenerationSettlementState: jest.fn(async (streamId: string) => { + const status = statuses.get(streamId); + return status == null + ? undefined + : { + createdAt: epochs.get(streamId) ?? 1, + status, + terminalPersistencePending: pending.has(streamId), + }; + }), + onGenerationSettled: (listener: GenerationSettledListener) => { + listeners.add(listener); + return () => { + listeners.delete(listener); + }; + }, + settle(event: Omit) { + statuses.set(event.conversationId, event.status as GenerationJobStatus); + for (const listener of listeners) { + listener({ ...event, streamId: event.conversationId, userId: 'user-1' }); + } + }, + }; +} + +describe('waitForGenerationSettled', () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it.each([ + ['no generation', {}], + ['a completed generation', { 'conversation-1': 'complete' as const }], + ])('resolves false at once for %s', async (_label, initial) => { + const source = createSource(initial); + + await expect(waitForGenerationSettled(source, 'conversation-1')).resolves.toBe(false); + expect(source.listeners.size).toBe(0); + }); + + it('resolves when this conversation settles, ignoring others', async () => { + const source = createSource({ 'conversation-1': 'running', 'conversation-2': 'running' }); + let resolved: boolean | undefined; + void waitForGenerationSettled(source, 'conversation-1').then((settled) => { + resolved = settled; + }); + await jest.advanceTimersByTimeAsync(0); + + source.settle({ conversationId: 'conversation-2', status: 'complete' }); + await jest.advanceTimersByTimeAsync(0); + expect(resolved).toBeUndefined(); + + source.settle({ conversationId: 'conversation-1', status: 'complete' }); + await jest.advanceTimersByTimeAsync(0); + expect(resolved).toBe(true); + expect(source.listeners.size).toBe(0); + }); + + it('waits through an approval pause', async () => { + const source = createSource({ 'conversation-1': 'requires_action' }); + let resolved: boolean | undefined; + void waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 }).then( + (settled) => { + resolved = settled; + }, + ); + + await jest.advanceTimersByTimeAsync(5_000); + expect(resolved).toBeUndefined(); + }); + + it('observes a generation that settles on another replica', async () => { + const source = createSource({ 'conversation-1': 'running' }); + const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 }); + await jest.advanceTimersByTimeAsync(0); + + source.statuses.set('conversation-1', 'complete'); + await jest.advanceTimersByTimeAsync(1_000); + + await expect(waiting).resolves.toBe(true); + expect(source.listeners.size).toBe(0); + }); + + it('keeps waiting for as long as the generation stays active by default', async () => { + const source = createSource({ 'conversation-1': 'running' }); + let resolved: boolean | undefined; + void waitForGenerationSettled(source, 'conversation-1', { recheckMs: 60 * 60 * 1_000 }).then( + (settled) => { + resolved = settled; + }, + ); + + await jest.advanceTimersByTimeAsync(48 * 60 * 60 * 1_000); + expect(resolved).toBeUndefined(); + + source.settle({ conversationId: 'conversation-1', status: 'complete' }); + await jest.advanceTimersByTimeAsync(0); + expect(resolved).toBe(true); + }); + + it('gives up after the maximum wait', async () => { + const source = createSource({ 'conversation-1': 'running' }); + const waiting = waitForGenerationSettled(source, 'conversation-1', { + recheckMs: 1_000, + maxWaitMs: 10_000, + }); + + await jest.advanceTimersByTimeAsync(10_000); + + await expect(waiting).resolves.toBe(false); + expect(source.listeners.size).toBe(0); + }); + + it('stops waiting when the caller aborts', async () => { + const source = createSource({ 'conversation-1': 'running' }); + const controller = new AbortController(); + const waiting = waitForGenerationSettled(source, 'conversation-1', { + signal: controller.signal, + }); + await jest.advanceTimersByTimeAsync(0); + + controller.abort(); + + await expect(waiting).resolves.toBe(false); + expect(source.listeners.size).toBe(0); + }); + + it('recovers from initial and later read failures without unsubscribing', async () => { + const source = createSource({ 'conversation-1': 'running' }); + source.getGenerationSettlementState.mockRejectedValueOnce(new Error('job store unavailable')); + const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 }); + await jest.advanceTimersByTimeAsync(1_000); + expect(source.getGenerationSettlementState).toHaveBeenCalledTimes(2); + expect(source.listeners.size).toBe(1); + source.getGenerationSettlementState.mockRejectedValueOnce(new Error('later outage')); + await jest.advanceTimersByTimeAsync(1_000); + expect(source.listeners.size).toBe(1); + source.statuses.set('conversation-1', 'complete'); + await jest.advanceTimersByTimeAsync(1_000); + await expect(waiting).resolves.toBe(true); + expect(source.listeners.size).toBe(0); + }); + + it.each(['running', 'requires_action'] as const)( + 'does not follow a %s successor on another replica', + async (status) => { + const source = createSource({ 'conversation-1': 'running' }); + const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 }); + await jest.advanceTimersByTimeAsync(0); + source.statuses.set('conversation-1', status); + source.epochs.set('conversation-1', 2); + await jest.advanceTimersByTimeAsync(1_000); + await expect(waiting).resolves.toBe(true); + expect(source.listeners.size).toBe(0); + }, + ); + + it('uses the dispatch epoch even when the first successful read sees its successor', async () => { + const source = createSource({ 'conversation-1': 'requires_action' }); + source.epochs.set('conversation-1', 2); + source.getGenerationSettlementState.mockRejectedValueOnce(new Error('outage')); + const waiting = waitForGenerationSettled(source, 'conversation-1', { + generationCreatedAt: 1, + recheckMs: 1_000, + }); + await jest.advanceTimersByTimeAsync(1_000); + await expect(waiting).resolves.toBe(true); + }); + + it.each(['complete', 'error', 'aborted'] as const)( + 'waits for %s terminal persistence even after a stale settlement event', + async (status) => { + const source = createSource({ 'conversation-1': status }); + source.pending.add('conversation-1'); + let resolved = false; + const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 }).then( + (value) => { + resolved = true; + return value; + }, + ); + await jest.advanceTimersByTimeAsync(120_000); + source.settle({ conversationId: 'conversation-1', status }); + await jest.advanceTimersByTimeAsync(0); + expect(resolved).toBe(false); + source.pending.delete('conversation-1'); + await jest.advanceTimersByTimeAsync(1_000); + await expect(waiting).resolves.toBe(true); + }, + ); + + it('finishes when the observed job disappears', async () => { + const source = createSource({ 'conversation-1': 'running' }); + const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 }); + await jest.advanceTimersByTimeAsync(0); + source.statuses.delete('conversation-1'); + await jest.advanceTimersByTimeAsync(1_000); + await expect(waiting).resolves.toBe(true); + }); + + it('resolves a pre-aborted wait without subscribing or creating timers', async () => { + const source = createSource({ 'conversation-1': 'running' }); + const controller = new AbortController(); + controller.abort(); + await expect( + waitForGenerationSettled(source, 'conversation-1', { signal: controller.signal }), + ).resolves.toBe(false); + expect(source.getGenerationSettlementState).not.toHaveBeenCalled(); + expect(source.listeners.size).toBe(0); + expect(jest.getTimerCount()).toBe(0); + }); + + it('serializes reads and remembers a settlement event during an in-flight read', async () => { + const source = createSource({ 'conversation-1': 'running' }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + source.getGenerationSettlementState.mockImplementationOnce(async () => { + await gate; + return { createdAt: 1, status: 'running', terminalPersistencePending: false }; + }); + const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 }); + await jest.advanceTimersByTimeAsync(10_000); + source.settle({ conversationId: 'conversation-1', status: 'complete' }); + source.settle({ conversationId: 'conversation-1', status: 'complete' }); + expect(source.getGenerationSettlementState).toHaveBeenCalledTimes(1); + release(); + await expect(waiting).resolves.toBe(true); + expect(source.getGenerationSettlementState).toHaveBeenCalledTimes(2); + expect(jest.getTimerCount()).toBe(0); + }); + + it('removes abort listeners and ignores reads completing after cancellation', async () => { + const source = createSource({ 'conversation-1': 'running' }); + const controller = new AbortController(); + const remove = jest.spyOn(controller.signal, 'removeEventListener'); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + source.getGenerationSettlementState.mockImplementationOnce(async () => { + await gate; + return undefined; + }); + const waiting = waitForGenerationSettled(source, 'conversation-1', { + signal: controller.signal, + maxWaitMs: 500, + }); + controller.abort(); + await expect(waiting).resolves.toBe(false); + release(); + await jest.advanceTimersByTimeAsync(1_000); + expect(remove).toHaveBeenCalledWith('abort', expect.any(Function)); + expect(source.listeners.size).toBe(0); + expect(jest.getTimerCount()).toBe(0); + }); +}); diff --git a/packages/api/src/stream/index.ts b/packages/api/src/stream/index.ts index ef72b19ee5..9c8180e108 100644 --- a/packages/api/src/stream/index.ts +++ b/packages/api/src/stream/index.ts @@ -85,3 +85,7 @@ export { RedisJobStore } from './implementations/RedisJobStore'; export { RedisEventTransport } from './implementations/RedisEventTransport'; export { waitForGenerationPersistence } from './persistence'; +export { waitForGenerationSettled } from './settled'; +export type { GenerationSettledSource, GenerationSettledWaitOptions } from './settled'; + +export type { GenerationSettlementState } from './interfaces/IJobStore'; diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.ts index 4507661aeb..2e7d471b36 100644 --- a/packages/api/src/stream/interfaces/IJobStore.ts +++ b/packages/api/src/stream/interfaces/IJobStore.ts @@ -103,9 +103,13 @@ export type JobStatus = 'running' | 'complete' | 'error' | 'aborted' | 'requires * Missing markers on pre-rollout records are interpreted as protocol v1. */ export type GenerationProtocolVersion = 1 | 2; -/** - * Serializable job data - no object references, suitable for Redis/external storage - */ +/** Read-only durable evidence, without attaching a runtime or recovering a slow save. */ +export type GenerationSettlementState = Pick< + SerializableJobData, + 'createdAt' | 'status' | 'terminalPersistencePending' +>; + +/** Serializable job data without object references, suitable for external storage. */ export interface SerializableJobData { streamId: string; userId: string; diff --git a/packages/api/src/stream/settled.ts b/packages/api/src/stream/settled.ts new file mode 100644 index 0000000000..38fc7a092c --- /dev/null +++ b/packages/api/src/stream/settled.ts @@ -0,0 +1,114 @@ +import { logger } from '@librechat/data-schemas'; +import type { GenerationSettledListener } from './GenerationJobManager'; +import type { GenerationSettlementState } from './interfaces/IJobStore'; + +const DEFAULT_RECHECK_MS = 60_000; + +export interface GenerationSettledSource { + getGenerationSettlementState: ( + streamId: string, + ) => Promise; + onGenerationSettled: (listener: GenerationSettledListener) => () => void; +} + +export interface GenerationSettledWaitOptions { + recheckMs?: number; + /** Pin the dispatch generation even if it was replaced before the first read. */ + generationCreatedAt?: number; + /** Optional caller deadline. Store errors are not evidence of settlement. */ + maxWaitMs?: number; + /** Stops waiting (resolving `false`) once the caller no longer needs the answer. */ + signal?: AbortSignal; +} + +/** Waits for the dispatch generation, including terminal persistence. Events request + * a fresh read; they cannot settle an unrelated epoch or bypass an unfinished save. + * The first successful read pins the epoch if the caller did not supply it. Reads + * are single-flight, with a bounded periodic fallback for remote events and outages. */ +export function waitForGenerationSettled( + source: GenerationSettledSource, + conversationId: string, + { + recheckMs = DEFAULT_RECHECK_MS, + generationCreatedAt, + maxWaitMs, + signal, + }: GenerationSettledWaitOptions = {}, +): Promise { + if (signal?.aborted) { + return Promise.resolve(false); + } + return new Promise((resolve) => { + let done = false; + let reading = false; + let readRequested = false; + let epoch = generationCreatedAt; + let observedActive = epoch != null; + let unsubscribe: (() => void) | undefined = undefined; + let recheck: NodeJS.Timeout | undefined; + let deadline: NodeJS.Timeout | undefined; + const abort = (): void => finish(false); + const finish = (settled: boolean): void => { + if (done) return; + done = true; + unsubscribe?.(); + signal?.removeEventListener('abort', abort); + clearTimeout(recheck); + clearTimeout(deadline); + resolve(settled); + }; + const readState = async (): Promise => { + if (done) return; + if (reading) { + readRequested = true; + return; + } + reading = true; + clearTimeout(recheck); + try { + const job = await source.getGenerationSettlementState(conversationId); + if (done) return; + if (job == null || (epoch != null && epoch !== job.createdAt)) { + finish(observedActive); + return; + } + epoch = job.createdAt; + const active = + job.status === 'running' || + job.status === 'requires_action' || + job.terminalPersistencePending === true; + if (!active) { + finish(observedActive); + return; + } + observedActive = true; + } catch (error) { + logger.warn( + `[GenerationSettled] Failed to read generation state for ${conversationId}:`, + error, + ); + } finally { + reading = false; + if (!done) { + if (readRequested) { + readRequested = false; + void readState(); + } else { + recheck = setTimeout(() => void readState(), recheckMs); + recheck.unref?.(); + } + } + } + }; + if (maxWaitMs != null) { + deadline = setTimeout(() => finish(false), maxWaitMs); + deadline.unref?.(); + } + signal?.addEventListener('abort', abort, { once: true }); + unsubscribe = source.onGenerationSettled((event) => { + if (event.conversationId === conversationId) void readState(); + }); + if (done) unsubscribe(); + else void readState(); + }); +}