🧷 fix: Wait for Long Dispatch Turns Before Giving Up Background Code Results (#16345)

* 🐢 fix: Back Off Waiting Completion Wake-ups and Deliver Them When Ready

* fix: Signal every readiness path and make the completion wait cap configurable

* 🧷 fix: Wait for Long Dispatch Turns Before Giving Up Background Code Results

* fix: Mark held completion deliveries instead of pipeline expedite, and announce store-won approval expiry

* fix: Keep code tasks protected and wait without a fixed cutoff

* fix: Scope settle expedites to the resumed conversation and close the remaining signal gaps

* style: Format the settlement waiter

* fix: Announce a won approval expiry once and release subagent wake-up registrations

* fix: Outlast terminal persistence before giving up an unanchored code result

* fix: Anchor code results the moment their dispatch turn settles

* fix: Scope harvest epoch pinning to the owning stream

---------

Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
Danny Avila 2026-09-24 23:15:50 -04:00 • committed by GitHub
parent dd21e7fb6d
commit c6282fa888
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 1065 additions and 44 deletions

View file

@ -20,6 +20,7 @@ const {
sendEvent,
computeUsageCostUSD,
GenerationJobManager,
waitForGenerationSettled,
writeAttachmentEvent,
createToolExecuteHandler,
createOwnedToolEndHandler,
@ -1286,14 +1287,20 @@ function createPtcProgressEmitter({ res, streamId = null, jobCreatedAt }) {
* output?: string;
* attachments?: Object[];
* }) => Promise<boolean>} params.updateToolCallResult
* @param {number} [params.jobCreatedAt] - Immutable dispatch generation epoch.
* @param {string} [params.streamId] - The stream owning that epoch.
*/
function createBackgroundCodeResultHandler({ req, updateToolCallResult }) {
function createBackgroundCodeResultHandler({ req, updateToolCallResult, jobCreatedAt, streamId }) {
return createCodeHarvestHandler({
req,
updateToolCallResult,
preflightCodeOutputBatch,
processCodeOutput,
runPreviewFinalize,
generationCreatedAt: jobCreatedAt,
generationStreamId: streamId,
waitForGenerationSettled: (conversationId, options) =>
waitForGenerationSettled(GenerationJobManager, conversationId, options),
});
}

View file

@ -511,6 +511,8 @@ const initializeClientWithProvider = async ({
},
persistBackgroundCodeResult: createBackgroundCodeResultHandler({
req,
streamId,
jobCreatedAt,
updateToolCallResult: db.updateToolCallResult,
}),
backgroundToolCompletion: {

View file

@ -304,6 +304,9 @@ describe('initializeClient — processAgent ACL gate', () => {
jobCreatedAt: 1234,
}),
);
expect(
require('~/server/controllers/agents/callbacks').createBackgroundCodeResultHandler,
).toHaveBeenCalledWith(expect.objectContaining({ jobCreatedAt: 1234 }));
expect(createToolEndCallback).toHaveBeenCalledWith(
expect.objectContaining({
streamId: 'conv_1',

View file

@ -1369,6 +1369,54 @@ describe('BackgroundTaskRegistryClass', () => {
expect(registry.get('u1', 'c1', created.task.id)).toBeUndefined();
});
it.each(['completed', 'error', 'cancelled', 'blocked'] as const)(
'preserves pending %s results through both TTLs and releases protection afterward',
async (status) => {
jest.useFakeTimers();
try {
const registry = new BackgroundTaskRegistryClass();
const created = registry.create({
userId: 'u',
conversationId: 'c',
toolCallId: 'call',
toolName: 'execute_code',
harvestStarted: true,
});
if ('atCapacity' in created) throw new Error('Unexpected capacity rejection');
if (status === 'cancelled') registry.cancel('u', 'c', created.task.id, 'cancelled');
else if (status === 'error') registry.fail('u', 'c', created.task.id, 'failed');
else
registry.complete('u', 'c', created.task.id, { content: 'result', harvestStarted: true });
registry.markCompletionPersistencePending('u', 'c', created.task.id);
registry.finishHarvest('u', 'c', created.task.id, [{ file_id: 'file' }]);
if (status === 'blocked') registry.blockArtifact('u', 'c', created.task.id, 'blocked');
registry.claimResult('u', 'c', created.task.id, { kind: 'manual', claimId: 'poll' });
await jest.advanceTimersByTimeAsync(7 * 60 * 60_000);
expect(registry.get('u', 'c', created.task.id)).toMatchObject({
completionPersistencePending: true,
});
registry.markCompletionPersistenceFinished('u', 'c', created.task.id);
expect(
registry.get('u', 'c', created.task.id)?.completionPersistencePending,
).toBeUndefined();
if (status === 'blocked')
expect(registry.get('u', 'c', created.task.id)).toMatchObject({
artifactBlocked: true,
artifact: undefined,
});
if (status === 'completed')
expect(registry.get('u', 'c', created.task.id)).toMatchObject({
result: 'result',
resultClaim: { kind: 'manual', claimId: 'poll' },
});
await jest.advanceTimersByTimeAsync(61 * 60_000);
expect(registry.get('u', 'c', created.task.id)).toBeUndefined();
} finally {
jest.useRealTimers();
}
},
);
it('caps concurrent running tasks per conversation', () => {
const registry = new BackgroundTaskRegistryClass();
let atCapacity = false;

View file

@ -834,7 +834,11 @@ export class BackgroundTaskRegistryClass {
private sweepBucketTasks(bucket: TaskBucket, now: number): void {
for (const [taskId, task] of bucket.tasks) {
if (task.status !== 'running' && now - task.updatedAt > COMPLETED_TASK_TTL_MS) {
if (
task.status !== 'running' &&
task.completionPersistencePending !== true &&
now - task.updatedAt > COMPLETED_TASK_TTL_MS
) {
bucket.tasks.delete(taskId);
}
}
@ -859,7 +863,13 @@ export class BackgroundTaskRegistryClass {
}
this.lastGlobalSweepAt = now;
for (const [bucketKey, bucket] of this.buckets) {
if (now - bucket.lastAccess > IDLE_BUCKET_TTL_MS && bucket.capacityPermits.size === 0) {
if (
now - bucket.lastAccess > IDLE_BUCKET_TTL_MS &&
bucket.capacityPermits.size === 0 &&
![...bucket.tasks.values()].some(
(task) => task.status === 'running' || task.completionPersistencePending === true,
)
) {
this.buckets.delete(bucketKey);
continue;
}
@ -1615,7 +1625,14 @@ export class BackgroundTaskRegistryClass {
}
markCompletionPersistenceFinished(userId: string, conversationId: string, taskId: string): void {
this.update(userId, conversationId, taskId, { completionPersistencePending: undefined });
const bucket = this.buckets.get(this.key(userId, conversationId));
const task = bucket?.tasks.get(taskId);
if (bucket == null || task == null) return;
/** Even a policy-blocked task must release retention protection. This only
* clears lifecycle state; the immutable artifact block remains intact. */
task.completionPersistencePending = undefined;
task.updatedAt = Date.now();
bucket.lastAccess = task.updatedAt;
}
markCompletionPersistenceFailed(userId: string, conversationId: string, taskId: string): void {

View file

@ -2621,6 +2621,59 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
expect(poll[0].artifact).toBeUndefined();
});
it('releases stored files to same-turn polls while the row patch waits for the dispatch turn', async () => {
const state: CodeToolState = { calls: 0 };
const emitted: unknown[] = [];
const handler = createToolExecuteHandler({
loadTools: async () => ({ loadedTools: [makeCodeTool(state)] }),
/** Files are stored; the row patch then waits for the long dispatch turn. */
persistBackgroundCodeResult: (params) => {
params.onFilesPersisted?.([{ file_id: 'f1', toolCallId: params.toolCallId }]);
return new Promise(() => undefined);
},
emitAttachment: (attachment) => {
emitted.push(attachment);
},
});
const configurable = buildConfig(['execute_code']);
const dispatch = await runBatch(handler, {
toolCalls: [codeCall({ id: 'call_code_long_turn' })],
agentId: 'a',
configurable,
metadata: { thread_id: 'exec_convo_code_long_turn', run_id: 'msg-long-turn' },
});
await flushMicrotasks();
await flushMicrotasks();
const poll = (await runBatch(handler, {
toolCalls: [
{
id: 'call_poll_long_turn',
name: CHECK_BACKGROUND_TASK_NAME,
args: { background_task_id: JSON.parse(dispatch[0].content).background_task_id },
},
],
agentId: 'a',
configurable,
metadata: { thread_id: 'exec_convo_code_long_turn', run_id: 'msg-long-turn' },
})) as Array<{ content: string; artifact?: unknown }>;
const polled = JSON.parse(poll[0].content);
expect(polled.status).toBe('completed');
expect(polled.note).toContain('attached to the tool call');
expect(emitted[0]).toEqual({ file_id: 'f1', toolCallId: 'call_code_long_turn' });
expect(poll[0].artifact).toEqual(CODE_ARTIFACT);
/** Still protected from retention eviction while the row patch waits. */
expect(
backgroundTaskRegistry.get(
'exec_user',
'exec_convo_code_long_turn',
JSON.parse(dispatch[0].content).background_task_id,
)?.completionPersistencePending,
).toBe(true);
});
it('falls back to poll-turn delivery when the harvest fails (files not lost)', async () => {
const state: CodeToolState = { calls: 0 };
const toolEndCalls: Array<{ name?: string; artifact?: unknown }> = [];

View file

@ -315,6 +315,7 @@ export interface ToolExecuteOptions {
reapply?: boolean;
backgroundTask?: BackgroundToolResultState;
resolveBackgroundTask?: () => BackgroundToolResultState;
onFilesPersisted?: (attachments: unknown[]) => void;
}) => Promise<{ attachments?: unknown[]; deliveryReady?: boolean } | null>;
/** Shared ordinary-tool completion lifecycle. The delivery is registered
* before invoke; settlement is persisted onto the original response row. */
@ -6169,6 +6170,13 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
: { backgroundTask, resolveBackgroundTask }),
output: params.output ?? localTask?.result,
artifact: params.artifact,
onFilesPersisted: (attachments) =>
backgroundTaskRegistry.finishHarvest(
backgroundUserId,
backgroundConversationId,
task.id,
attachments,
),
});
if (persisted == null) {
/** Harvest never persisted anything (missing anchor
@ -6257,10 +6265,8 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
artifact?: unknown;
status: 'completed' | 'error' | 'cancelled';
}): Promise<void> => {
if (harvestEnabled) {
await persistBackgroundResult(params);
return;
}
/** Held for the whole persist, including a code harvest that waits for
* a long dispatch turn, so retention pressure cannot evict the task. */
backgroundTaskRegistry.markCompletionPersistencePending(
backgroundUserId,
backgroundConversationId,

View file

@ -0,0 +1,156 @@
import type { GenerationSettlementState } from '../stream/interfaces/IJobStore';
import type { GenerationSettledListener } from '../stream/GenerationJobManager';
import type { GenerationSettledSource } from '../stream/settled';
import type { GenerationJobStatus } from '../types/stream';
import type { ServerRequest } from '~/types';
import { InMemoryEventTransport } from '../stream/implementations/InMemoryEventTransport';
import { InMemoryJobStore } from '../stream/implementations/InMemoryJobStore';
import { GenerationJobManagerClass } from '../stream/GenerationJobManager';
import { createBackgroundCodeResultHandler } from './harvest';
import { waitForGenerationSettled } from '../stream/settled';
function sourceForRunningTurn() {
const listeners = new Set<GenerationSettledListener>();
const job: {
status: GenerationJobStatus;
createdAt: number;
terminalPersistencePending?: boolean;
} = { status: 'running', createdAt: 1 };
return {
job,
listeners,
getGenerationSettlementState: jest.fn(
async (): Promise<GenerationSettlementState | undefined> => ({ ...job }),
),
onGenerationSettled: (listener: GenerationSettledListener) => {
listeners.add(listener);
return () => {
listeners.delete(listener);
};
},
settle: () => {
job.status = 'complete';
for (const listener of [...listeners])
listener({
streamId: 'conversation',
conversationId: 'conversation',
userId: 'user',
status: 'complete',
});
},
};
}
function harvest(source: GenerationSettledSource, row: { unfinished: boolean }) {
const update = jest.fn(async () => ({ matched: true, unfinished: row.unfinished }));
const handler = createBackgroundCodeResultHandler({
req: { user: { id: 'user' } } as ServerRequest,
preflightCodeOutputBatch: async () => [],
processCodeOutput: jest.fn(),
updateToolCallResult: update,
runPreviewFinalize: jest.fn(),
waitForGenerationSettled: (conversationId, options) =>
waitForGenerationSettled(source, conversationId, options),
});
let result: boolean | undefined;
const pending = handler({
toolName: 'execute_code',
toolCallId: 'call',
stepId: 'step',
messageId: 'message',
conversationId: 'conversation',
output: 'finished code',
backgroundTask: {
taskId: 'task',
toolName: 'execute_code',
status: 'completed',
settledAt: new Date(),
completionWakeup: true,
},
}).then((value) => {
result = value?.deliveryReady;
});
return { update, pending, getResult: () => result };
}
describe('background code harvest generation lifecycle', () => {
beforeEach(() => {
jest.useFakeTimers();
});
afterEach(() => {
jest.clearAllTimers();
jest.useRealTimers();
});
it('keeps harvesting a live turn after a transient first status-read failure', async () => {
const source = sourceForRunningTurn();
source.getGenerationSettlementState.mockRejectedValueOnce(new Error('transient Redis outage'));
const row = { unfinished: true };
const running = harvest(source, row);
await jest.advanceTimersByTimeAsync(1_100_000);
const beforeSettle = running.getResult();
expect(source.getGenerationSettlementState.mock.calls.length).toBeGreaterThan(1);
row.unfinished = false;
source.settle();
await jest.advanceTimersByTimeAsync(0);
expect(beforeSettle).toBeUndefined();
await running.pending;
expect(running.getResult()).toBe(true);
});
it('anchors the original result when a remote successor replaces the observed generation', async () => {
const source = sourceForRunningTurn();
const row = { unfinished: true };
const running = harvest(source, row);
await jest.advanceTimersByTimeAsync(1_000_000);
expect(running.update.mock.calls.length).toBe(14);
row.unfinished = false;
source.job.createdAt = 2;
source.job.status = 'requires_action';
await jest.advanceTimersByTimeAsync(120_000);
const beforeSuccessorSettles = running.getResult();
expect(running.update.mock.calls.length).toBe(15);
source.settle();
await jest.advanceTimersByTimeAsync(0);
await running.pending;
expect(beforeSuccessorSettles).toBe(true);
});
it('does not retire a result while the real manager is awaiting terminal message persistence', async () => {
const manager = new GenerationJobManagerClass();
const store = new InMemoryJobStore({ ttlAfterComplete: 0 });
manager.configure({
jobStore: store,
eventTransport: new InMemoryEventTransport(),
isRedis: false,
cleanupOnComplete: false,
});
manager.initialize();
let finishSave!: () => void;
const save = new Promise<void>((resolve) => {
finishSave = resolve;
});
try {
const job = await manager.createJob('conversation', 'user', 'conversation');
const row = { unfinished: true };
const running = harvest(manager, row);
await jest.advanceTimersByTimeAsync(1_000_000);
const completing = manager.completeJob('conversation', 'provider error', job.createdAt, {
beforeErrorPublication: () => save,
});
await jest.advanceTimersByTimeAsync(120_000);
const beforePersistence = running.getResult();
expect((await store.getJob('conversation'))?.terminalPersistencePending).toBe(true);
row.unfinished = false;
finishSave();
await completing;
await jest.advanceTimersByTimeAsync(0);
expect(beforePersistence).toBeUndefined();
await running.pending;
expect(running.getResult()).toBe(true);
} finally {
finishSave();
await manager.destroy();
}
});
});

View file

@ -153,3 +153,218 @@ describe('createBackgroundToolResultHandler claim ownership', () => {
);
});
});
describe('createBackgroundCodeResultHandler long dispatch turns', () => {
const RETRY_SCHEDULE_MS = 1_000_000;
const backgroundTask = {
taskId: 'task-1',
toolName: 'execute_code',
status: 'completed' as const,
settledAt: new Date('2026-09-24T00:00:00Z'),
completionWakeup: true as const,
};
beforeEach(() => {
jest.useFakeTimers();
});
afterEach(() => {
jest.useRealTimers();
});
function createHandler(
updateToolCallResult: jest.Mock,
waitForGenerationSettled?: (conversationId: string) => Promise<boolean>,
) {
return createBackgroundCodeResultHandler({
req,
preflightCodeOutputBatch: jest
.fn()
.mockResolvedValue([{ file: params.artifact.files[0], sessionId: 'storage-session' }]),
processCodeOutput: jest.fn().mockResolvedValue({ file: { file_id: 'persisted-file' } }),
updateToolCallResult,
runPreviewFinalize: jest.fn(),
...(waitForGenerationSettled != null && { waitForGenerationSettled }),
});
}
it('waits for a dispatch turn that outlives the retry schedule, then anchors', async () => {
let turnRunning = true;
const updateToolCallResult = jest.fn(async () => ({
matched: true,
unfinished: turnRunning,
}));
let settle: (settled: boolean) => void = () => undefined;
const waitForGenerationSettled = jest.fn(
() =>
new Promise<boolean>((resolve) => {
settle = resolve;
}),
);
const onFilesPersisted = jest.fn();
const handler = createHandler(updateToolCallResult, waitForGenerationSettled);
const result = handler({ ...params, backgroundTask, onFilesPersisted });
await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS);
expect(onFilesPersisted).toHaveBeenCalledWith([
{ file_id: 'persisted-file', stepId: 'step-1' },
]);
expect(onFilesPersisted.mock.invocationCallOrder[0]).toBeLessThan(
updateToolCallResult.mock.invocationCallOrder[0],
);
expect(waitForGenerationSettled).toHaveBeenCalledWith('conversation-1', {
signal: expect.any(AbortSignal),
});
const attemptsWhileRunning = updateToolCallResult.mock.calls.length;
await jest.advanceTimersByTimeAsync(60 * 60 * 1_000);
expect(updateToolCallResult).toHaveBeenCalledTimes(attemptsWhileRunning);
turnRunning = false;
settle(true);
await jest.advanceTimersByTimeAsync(0);
await expect(result).resolves.toEqual({
attachments: [{ file_id: 'persisted-file', stepId: 'step-1' }],
deliveryReady: true,
});
expect(updateToolCallResult).toHaveBeenCalledTimes(attemptsWhileRunning + 1);
});
it('waits for a running turn that has not saved its row yet', async () => {
let rowSaved = false;
const updateToolCallResult = jest.fn(async () => ({ matched: rowSaved, unfinished: false }));
const waitForGenerationSettled = jest.fn(async () => {
rowSaved = true;
return true;
});
const handler = createHandler(updateToolCallResult, waitForGenerationSettled);
const result = handler({ ...params, backgroundTask });
await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS);
await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: true }));
expect(waitForGenerationSettled).toHaveBeenCalledTimes(1);
});
it('gives the result up when no generation is running and the row never appears', async () => {
const updateToolCallResult = jest.fn(async () => ({ matched: false, unfinished: false }));
const waitForGenerationSettled = jest.fn(async () => false);
const handler = createHandler(updateToolCallResult, waitForGenerationSettled);
const result = handler({ ...params, backgroundTask });
await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS + 60_000);
await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: false }));
expect(updateToolCallResult).toHaveBeenCalledTimes(14 + 5);
});
it('anchors a final save that lands well after the generation reads as settled', async () => {
let settledAt: number | undefined;
const updateToolCallResult = jest.fn(async () => ({
matched: true,
unfinished: settledAt == null || Date.now() - settledAt < 40_000,
}));
const waitForGenerationSettled = jest.fn(async () => {
settledAt = Date.now();
return true;
});
const handler = createHandler(updateToolCallResult, waitForGenerationSettled);
const result = handler({ ...params, backgroundTask });
await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS + 60_000);
await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: true }));
});
it('gives the result up when the turn ends without saving the tool call', async () => {
const updateToolCallResult = jest.fn(async () => ({ matched: true, unfinished: true }));
const waitForGenerationSettled = jest.fn(async () => true);
const handler = createHandler(updateToolCallResult, waitForGenerationSettled);
const result = handler({ ...params, backgroundTask });
await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS + 60_000);
await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: false }));
expect(waitForGenerationSettled).toHaveBeenCalledTimes(1);
});
it('anchors as soon as the dispatch turn settles instead of at the next scheduled retry', async () => {
let turnRunning = true;
const updateToolCallResult = jest.fn(async () => ({ matched: true, unfinished: turnRunning }));
let settle: (settled: boolean) => void = () => undefined;
const waitForGenerationSettled = jest.fn(
() =>
new Promise<boolean>((resolve) => {
settle = resolve;
}),
);
const handler = createHandler(updateToolCallResult, waitForGenerationSettled);
let finished = false;
const result = handler({ ...params, backgroundTask }).then((value) => {
finished = true;
return value;
});
/** 45 s in, the schedule is inside a 30 s gap (next step near 69 s). */
await jest.advanceTimersByTimeAsync(45_000);
const attemptsBeforeSettle = updateToolCallResult.mock.calls.length;
turnRunning = false;
settle(true);
await jest.advanceTimersByTimeAsync(0);
expect(finished).toBe(true);
await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: true }));
expect(updateToolCallResult).toHaveBeenCalledTimes(attemptsBeforeSettle + 1);
});
it.each([params.conversationId, 'another-conversation'])(
'pins only the epoch owned by the host stream %s',
async (generationStreamId) => {
const wait = jest.fn(async () => true);
const handler = createBackgroundCodeResultHandler({
req,
preflightCodeOutputBatch: async () => [],
processCodeOutput: jest.fn(),
runPreviewFinalize: jest.fn(),
updateToolCallResult: async () => ({ matched: true, unfinished: false }),
generationCreatedAt: 1234,
generationStreamId,
waitForGenerationSettled: wait,
});
await handler({ ...params, backgroundTask });
expect(wait).toHaveBeenCalledWith(params.conversationId, {
...(generationStreamId === params.conversationId && { generationCreatedAt: 1234 }),
signal: expect.any(AbortSignal),
});
},
);
it('stops listening for the turn once the result is anchored', async () => {
const updateToolCallResult = jest.fn(async () => ({ matched: true, unfinished: false }));
let signal: AbortSignal | undefined;
const waitForGenerationSettled = jest.fn(
(_conversationId: string, options?: { signal?: AbortSignal }) => {
signal = options?.signal;
return new Promise<boolean>(() => undefined);
},
);
const handler = createHandler(updateToolCallResult, waitForGenerationSettled);
await expect(handler({ ...params, backgroundTask })).resolves.toEqual(
expect.objectContaining({ deliveryReady: true }),
);
expect(signal?.aborted).toBe(true);
});
it('keeps the bounded schedule when no settlement signal is wired', async () => {
const updateToolCallResult = jest.fn(async () => ({ matched: true, unfinished: true }));
const handler = createHandler(updateToolCallResult);
const result = handler({ ...params, backgroundTask });
await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS);
await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: false }));
expect(updateToolCallResult).toHaveBeenCalledTimes(14);
});
});

View file

@ -1,5 +1,6 @@
import { logger } from '@librechat/data-schemas';
import type { PreparedCodeOutputEntry } from '~/files/code/preflight';
import type { GenerationSettledWaitOptions } from '../stream/settled';
import type { CodeExecutionContext } from './execution';
import type { ServerRequest } from '~/types';
@ -12,6 +13,9 @@ import type { ServerRequest } from '~/types';
const BACKGROUND_PATCH_RETRY_DELAYS_MS = [
250, 500, 1_000, 2_000, 5_000, 10_000, 20_000, 30_000, 60_000, 120_000, 180_000, 240_000, 300_000,
];
/** Bounded final attempts after the dispatch epoch has finished or disappeared.
* Live terminal persistence is waited for explicitly, not inferred from this delay. */
const SETTLED_PATCH_RETRY_DELAYS_MS = [1_000, 5_000, 15_000, 30_000];
interface HarvestFileRef {
id: string;
name: string;
@ -51,6 +55,10 @@ export interface BackgroundToolResultState {
}
export interface CodeHarvestDeps {
/** Generation identity captured by the host at dispatch, not at harvest time. */
generationCreatedAt?: number;
/** The stream owning that epoch; nested child conversations must observe their own. */
generationStreamId?: string | null;
req: ServerRequest;
/** Data-schemas method: idempotent tool-call part patch + attachment append. */
updateToolCallResult: (params: {
@ -95,6 +103,12 @@ export interface CodeHarvestDeps {
fileId: string;
previewRevision?: number;
}) => void;
/** Waits for the dispatch epoch, including terminal persistence. Read outages
* remain retryable; replacement ends the wait without following a successor. */
waitForGenerationSettled?: (
conversationId: string,
options?: GenerationSettledWaitOptions,
) => Promise<boolean>;
}
export interface CodeHarvestParams {
@ -119,31 +133,48 @@ export interface CodeHarvestParams {
/** Re-reads local claim ownership on every retry so a same-generation
* manual poll cannot be overwritten by a later automatic continuation. */
resolveBackgroundTask?: () => BackgroundToolResultState;
/** Called once generated files are stored, before the row patch, which can
* wait for the dispatch turn to end: a poll in that turn needs them now. */
onFilesPersisted?: (attachments: unknown[]) => void;
}
export type CodeHarvestHandler = (
params: CodeHarvestParams,
) => Promise<{ attachments: unknown[]; deliveryReady?: boolean } | null>;
const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
type BackgroundResultRowParams = {
userId: string;
messageId: string;
conversationId: string;
toolCallId: string;
stepId?: string;
agentId?: string;
output?: string;
attachments?: unknown[];
backgroundTask?: BackgroundToolResultState;
resolveBackgroundTask?: () => BackgroundToolResultState;
};
async function persistBackgroundToolResultRow(
/** Resolves `true` once `interrupt` fires, or `false` after `ms`; never both. */
function sleepUnlessInterrupted(ms: number, interrupt?: Promise<void>): Promise<boolean> {
return new Promise((resolve) => {
const timer = setTimeout(() => resolve(false), ms);
void interrupt?.then(() => {
clearTimeout(timer);
resolve(true);
});
});
}
/** `false` once `interrupt` cut the schedule short, so the caller can retry at once. */
async function anchorBackgroundToolResultRow(
updateToolCallResult: CodeHarvestDeps['updateToolCallResult'],
params: {
userId: string;
messageId: string;
conversationId: string;
toolCallId: string;
stepId?: string;
agentId?: string;
output?: string;
attachments?: unknown[];
backgroundTask?: BackgroundToolResultState;
resolveBackgroundTask?: () => BackgroundToolResultState;
},
params: BackgroundResultRowParams,
retryDelaysMs: readonly number[],
interrupt?: Promise<void>,
): Promise<boolean> {
const { resolveBackgroundTask, ...persistedParams } = params;
for (let attempt = 0; attempt <= BACKGROUND_PATCH_RETRY_DELAYS_MS.length; attempt++) {
for (let attempt = 0; attempt <= retryDelaysMs.length; attempt++) {
const currentBackgroundTask = resolveBackgroundTask?.() ?? persistedParams.backgroundTask;
const result = await updateToolCallResult({
...persistedParams,
@ -153,14 +184,74 @@ async function persistBackgroundToolResultRow(
if (result.matched && !result.unfinished) {
return true;
}
if (attempt === BACKGROUND_PATCH_RETRY_DELAYS_MS.length) {
if (attempt === retryDelaysMs.length) {
break;
}
await sleep(BACKGROUND_PATCH_RETRY_DELAYS_MS[attempt]);
if (await sleepUnlessInterrupted(retryDelaysMs[attempt], interrupt)) {
return false;
}
}
return false;
}
/**
* Patches the result onto the dispatch turn's row, which is absent or unfinished
* while that turn streams. The patch is retried on a schedule, but the turn
* settling is what actually makes the row patchable, so a settle observed
* mid-schedule retries at once instead of at the next scheduled step. A turn that
* outlives the schedule is waited for rather than given up. Only a positive
* settle cuts the schedule short: an unknown generation state keeps the schedule.
*/
async function persistBackgroundToolResultRow(
updateToolCallResult: CodeHarvestDeps['updateToolCallResult'],
params: BackgroundResultRowParams,
waitForGenerationSettled?: CodeHarvestDeps['waitForGenerationSettled'],
generationCreatedAt?: number,
): Promise<boolean> {
if (waitForGenerationSettled == null) {
return anchorBackgroundToolResultRow(
updateToolCallResult,
params,
BACKGROUND_PATCH_RETRY_DELAYS_MS,
);
}
const listening = new AbortController();
const settlement = waitForGenerationSettled(params.conversationId, {
signal: listening.signal,
...(generationCreatedAt == null ? {} : { generationCreatedAt }),
}).catch((error: unknown) => {
logger.warn(
`[background] Failed waiting for the dispatch turn of message ${params.messageId} to settle:`,
error,
);
return false;
});
const settledTurn = new Promise<void>((resolve) => {
void settlement.then((settled) => {
if (settled) resolve();
});
});
try {
const anchored = await anchorBackgroundToolResultRow(
updateToolCallResult,
params,
BACKGROUND_PATCH_RETRY_DELAYS_MS,
settledTurn,
);
if (anchored) {
return true;
}
await settlement;
return anchorBackgroundToolResultRow(
updateToolCallResult,
params,
SETTLED_PATCH_RETRY_DELAYS_MS,
);
} finally {
listening.abort();
}
}
/** Persists an ordinary detached tool result without invoking code-artifact processing. */
export function createBackgroundToolResultHandler(
deps: Pick<CodeHarvestDeps, 'req' | 'updateToolCallResult'>,
@ -218,6 +309,9 @@ export function createBackgroundCodeResultHandler(deps: CodeHarvestDeps): CodeHa
preflightCodeOutputBatch,
processCodeOutput,
runPreviewFinalize,
waitForGenerationSettled,
generationCreatedAt,
generationStreamId,
} = deps;
return async ({
toolCallId,
@ -233,6 +327,7 @@ export function createBackgroundCodeResultHandler(deps: CodeHarvestDeps): CodeHa
reapply,
backgroundTask,
resolveBackgroundTask,
onFilesPersisted,
}) => {
const userId = req.user?.id;
if (!userId || !messageId || !conversationId) {
@ -311,22 +406,30 @@ export function createBackgroundCodeResultHandler(deps: CodeHarvestDeps): CodeHa
}
}
const deliveryReady = await persistBackgroundToolResultRow(updateToolCallResult, {
userId,
messageId,
conversationId,
toolCallId,
stepId,
agentId,
output,
attachments,
...(backgroundTask != null ? { backgroundTask } : {}),
...(resolveBackgroundTask != null ? { resolveBackgroundTask } : {}),
});
onFilesPersisted?.(attachments);
const deliveryReady = await persistBackgroundToolResultRow(
updateToolCallResult,
{
userId,
messageId,
conversationId,
toolCallId,
stepId,
agentId,
output,
attachments,
...(backgroundTask != null ? { backgroundTask } : {}),
...(resolveBackgroundTask != null ? { resolveBackgroundTask } : {}),
},
waitForGenerationSettled,
generationStreamId == null || generationStreamId === conversationId
? generationCreatedAt
: undefined,
);
if (!deliveryReady) {
logger.warn(
`[background] Could not anchor code result onto message ${messageId} (tool call ${toolCallId}); ` +
'the dispatch turn never persisted. Poll delivery still returns the result.',
'the dispatch turn ended without saving that tool call. Poll delivery still returns the result.',
);
}
return { attachments, ...(backgroundTask != null ? { deliveryReady } : {}) };

View file

@ -20,6 +20,7 @@ import type {
import type { StandardGraph } from '@librechat/agents';
import type {
SerializableJobData,
GenerationSettlementState,
CreatedJobData,
IEventTransport,
UsageMetadata,
@ -3772,9 +3773,21 @@ class GenerationJobManagerClass {
await this.jobStore.releaseIdempotencyKey(legacyKey, expectedClaim);
}
/**
* Get job status.
*/
/** Observes identity and final-save ownership without attaching a runtime or
* promoting a slow terminal writer to stale-owner recovery. */
async getGenerationSettlementState(
streamId: string,
): Promise<GenerationSettlementState | undefined> {
const job = await this.jobStore.getJob(streamId);
if (job == null) return undefined;
return {
createdAt: job.createdAt,
status: job.status,
terminalPersistencePending: job.terminalPersistencePending,
};
}
/** Get job status. */
async getJobStatus(streamId: string): Promise<t.GenerationJobStatus | undefined> {
const jobData = await this.jobStore.getJob(streamId);
return jobData?.status as t.GenerationJobStatus | undefined;

View file

@ -0,0 +1,276 @@
import type { GenerationSettledEvent, GenerationSettledListener } from '../GenerationJobManager';
import type { GenerationJobStatus } from '../../types/stream';
import { waitForGenerationSettled } from '../settled';
function createSource(initial: Record<string, GenerationJobStatus>) {
const statuses = new Map(Object.entries(initial));
const listeners = new Set<GenerationSettledListener>();
const epochs = new Map<string, number>();
const pending = new Set<string>();
return {
statuses,
epochs,
pending,
listeners,
getGenerationSettlementState: jest.fn(async (streamId: string) => {
const status = statuses.get(streamId);
return status == null
? undefined
: {
createdAt: epochs.get(streamId) ?? 1,
status,
terminalPersistencePending: pending.has(streamId),
};
}),
onGenerationSettled: (listener: GenerationSettledListener) => {
listeners.add(listener);
return () => {
listeners.delete(listener);
};
},
settle(event: Omit<GenerationSettledEvent, 'streamId' | 'userId'>) {
statuses.set(event.conversationId, event.status as GenerationJobStatus);
for (const listener of listeners) {
listener({ ...event, streamId: event.conversationId, userId: 'user-1' });
}
},
};
}
describe('waitForGenerationSettled', () => {
beforeEach(() => {
jest.useFakeTimers();
});
afterEach(() => {
jest.useRealTimers();
});
it.each([
['no generation', {}],
['a completed generation', { 'conversation-1': 'complete' as const }],
])('resolves false at once for %s', async (_label, initial) => {
const source = createSource(initial);
await expect(waitForGenerationSettled(source, 'conversation-1')).resolves.toBe(false);
expect(source.listeners.size).toBe(0);
});
it('resolves when this conversation settles, ignoring others', async () => {
const source = createSource({ 'conversation-1': 'running', 'conversation-2': 'running' });
let resolved: boolean | undefined;
void waitForGenerationSettled(source, 'conversation-1').then((settled) => {
resolved = settled;
});
await jest.advanceTimersByTimeAsync(0);
source.settle({ conversationId: 'conversation-2', status: 'complete' });
await jest.advanceTimersByTimeAsync(0);
expect(resolved).toBeUndefined();
source.settle({ conversationId: 'conversation-1', status: 'complete' });
await jest.advanceTimersByTimeAsync(0);
expect(resolved).toBe(true);
expect(source.listeners.size).toBe(0);
});
it('waits through an approval pause', async () => {
const source = createSource({ 'conversation-1': 'requires_action' });
let resolved: boolean | undefined;
void waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 }).then(
(settled) => {
resolved = settled;
},
);
await jest.advanceTimersByTimeAsync(5_000);
expect(resolved).toBeUndefined();
});
it('observes a generation that settles on another replica', async () => {
const source = createSource({ 'conversation-1': 'running' });
const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 });
await jest.advanceTimersByTimeAsync(0);
source.statuses.set('conversation-1', 'complete');
await jest.advanceTimersByTimeAsync(1_000);
await expect(waiting).resolves.toBe(true);
expect(source.listeners.size).toBe(0);
});
it('keeps waiting for as long as the generation stays active by default', async () => {
const source = createSource({ 'conversation-1': 'running' });
let resolved: boolean | undefined;
void waitForGenerationSettled(source, 'conversation-1', { recheckMs: 60 * 60 * 1_000 }).then(
(settled) => {
resolved = settled;
},
);
await jest.advanceTimersByTimeAsync(48 * 60 * 60 * 1_000);
expect(resolved).toBeUndefined();
source.settle({ conversationId: 'conversation-1', status: 'complete' });
await jest.advanceTimersByTimeAsync(0);
expect(resolved).toBe(true);
});
it('gives up after the maximum wait', async () => {
const source = createSource({ 'conversation-1': 'running' });
const waiting = waitForGenerationSettled(source, 'conversation-1', {
recheckMs: 1_000,
maxWaitMs: 10_000,
});
await jest.advanceTimersByTimeAsync(10_000);
await expect(waiting).resolves.toBe(false);
expect(source.listeners.size).toBe(0);
});
it('stops waiting when the caller aborts', async () => {
const source = createSource({ 'conversation-1': 'running' });
const controller = new AbortController();
const waiting = waitForGenerationSettled(source, 'conversation-1', {
signal: controller.signal,
});
await jest.advanceTimersByTimeAsync(0);
controller.abort();
await expect(waiting).resolves.toBe(false);
expect(source.listeners.size).toBe(0);
});
it('recovers from initial and later read failures without unsubscribing', async () => {
const source = createSource({ 'conversation-1': 'running' });
source.getGenerationSettlementState.mockRejectedValueOnce(new Error('job store unavailable'));
const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 });
await jest.advanceTimersByTimeAsync(1_000);
expect(source.getGenerationSettlementState).toHaveBeenCalledTimes(2);
expect(source.listeners.size).toBe(1);
source.getGenerationSettlementState.mockRejectedValueOnce(new Error('later outage'));
await jest.advanceTimersByTimeAsync(1_000);
expect(source.listeners.size).toBe(1);
source.statuses.set('conversation-1', 'complete');
await jest.advanceTimersByTimeAsync(1_000);
await expect(waiting).resolves.toBe(true);
expect(source.listeners.size).toBe(0);
});
it.each(['running', 'requires_action'] as const)(
'does not follow a %s successor on another replica',
async (status) => {
const source = createSource({ 'conversation-1': 'running' });
const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 });
await jest.advanceTimersByTimeAsync(0);
source.statuses.set('conversation-1', status);
source.epochs.set('conversation-1', 2);
await jest.advanceTimersByTimeAsync(1_000);
await expect(waiting).resolves.toBe(true);
expect(source.listeners.size).toBe(0);
},
);
it('uses the dispatch epoch even when the first successful read sees its successor', async () => {
const source = createSource({ 'conversation-1': 'requires_action' });
source.epochs.set('conversation-1', 2);
source.getGenerationSettlementState.mockRejectedValueOnce(new Error('outage'));
const waiting = waitForGenerationSettled(source, 'conversation-1', {
generationCreatedAt: 1,
recheckMs: 1_000,
});
await jest.advanceTimersByTimeAsync(1_000);
await expect(waiting).resolves.toBe(true);
});
it.each(['complete', 'error', 'aborted'] as const)(
'waits for %s terminal persistence even after a stale settlement event',
async (status) => {
const source = createSource({ 'conversation-1': status });
source.pending.add('conversation-1');
let resolved = false;
const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 }).then(
(value) => {
resolved = true;
return value;
},
);
await jest.advanceTimersByTimeAsync(120_000);
source.settle({ conversationId: 'conversation-1', status });
await jest.advanceTimersByTimeAsync(0);
expect(resolved).toBe(false);
source.pending.delete('conversation-1');
await jest.advanceTimersByTimeAsync(1_000);
await expect(waiting).resolves.toBe(true);
},
);
it('finishes when the observed job disappears', async () => {
const source = createSource({ 'conversation-1': 'running' });
const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 });
await jest.advanceTimersByTimeAsync(0);
source.statuses.delete('conversation-1');
await jest.advanceTimersByTimeAsync(1_000);
await expect(waiting).resolves.toBe(true);
});
it('resolves a pre-aborted wait without subscribing or creating timers', async () => {
const source = createSource({ 'conversation-1': 'running' });
const controller = new AbortController();
controller.abort();
await expect(
waitForGenerationSettled(source, 'conversation-1', { signal: controller.signal }),
).resolves.toBe(false);
expect(source.getGenerationSettlementState).not.toHaveBeenCalled();
expect(source.listeners.size).toBe(0);
expect(jest.getTimerCount()).toBe(0);
});
it('serializes reads and remembers a settlement event during an in-flight read', async () => {
const source = createSource({ 'conversation-1': 'running' });
let release!: () => void;
const gate = new Promise<void>((resolve) => {
release = resolve;
});
source.getGenerationSettlementState.mockImplementationOnce(async () => {
await gate;
return { createdAt: 1, status: 'running', terminalPersistencePending: false };
});
const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 });
await jest.advanceTimersByTimeAsync(10_000);
source.settle({ conversationId: 'conversation-1', status: 'complete' });
source.settle({ conversationId: 'conversation-1', status: 'complete' });
expect(source.getGenerationSettlementState).toHaveBeenCalledTimes(1);
release();
await expect(waiting).resolves.toBe(true);
expect(source.getGenerationSettlementState).toHaveBeenCalledTimes(2);
expect(jest.getTimerCount()).toBe(0);
});
it('removes abort listeners and ignores reads completing after cancellation', async () => {
const source = createSource({ 'conversation-1': 'running' });
const controller = new AbortController();
const remove = jest.spyOn(controller.signal, 'removeEventListener');
let release!: () => void;
const gate = new Promise<void>((resolve) => {
release = resolve;
});
source.getGenerationSettlementState.mockImplementationOnce(async () => {
await gate;
return undefined;
});
const waiting = waitForGenerationSettled(source, 'conversation-1', {
signal: controller.signal,
maxWaitMs: 500,
});
controller.abort();
await expect(waiting).resolves.toBe(false);
release();
await jest.advanceTimersByTimeAsync(1_000);
expect(remove).toHaveBeenCalledWith('abort', expect.any(Function));
expect(source.listeners.size).toBe(0);
expect(jest.getTimerCount()).toBe(0);
});
});

View file

@ -85,3 +85,7 @@ export { RedisJobStore } from './implementations/RedisJobStore';
export { RedisEventTransport } from './implementations/RedisEventTransport';
export { waitForGenerationPersistence } from './persistence';
export { waitForGenerationSettled } from './settled';
export type { GenerationSettledSource, GenerationSettledWaitOptions } from './settled';
export type { GenerationSettlementState } from './interfaces/IJobStore';

View file

@ -103,9 +103,13 @@ export type JobStatus = 'running' | 'complete' | 'error' | 'aborted' | 'requires
* Missing markers on pre-rollout records are interpreted as protocol v1. */
export type GenerationProtocolVersion = 1 | 2;
/**
* Serializable job data - no object references, suitable for Redis/external storage
*/
/** Read-only durable evidence, without attaching a runtime or recovering a slow save. */
export type GenerationSettlementState = Pick<
SerializableJobData,
'createdAt' | 'status' | 'terminalPersistencePending'
>;
/** Serializable job data without object references, suitable for external storage. */
export interface SerializableJobData {
streamId: string;
userId: string;

View file

@ -0,0 +1,114 @@
import { logger } from '@librechat/data-schemas';
import type { GenerationSettledListener } from './GenerationJobManager';
import type { GenerationSettlementState } from './interfaces/IJobStore';
const DEFAULT_RECHECK_MS = 60_000;
export interface GenerationSettledSource {
getGenerationSettlementState: (
streamId: string,
) => Promise<GenerationSettlementState | undefined>;
onGenerationSettled: (listener: GenerationSettledListener) => () => void;
}
export interface GenerationSettledWaitOptions {
recheckMs?: number;
/** Pin the dispatch generation even if it was replaced before the first read. */
generationCreatedAt?: number;
/** Optional caller deadline. Store errors are not evidence of settlement. */
maxWaitMs?: number;
/** Stops waiting (resolving `false`) once the caller no longer needs the answer. */
signal?: AbortSignal;
}
/** Waits for the dispatch generation, including terminal persistence. Events request
* a fresh read; they cannot settle an unrelated epoch or bypass an unfinished save.
* The first successful read pins the epoch if the caller did not supply it. Reads
* are single-flight, with a bounded periodic fallback for remote events and outages. */
export function waitForGenerationSettled(
source: GenerationSettledSource,
conversationId: string,
{
recheckMs = DEFAULT_RECHECK_MS,
generationCreatedAt,
maxWaitMs,
signal,
}: GenerationSettledWaitOptions = {},
): Promise<boolean> {
if (signal?.aborted) {
return Promise.resolve(false);
}
return new Promise((resolve) => {
let done = false;
let reading = false;
let readRequested = false;
let epoch = generationCreatedAt;
let observedActive = epoch != null;
let unsubscribe: (() => void) | undefined = undefined;
let recheck: NodeJS.Timeout | undefined;
let deadline: NodeJS.Timeout | undefined;
const abort = (): void => finish(false);
const finish = (settled: boolean): void => {
if (done) return;
done = true;
unsubscribe?.();
signal?.removeEventListener('abort', abort);
clearTimeout(recheck);
clearTimeout(deadline);
resolve(settled);
};
const readState = async (): Promise<void> => {
if (done) return;
if (reading) {
readRequested = true;
return;
}
reading = true;
clearTimeout(recheck);
try {
const job = await source.getGenerationSettlementState(conversationId);
if (done) return;
if (job == null || (epoch != null && epoch !== job.createdAt)) {
finish(observedActive);
return;
}
epoch = job.createdAt;
const active =
job.status === 'running' ||
job.status === 'requires_action' ||
job.terminalPersistencePending === true;
if (!active) {
finish(observedActive);
return;
}
observedActive = true;
} catch (error) {
logger.warn(
`[GenerationSettled] Failed to read generation state for ${conversationId}:`,
error,
);
} finally {
reading = false;
if (!done) {
if (readRequested) {
readRequested = false;
void readState();
} else {
recheck = setTimeout(() => void readState(), recheckMs);
recheck.unref?.();
}
}
}
};
if (maxWaitMs != null) {
deadline = setTimeout(() => finish(false), maxWaitMs);
deadline.unref?.();
}
signal?.addEventListener('abort', abort, { once: true });
unsubscribe = source.onGenerationSettled((event) => {
if (event.conversationId === conversationId) void readState();
});
if (done) unsubscribe();
else void readState();
});
}