mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-20 15:02:02 +00:00
🧷 fix: Wait for Long Dispatch Turns Before Giving Up Background Code Results (#16345)
* 🐢 fix: Back Off Waiting Completion Wake-ups and Deliver Them When Ready * fix: Signal every readiness path and make the completion wait cap configurable * 🧷 fix: Wait for Long Dispatch Turns Before Giving Up Background Code Results * fix: Mark held completion deliveries instead of pipeline expedite, and announce store-won approval expiry * fix: Keep code tasks protected and wait without a fixed cutoff * fix: Scope settle expedites to the resumed conversation and close the remaining signal gaps * style: Format the settlement waiter * fix: Announce a won approval expiry once and release subagent wake-up registrations * fix: Outlast terminal persistence before giving up an unanchored code result * fix: Anchor code results the moment their dispatch turn settles * fix: Scope harvest epoch pinning to the owning stream --------- Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
parent
dd21e7fb6d
commit
c6282fa888
15 changed files with 1065 additions and 44 deletions
|
|
@ -20,6 +20,7 @@ const {
|
|||
sendEvent,
|
||||
computeUsageCostUSD,
|
||||
GenerationJobManager,
|
||||
waitForGenerationSettled,
|
||||
writeAttachmentEvent,
|
||||
createToolExecuteHandler,
|
||||
createOwnedToolEndHandler,
|
||||
|
|
@ -1286,14 +1287,20 @@ function createPtcProgressEmitter({ res, streamId = null, jobCreatedAt }) {
|
|||
* output?: string;
|
||||
* attachments?: Object[];
|
||||
* }) => Promise<boolean>} params.updateToolCallResult
|
||||
* @param {number} [params.jobCreatedAt] - Immutable dispatch generation epoch.
|
||||
* @param {string} [params.streamId] - The stream owning that epoch.
|
||||
*/
|
||||
function createBackgroundCodeResultHandler({ req, updateToolCallResult }) {
|
||||
function createBackgroundCodeResultHandler({ req, updateToolCallResult, jobCreatedAt, streamId }) {
|
||||
return createCodeHarvestHandler({
|
||||
req,
|
||||
updateToolCallResult,
|
||||
preflightCodeOutputBatch,
|
||||
processCodeOutput,
|
||||
runPreviewFinalize,
|
||||
generationCreatedAt: jobCreatedAt,
|
||||
generationStreamId: streamId,
|
||||
waitForGenerationSettled: (conversationId, options) =>
|
||||
waitForGenerationSettled(GenerationJobManager, conversationId, options),
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -511,6 +511,8 @@ const initializeClientWithProvider = async ({
|
|||
},
|
||||
persistBackgroundCodeResult: createBackgroundCodeResultHandler({
|
||||
req,
|
||||
streamId,
|
||||
jobCreatedAt,
|
||||
updateToolCallResult: db.updateToolCallResult,
|
||||
}),
|
||||
backgroundToolCompletion: {
|
||||
|
|
|
|||
|
|
@ -304,6 +304,9 @@ describe('initializeClient — processAgent ACL gate', () => {
|
|||
jobCreatedAt: 1234,
|
||||
}),
|
||||
);
|
||||
expect(
|
||||
require('~/server/controllers/agents/callbacks').createBackgroundCodeResultHandler,
|
||||
).toHaveBeenCalledWith(expect.objectContaining({ jobCreatedAt: 1234 }));
|
||||
expect(createToolEndCallback).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
streamId: 'conv_1',
|
||||
|
|
|
|||
|
|
@ -1369,6 +1369,54 @@ describe('BackgroundTaskRegistryClass', () => {
|
|||
expect(registry.get('u1', 'c1', created.task.id)).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each(['completed', 'error', 'cancelled', 'blocked'] as const)(
|
||||
'preserves pending %s results through both TTLs and releases protection afterward',
|
||||
async (status) => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const registry = new BackgroundTaskRegistryClass();
|
||||
const created = registry.create({
|
||||
userId: 'u',
|
||||
conversationId: 'c',
|
||||
toolCallId: 'call',
|
||||
toolName: 'execute_code',
|
||||
harvestStarted: true,
|
||||
});
|
||||
if ('atCapacity' in created) throw new Error('Unexpected capacity rejection');
|
||||
if (status === 'cancelled') registry.cancel('u', 'c', created.task.id, 'cancelled');
|
||||
else if (status === 'error') registry.fail('u', 'c', created.task.id, 'failed');
|
||||
else
|
||||
registry.complete('u', 'c', created.task.id, { content: 'result', harvestStarted: true });
|
||||
registry.markCompletionPersistencePending('u', 'c', created.task.id);
|
||||
registry.finishHarvest('u', 'c', created.task.id, [{ file_id: 'file' }]);
|
||||
if (status === 'blocked') registry.blockArtifact('u', 'c', created.task.id, 'blocked');
|
||||
registry.claimResult('u', 'c', created.task.id, { kind: 'manual', claimId: 'poll' });
|
||||
await jest.advanceTimersByTimeAsync(7 * 60 * 60_000);
|
||||
expect(registry.get('u', 'c', created.task.id)).toMatchObject({
|
||||
completionPersistencePending: true,
|
||||
});
|
||||
registry.markCompletionPersistenceFinished('u', 'c', created.task.id);
|
||||
expect(
|
||||
registry.get('u', 'c', created.task.id)?.completionPersistencePending,
|
||||
).toBeUndefined();
|
||||
if (status === 'blocked')
|
||||
expect(registry.get('u', 'c', created.task.id)).toMatchObject({
|
||||
artifactBlocked: true,
|
||||
artifact: undefined,
|
||||
});
|
||||
if (status === 'completed')
|
||||
expect(registry.get('u', 'c', created.task.id)).toMatchObject({
|
||||
result: 'result',
|
||||
resultClaim: { kind: 'manual', claimId: 'poll' },
|
||||
});
|
||||
await jest.advanceTimersByTimeAsync(61 * 60_000);
|
||||
expect(registry.get('u', 'c', created.task.id)).toBeUndefined();
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it('caps concurrent running tasks per conversation', () => {
|
||||
const registry = new BackgroundTaskRegistryClass();
|
||||
let atCapacity = false;
|
||||
|
|
|
|||
|
|
@ -834,7 +834,11 @@ export class BackgroundTaskRegistryClass {
|
|||
|
||||
private sweepBucketTasks(bucket: TaskBucket, now: number): void {
|
||||
for (const [taskId, task] of bucket.tasks) {
|
||||
if (task.status !== 'running' && now - task.updatedAt > COMPLETED_TASK_TTL_MS) {
|
||||
if (
|
||||
task.status !== 'running' &&
|
||||
task.completionPersistencePending !== true &&
|
||||
now - task.updatedAt > COMPLETED_TASK_TTL_MS
|
||||
) {
|
||||
bucket.tasks.delete(taskId);
|
||||
}
|
||||
}
|
||||
|
|
@ -859,7 +863,13 @@ export class BackgroundTaskRegistryClass {
|
|||
}
|
||||
this.lastGlobalSweepAt = now;
|
||||
for (const [bucketKey, bucket] of this.buckets) {
|
||||
if (now - bucket.lastAccess > IDLE_BUCKET_TTL_MS && bucket.capacityPermits.size === 0) {
|
||||
if (
|
||||
now - bucket.lastAccess > IDLE_BUCKET_TTL_MS &&
|
||||
bucket.capacityPermits.size === 0 &&
|
||||
![...bucket.tasks.values()].some(
|
||||
(task) => task.status === 'running' || task.completionPersistencePending === true,
|
||||
)
|
||||
) {
|
||||
this.buckets.delete(bucketKey);
|
||||
continue;
|
||||
}
|
||||
|
|
@ -1615,7 +1625,14 @@ export class BackgroundTaskRegistryClass {
|
|||
}
|
||||
|
||||
markCompletionPersistenceFinished(userId: string, conversationId: string, taskId: string): void {
|
||||
this.update(userId, conversationId, taskId, { completionPersistencePending: undefined });
|
||||
const bucket = this.buckets.get(this.key(userId, conversationId));
|
||||
const task = bucket?.tasks.get(taskId);
|
||||
if (bucket == null || task == null) return;
|
||||
/** Even a policy-blocked task must release retention protection. This only
|
||||
* clears lifecycle state; the immutable artifact block remains intact. */
|
||||
task.completionPersistencePending = undefined;
|
||||
task.updatedAt = Date.now();
|
||||
bucket.lastAccess = task.updatedAt;
|
||||
}
|
||||
|
||||
markCompletionPersistenceFailed(userId: string, conversationId: string, taskId: string): void {
|
||||
|
|
|
|||
|
|
@ -2621,6 +2621,59 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
|
|||
expect(poll[0].artifact).toBeUndefined();
|
||||
});
|
||||
|
||||
it('releases stored files to same-turn polls while the row patch waits for the dispatch turn', async () => {
|
||||
const state: CodeToolState = { calls: 0 };
|
||||
const emitted: unknown[] = [];
|
||||
const handler = createToolExecuteHandler({
|
||||
loadTools: async () => ({ loadedTools: [makeCodeTool(state)] }),
|
||||
/** Files are stored; the row patch then waits for the long dispatch turn. */
|
||||
persistBackgroundCodeResult: (params) => {
|
||||
params.onFilesPersisted?.([{ file_id: 'f1', toolCallId: params.toolCallId }]);
|
||||
return new Promise(() => undefined);
|
||||
},
|
||||
emitAttachment: (attachment) => {
|
||||
emitted.push(attachment);
|
||||
},
|
||||
});
|
||||
const configurable = buildConfig(['execute_code']);
|
||||
|
||||
const dispatch = await runBatch(handler, {
|
||||
toolCalls: [codeCall({ id: 'call_code_long_turn' })],
|
||||
agentId: 'a',
|
||||
configurable,
|
||||
metadata: { thread_id: 'exec_convo_code_long_turn', run_id: 'msg-long-turn' },
|
||||
});
|
||||
await flushMicrotasks();
|
||||
await flushMicrotasks();
|
||||
|
||||
const poll = (await runBatch(handler, {
|
||||
toolCalls: [
|
||||
{
|
||||
id: 'call_poll_long_turn',
|
||||
name: CHECK_BACKGROUND_TASK_NAME,
|
||||
args: { background_task_id: JSON.parse(dispatch[0].content).background_task_id },
|
||||
},
|
||||
],
|
||||
agentId: 'a',
|
||||
configurable,
|
||||
metadata: { thread_id: 'exec_convo_code_long_turn', run_id: 'msg-long-turn' },
|
||||
})) as Array<{ content: string; artifact?: unknown }>;
|
||||
|
||||
const polled = JSON.parse(poll[0].content);
|
||||
expect(polled.status).toBe('completed');
|
||||
expect(polled.note).toContain('attached to the tool call');
|
||||
expect(emitted[0]).toEqual({ file_id: 'f1', toolCallId: 'call_code_long_turn' });
|
||||
expect(poll[0].artifact).toEqual(CODE_ARTIFACT);
|
||||
/** Still protected from retention eviction while the row patch waits. */
|
||||
expect(
|
||||
backgroundTaskRegistry.get(
|
||||
'exec_user',
|
||||
'exec_convo_code_long_turn',
|
||||
JSON.parse(dispatch[0].content).background_task_id,
|
||||
)?.completionPersistencePending,
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('falls back to poll-turn delivery when the harvest fails (files not lost)', async () => {
|
||||
const state: CodeToolState = { calls: 0 };
|
||||
const toolEndCalls: Array<{ name?: string; artifact?: unknown }> = [];
|
||||
|
|
|
|||
|
|
@ -315,6 +315,7 @@ export interface ToolExecuteOptions {
|
|||
reapply?: boolean;
|
||||
backgroundTask?: BackgroundToolResultState;
|
||||
resolveBackgroundTask?: () => BackgroundToolResultState;
|
||||
onFilesPersisted?: (attachments: unknown[]) => void;
|
||||
}) => Promise<{ attachments?: unknown[]; deliveryReady?: boolean } | null>;
|
||||
/** Shared ordinary-tool completion lifecycle. The delivery is registered
|
||||
* before invoke; settlement is persisted onto the original response row. */
|
||||
|
|
@ -6169,6 +6170,13 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
|
|||
: { backgroundTask, resolveBackgroundTask }),
|
||||
output: params.output ?? localTask?.result,
|
||||
artifact: params.artifact,
|
||||
onFilesPersisted: (attachments) =>
|
||||
backgroundTaskRegistry.finishHarvest(
|
||||
backgroundUserId,
|
||||
backgroundConversationId,
|
||||
task.id,
|
||||
attachments,
|
||||
),
|
||||
});
|
||||
if (persisted == null) {
|
||||
/** Harvest never persisted anything (missing anchor
|
||||
|
|
@ -6257,10 +6265,8 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
|
|||
artifact?: unknown;
|
||||
status: 'completed' | 'error' | 'cancelled';
|
||||
}): Promise<void> => {
|
||||
if (harvestEnabled) {
|
||||
await persistBackgroundResult(params);
|
||||
return;
|
||||
}
|
||||
/** Held for the whole persist, including a code harvest that waits for
|
||||
* a long dispatch turn, so retention pressure cannot evict the task. */
|
||||
backgroundTaskRegistry.markCompletionPersistencePending(
|
||||
backgroundUserId,
|
||||
backgroundConversationId,
|
||||
|
|
|
|||
156
packages/api/src/agents/harvest.lifecycle.spec.ts
Normal file
156
packages/api/src/agents/harvest.lifecycle.spec.ts
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
import type { GenerationSettlementState } from '../stream/interfaces/IJobStore';
|
||||
import type { GenerationSettledListener } from '../stream/GenerationJobManager';
|
||||
import type { GenerationSettledSource } from '../stream/settled';
|
||||
import type { GenerationJobStatus } from '../types/stream';
|
||||
import type { ServerRequest } from '~/types';
|
||||
import { InMemoryEventTransport } from '../stream/implementations/InMemoryEventTransport';
|
||||
import { InMemoryJobStore } from '../stream/implementations/InMemoryJobStore';
|
||||
import { GenerationJobManagerClass } from '../stream/GenerationJobManager';
|
||||
import { createBackgroundCodeResultHandler } from './harvest';
|
||||
import { waitForGenerationSettled } from '../stream/settled';
|
||||
|
||||
function sourceForRunningTurn() {
|
||||
const listeners = new Set<GenerationSettledListener>();
|
||||
const job: {
|
||||
status: GenerationJobStatus;
|
||||
createdAt: number;
|
||||
terminalPersistencePending?: boolean;
|
||||
} = { status: 'running', createdAt: 1 };
|
||||
return {
|
||||
job,
|
||||
listeners,
|
||||
getGenerationSettlementState: jest.fn(
|
||||
async (): Promise<GenerationSettlementState | undefined> => ({ ...job }),
|
||||
),
|
||||
onGenerationSettled: (listener: GenerationSettledListener) => {
|
||||
listeners.add(listener);
|
||||
return () => {
|
||||
listeners.delete(listener);
|
||||
};
|
||||
},
|
||||
settle: () => {
|
||||
job.status = 'complete';
|
||||
for (const listener of [...listeners])
|
||||
listener({
|
||||
streamId: 'conversation',
|
||||
conversationId: 'conversation',
|
||||
userId: 'user',
|
||||
status: 'complete',
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function harvest(source: GenerationSettledSource, row: { unfinished: boolean }) {
|
||||
const update = jest.fn(async () => ({ matched: true, unfinished: row.unfinished }));
|
||||
const handler = createBackgroundCodeResultHandler({
|
||||
req: { user: { id: 'user' } } as ServerRequest,
|
||||
preflightCodeOutputBatch: async () => [],
|
||||
processCodeOutput: jest.fn(),
|
||||
updateToolCallResult: update,
|
||||
runPreviewFinalize: jest.fn(),
|
||||
waitForGenerationSettled: (conversationId, options) =>
|
||||
waitForGenerationSettled(source, conversationId, options),
|
||||
});
|
||||
let result: boolean | undefined;
|
||||
const pending = handler({
|
||||
toolName: 'execute_code',
|
||||
toolCallId: 'call',
|
||||
stepId: 'step',
|
||||
messageId: 'message',
|
||||
conversationId: 'conversation',
|
||||
output: 'finished code',
|
||||
backgroundTask: {
|
||||
taskId: 'task',
|
||||
toolName: 'execute_code',
|
||||
status: 'completed',
|
||||
settledAt: new Date(),
|
||||
completionWakeup: true,
|
||||
},
|
||||
}).then((value) => {
|
||||
result = value?.deliveryReady;
|
||||
});
|
||||
return { update, pending, getResult: () => result };
|
||||
}
|
||||
|
||||
describe('background code harvest generation lifecycle', () => {
|
||||
beforeEach(() => {
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
afterEach(() => {
|
||||
jest.clearAllTimers();
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
it('keeps harvesting a live turn after a transient first status-read failure', async () => {
|
||||
const source = sourceForRunningTurn();
|
||||
source.getGenerationSettlementState.mockRejectedValueOnce(new Error('transient Redis outage'));
|
||||
const row = { unfinished: true };
|
||||
const running = harvest(source, row);
|
||||
await jest.advanceTimersByTimeAsync(1_100_000);
|
||||
const beforeSettle = running.getResult();
|
||||
expect(source.getGenerationSettlementState.mock.calls.length).toBeGreaterThan(1);
|
||||
row.unfinished = false;
|
||||
source.settle();
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
expect(beforeSettle).toBeUndefined();
|
||||
await running.pending;
|
||||
expect(running.getResult()).toBe(true);
|
||||
});
|
||||
|
||||
it('anchors the original result when a remote successor replaces the observed generation', async () => {
|
||||
const source = sourceForRunningTurn();
|
||||
const row = { unfinished: true };
|
||||
const running = harvest(source, row);
|
||||
await jest.advanceTimersByTimeAsync(1_000_000);
|
||||
expect(running.update.mock.calls.length).toBe(14);
|
||||
row.unfinished = false;
|
||||
source.job.createdAt = 2;
|
||||
source.job.status = 'requires_action';
|
||||
await jest.advanceTimersByTimeAsync(120_000);
|
||||
const beforeSuccessorSettles = running.getResult();
|
||||
expect(running.update.mock.calls.length).toBe(15);
|
||||
source.settle();
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
await running.pending;
|
||||
expect(beforeSuccessorSettles).toBe(true);
|
||||
});
|
||||
|
||||
it('does not retire a result while the real manager is awaiting terminal message persistence', async () => {
|
||||
const manager = new GenerationJobManagerClass();
|
||||
const store = new InMemoryJobStore({ ttlAfterComplete: 0 });
|
||||
manager.configure({
|
||||
jobStore: store,
|
||||
eventTransport: new InMemoryEventTransport(),
|
||||
isRedis: false,
|
||||
cleanupOnComplete: false,
|
||||
});
|
||||
manager.initialize();
|
||||
let finishSave!: () => void;
|
||||
const save = new Promise<void>((resolve) => {
|
||||
finishSave = resolve;
|
||||
});
|
||||
try {
|
||||
const job = await manager.createJob('conversation', 'user', 'conversation');
|
||||
const row = { unfinished: true };
|
||||
const running = harvest(manager, row);
|
||||
await jest.advanceTimersByTimeAsync(1_000_000);
|
||||
const completing = manager.completeJob('conversation', 'provider error', job.createdAt, {
|
||||
beforeErrorPublication: () => save,
|
||||
});
|
||||
await jest.advanceTimersByTimeAsync(120_000);
|
||||
const beforePersistence = running.getResult();
|
||||
expect((await store.getJob('conversation'))?.terminalPersistencePending).toBe(true);
|
||||
row.unfinished = false;
|
||||
finishSave();
|
||||
await completing;
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
expect(beforePersistence).toBeUndefined();
|
||||
await running.pending;
|
||||
expect(running.getResult()).toBe(true);
|
||||
} finally {
|
||||
finishSave();
|
||||
await manager.destroy();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
@ -153,3 +153,218 @@ describe('createBackgroundToolResultHandler claim ownership', () => {
|
|||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('createBackgroundCodeResultHandler long dispatch turns', () => {
|
||||
const RETRY_SCHEDULE_MS = 1_000_000;
|
||||
const backgroundTask = {
|
||||
taskId: 'task-1',
|
||||
toolName: 'execute_code',
|
||||
status: 'completed' as const,
|
||||
settledAt: new Date('2026-09-24T00:00:00Z'),
|
||||
completionWakeup: true as const,
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
function createHandler(
|
||||
updateToolCallResult: jest.Mock,
|
||||
waitForGenerationSettled?: (conversationId: string) => Promise<boolean>,
|
||||
) {
|
||||
return createBackgroundCodeResultHandler({
|
||||
req,
|
||||
preflightCodeOutputBatch: jest
|
||||
.fn()
|
||||
.mockResolvedValue([{ file: params.artifact.files[0], sessionId: 'storage-session' }]),
|
||||
processCodeOutput: jest.fn().mockResolvedValue({ file: { file_id: 'persisted-file' } }),
|
||||
updateToolCallResult,
|
||||
runPreviewFinalize: jest.fn(),
|
||||
...(waitForGenerationSettled != null && { waitForGenerationSettled }),
|
||||
});
|
||||
}
|
||||
|
||||
it('waits for a dispatch turn that outlives the retry schedule, then anchors', async () => {
|
||||
let turnRunning = true;
|
||||
const updateToolCallResult = jest.fn(async () => ({
|
||||
matched: true,
|
||||
unfinished: turnRunning,
|
||||
}));
|
||||
let settle: (settled: boolean) => void = () => undefined;
|
||||
const waitForGenerationSettled = jest.fn(
|
||||
() =>
|
||||
new Promise<boolean>((resolve) => {
|
||||
settle = resolve;
|
||||
}),
|
||||
);
|
||||
const onFilesPersisted = jest.fn();
|
||||
const handler = createHandler(updateToolCallResult, waitForGenerationSettled);
|
||||
|
||||
const result = handler({ ...params, backgroundTask, onFilesPersisted });
|
||||
await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS);
|
||||
|
||||
expect(onFilesPersisted).toHaveBeenCalledWith([
|
||||
{ file_id: 'persisted-file', stepId: 'step-1' },
|
||||
]);
|
||||
expect(onFilesPersisted.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
updateToolCallResult.mock.invocationCallOrder[0],
|
||||
);
|
||||
expect(waitForGenerationSettled).toHaveBeenCalledWith('conversation-1', {
|
||||
signal: expect.any(AbortSignal),
|
||||
});
|
||||
const attemptsWhileRunning = updateToolCallResult.mock.calls.length;
|
||||
await jest.advanceTimersByTimeAsync(60 * 60 * 1_000);
|
||||
expect(updateToolCallResult).toHaveBeenCalledTimes(attemptsWhileRunning);
|
||||
|
||||
turnRunning = false;
|
||||
settle(true);
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
|
||||
await expect(result).resolves.toEqual({
|
||||
attachments: [{ file_id: 'persisted-file', stepId: 'step-1' }],
|
||||
deliveryReady: true,
|
||||
});
|
||||
expect(updateToolCallResult).toHaveBeenCalledTimes(attemptsWhileRunning + 1);
|
||||
});
|
||||
|
||||
it('waits for a running turn that has not saved its row yet', async () => {
|
||||
let rowSaved = false;
|
||||
const updateToolCallResult = jest.fn(async () => ({ matched: rowSaved, unfinished: false }));
|
||||
const waitForGenerationSettled = jest.fn(async () => {
|
||||
rowSaved = true;
|
||||
return true;
|
||||
});
|
||||
const handler = createHandler(updateToolCallResult, waitForGenerationSettled);
|
||||
|
||||
const result = handler({ ...params, backgroundTask });
|
||||
await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS);
|
||||
|
||||
await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: true }));
|
||||
expect(waitForGenerationSettled).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('gives the result up when no generation is running and the row never appears', async () => {
|
||||
const updateToolCallResult = jest.fn(async () => ({ matched: false, unfinished: false }));
|
||||
const waitForGenerationSettled = jest.fn(async () => false);
|
||||
const handler = createHandler(updateToolCallResult, waitForGenerationSettled);
|
||||
|
||||
const result = handler({ ...params, backgroundTask });
|
||||
await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS + 60_000);
|
||||
|
||||
await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: false }));
|
||||
expect(updateToolCallResult).toHaveBeenCalledTimes(14 + 5);
|
||||
});
|
||||
|
||||
it('anchors a final save that lands well after the generation reads as settled', async () => {
|
||||
let settledAt: number | undefined;
|
||||
const updateToolCallResult = jest.fn(async () => ({
|
||||
matched: true,
|
||||
unfinished: settledAt == null || Date.now() - settledAt < 40_000,
|
||||
}));
|
||||
const waitForGenerationSettled = jest.fn(async () => {
|
||||
settledAt = Date.now();
|
||||
return true;
|
||||
});
|
||||
const handler = createHandler(updateToolCallResult, waitForGenerationSettled);
|
||||
|
||||
const result = handler({ ...params, backgroundTask });
|
||||
await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS + 60_000);
|
||||
|
||||
await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: true }));
|
||||
});
|
||||
|
||||
it('gives the result up when the turn ends without saving the tool call', async () => {
|
||||
const updateToolCallResult = jest.fn(async () => ({ matched: true, unfinished: true }));
|
||||
const waitForGenerationSettled = jest.fn(async () => true);
|
||||
const handler = createHandler(updateToolCallResult, waitForGenerationSettled);
|
||||
|
||||
const result = handler({ ...params, backgroundTask });
|
||||
await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS + 60_000);
|
||||
|
||||
await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: false }));
|
||||
expect(waitForGenerationSettled).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('anchors as soon as the dispatch turn settles instead of at the next scheduled retry', async () => {
|
||||
let turnRunning = true;
|
||||
const updateToolCallResult = jest.fn(async () => ({ matched: true, unfinished: turnRunning }));
|
||||
let settle: (settled: boolean) => void = () => undefined;
|
||||
const waitForGenerationSettled = jest.fn(
|
||||
() =>
|
||||
new Promise<boolean>((resolve) => {
|
||||
settle = resolve;
|
||||
}),
|
||||
);
|
||||
const handler = createHandler(updateToolCallResult, waitForGenerationSettled);
|
||||
let finished = false;
|
||||
|
||||
const result = handler({ ...params, backgroundTask }).then((value) => {
|
||||
finished = true;
|
||||
return value;
|
||||
});
|
||||
/** 45 s in, the schedule is inside a 30 s gap (next step near 69 s). */
|
||||
await jest.advanceTimersByTimeAsync(45_000);
|
||||
const attemptsBeforeSettle = updateToolCallResult.mock.calls.length;
|
||||
turnRunning = false;
|
||||
settle(true);
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
|
||||
expect(finished).toBe(true);
|
||||
await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: true }));
|
||||
expect(updateToolCallResult).toHaveBeenCalledTimes(attemptsBeforeSettle + 1);
|
||||
});
|
||||
|
||||
it.each([params.conversationId, 'another-conversation'])(
|
||||
'pins only the epoch owned by the host stream %s',
|
||||
async (generationStreamId) => {
|
||||
const wait = jest.fn(async () => true);
|
||||
const handler = createBackgroundCodeResultHandler({
|
||||
req,
|
||||
preflightCodeOutputBatch: async () => [],
|
||||
processCodeOutput: jest.fn(),
|
||||
runPreviewFinalize: jest.fn(),
|
||||
updateToolCallResult: async () => ({ matched: true, unfinished: false }),
|
||||
generationCreatedAt: 1234,
|
||||
generationStreamId,
|
||||
waitForGenerationSettled: wait,
|
||||
});
|
||||
await handler({ ...params, backgroundTask });
|
||||
expect(wait).toHaveBeenCalledWith(params.conversationId, {
|
||||
...(generationStreamId === params.conversationId && { generationCreatedAt: 1234 }),
|
||||
signal: expect.any(AbortSignal),
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it('stops listening for the turn once the result is anchored', async () => {
|
||||
const updateToolCallResult = jest.fn(async () => ({ matched: true, unfinished: false }));
|
||||
let signal: AbortSignal | undefined;
|
||||
const waitForGenerationSettled = jest.fn(
|
||||
(_conversationId: string, options?: { signal?: AbortSignal }) => {
|
||||
signal = options?.signal;
|
||||
return new Promise<boolean>(() => undefined);
|
||||
},
|
||||
);
|
||||
const handler = createHandler(updateToolCallResult, waitForGenerationSettled);
|
||||
|
||||
await expect(handler({ ...params, backgroundTask })).resolves.toEqual(
|
||||
expect.objectContaining({ deliveryReady: true }),
|
||||
);
|
||||
expect(signal?.aborted).toBe(true);
|
||||
});
|
||||
|
||||
it('keeps the bounded schedule when no settlement signal is wired', async () => {
|
||||
const updateToolCallResult = jest.fn(async () => ({ matched: true, unfinished: true }));
|
||||
const handler = createHandler(updateToolCallResult);
|
||||
|
||||
const result = handler({ ...params, backgroundTask });
|
||||
await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS);
|
||||
|
||||
await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: false }));
|
||||
expect(updateToolCallResult).toHaveBeenCalledTimes(14);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { logger } from '@librechat/data-schemas';
|
||||
import type { PreparedCodeOutputEntry } from '~/files/code/preflight';
|
||||
import type { GenerationSettledWaitOptions } from '../stream/settled';
|
||||
import type { CodeExecutionContext } from './execution';
|
||||
import type { ServerRequest } from '~/types';
|
||||
|
||||
|
|
@ -12,6 +13,9 @@ import type { ServerRequest } from '~/types';
|
|||
const BACKGROUND_PATCH_RETRY_DELAYS_MS = [
|
||||
250, 500, 1_000, 2_000, 5_000, 10_000, 20_000, 30_000, 60_000, 120_000, 180_000, 240_000, 300_000,
|
||||
];
|
||||
/** Bounded final attempts after the dispatch epoch has finished or disappeared.
|
||||
* Live terminal persistence is waited for explicitly, not inferred from this delay. */
|
||||
const SETTLED_PATCH_RETRY_DELAYS_MS = [1_000, 5_000, 15_000, 30_000];
|
||||
interface HarvestFileRef {
|
||||
id: string;
|
||||
name: string;
|
||||
|
|
@ -51,6 +55,10 @@ export interface BackgroundToolResultState {
|
|||
}
|
||||
|
||||
export interface CodeHarvestDeps {
|
||||
/** Generation identity captured by the host at dispatch, not at harvest time. */
|
||||
generationCreatedAt?: number;
|
||||
/** The stream owning that epoch; nested child conversations must observe their own. */
|
||||
generationStreamId?: string | null;
|
||||
req: ServerRequest;
|
||||
/** Data-schemas method: idempotent tool-call part patch + attachment append. */
|
||||
updateToolCallResult: (params: {
|
||||
|
|
@ -95,6 +103,12 @@ export interface CodeHarvestDeps {
|
|||
fileId: string;
|
||||
previewRevision?: number;
|
||||
}) => void;
|
||||
/** Waits for the dispatch epoch, including terminal persistence. Read outages
|
||||
* remain retryable; replacement ends the wait without following a successor. */
|
||||
waitForGenerationSettled?: (
|
||||
conversationId: string,
|
||||
options?: GenerationSettledWaitOptions,
|
||||
) => Promise<boolean>;
|
||||
}
|
||||
|
||||
export interface CodeHarvestParams {
|
||||
|
|
@ -119,31 +133,48 @@ export interface CodeHarvestParams {
|
|||
/** Re-reads local claim ownership on every retry so a same-generation
|
||||
* manual poll cannot be overwritten by a later automatic continuation. */
|
||||
resolveBackgroundTask?: () => BackgroundToolResultState;
|
||||
/** Called once generated files are stored, before the row patch, which can
|
||||
* wait for the dispatch turn to end: a poll in that turn needs them now. */
|
||||
onFilesPersisted?: (attachments: unknown[]) => void;
|
||||
}
|
||||
|
||||
export type CodeHarvestHandler = (
|
||||
params: CodeHarvestParams,
|
||||
) => Promise<{ attachments: unknown[]; deliveryReady?: boolean } | null>;
|
||||
|
||||
const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
type BackgroundResultRowParams = {
|
||||
userId: string;
|
||||
messageId: string;
|
||||
conversationId: string;
|
||||
toolCallId: string;
|
||||
stepId?: string;
|
||||
agentId?: string;
|
||||
output?: string;
|
||||
attachments?: unknown[];
|
||||
backgroundTask?: BackgroundToolResultState;
|
||||
resolveBackgroundTask?: () => BackgroundToolResultState;
|
||||
};
|
||||
|
||||
async function persistBackgroundToolResultRow(
|
||||
/** Resolves `true` once `interrupt` fires, or `false` after `ms`; never both. */
|
||||
function sleepUnlessInterrupted(ms: number, interrupt?: Promise<void>): Promise<boolean> {
|
||||
return new Promise((resolve) => {
|
||||
const timer = setTimeout(() => resolve(false), ms);
|
||||
void interrupt?.then(() => {
|
||||
clearTimeout(timer);
|
||||
resolve(true);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/** `false` once `interrupt` cut the schedule short, so the caller can retry at once. */
|
||||
async function anchorBackgroundToolResultRow(
|
||||
updateToolCallResult: CodeHarvestDeps['updateToolCallResult'],
|
||||
params: {
|
||||
userId: string;
|
||||
messageId: string;
|
||||
conversationId: string;
|
||||
toolCallId: string;
|
||||
stepId?: string;
|
||||
agentId?: string;
|
||||
output?: string;
|
||||
attachments?: unknown[];
|
||||
backgroundTask?: BackgroundToolResultState;
|
||||
resolveBackgroundTask?: () => BackgroundToolResultState;
|
||||
},
|
||||
params: BackgroundResultRowParams,
|
||||
retryDelaysMs: readonly number[],
|
||||
interrupt?: Promise<void>,
|
||||
): Promise<boolean> {
|
||||
const { resolveBackgroundTask, ...persistedParams } = params;
|
||||
for (let attempt = 0; attempt <= BACKGROUND_PATCH_RETRY_DELAYS_MS.length; attempt++) {
|
||||
for (let attempt = 0; attempt <= retryDelaysMs.length; attempt++) {
|
||||
const currentBackgroundTask = resolveBackgroundTask?.() ?? persistedParams.backgroundTask;
|
||||
const result = await updateToolCallResult({
|
||||
...persistedParams,
|
||||
|
|
@ -153,14 +184,74 @@ async function persistBackgroundToolResultRow(
|
|||
if (result.matched && !result.unfinished) {
|
||||
return true;
|
||||
}
|
||||
if (attempt === BACKGROUND_PATCH_RETRY_DELAYS_MS.length) {
|
||||
if (attempt === retryDelaysMs.length) {
|
||||
break;
|
||||
}
|
||||
await sleep(BACKGROUND_PATCH_RETRY_DELAYS_MS[attempt]);
|
||||
if (await sleepUnlessInterrupted(retryDelaysMs[attempt], interrupt)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Patches the result onto the dispatch turn's row, which is absent or unfinished
|
||||
* while that turn streams. The patch is retried on a schedule, but the turn
|
||||
* settling is what actually makes the row patchable, so a settle observed
|
||||
* mid-schedule retries at once instead of at the next scheduled step. A turn that
|
||||
* outlives the schedule is waited for rather than given up. Only a positive
|
||||
* settle cuts the schedule short: an unknown generation state keeps the schedule.
|
||||
*/
|
||||
async function persistBackgroundToolResultRow(
|
||||
updateToolCallResult: CodeHarvestDeps['updateToolCallResult'],
|
||||
params: BackgroundResultRowParams,
|
||||
waitForGenerationSettled?: CodeHarvestDeps['waitForGenerationSettled'],
|
||||
generationCreatedAt?: number,
|
||||
): Promise<boolean> {
|
||||
if (waitForGenerationSettled == null) {
|
||||
return anchorBackgroundToolResultRow(
|
||||
updateToolCallResult,
|
||||
params,
|
||||
BACKGROUND_PATCH_RETRY_DELAYS_MS,
|
||||
);
|
||||
}
|
||||
const listening = new AbortController();
|
||||
const settlement = waitForGenerationSettled(params.conversationId, {
|
||||
signal: listening.signal,
|
||||
...(generationCreatedAt == null ? {} : { generationCreatedAt }),
|
||||
}).catch((error: unknown) => {
|
||||
logger.warn(
|
||||
`[background] Failed waiting for the dispatch turn of message ${params.messageId} to settle:`,
|
||||
error,
|
||||
);
|
||||
return false;
|
||||
});
|
||||
const settledTurn = new Promise<void>((resolve) => {
|
||||
void settlement.then((settled) => {
|
||||
if (settled) resolve();
|
||||
});
|
||||
});
|
||||
try {
|
||||
const anchored = await anchorBackgroundToolResultRow(
|
||||
updateToolCallResult,
|
||||
params,
|
||||
BACKGROUND_PATCH_RETRY_DELAYS_MS,
|
||||
settledTurn,
|
||||
);
|
||||
if (anchored) {
|
||||
return true;
|
||||
}
|
||||
await settlement;
|
||||
return anchorBackgroundToolResultRow(
|
||||
updateToolCallResult,
|
||||
params,
|
||||
SETTLED_PATCH_RETRY_DELAYS_MS,
|
||||
);
|
||||
} finally {
|
||||
listening.abort();
|
||||
}
|
||||
}
|
||||
|
||||
/** Persists an ordinary detached tool result without invoking code-artifact processing. */
|
||||
export function createBackgroundToolResultHandler(
|
||||
deps: Pick<CodeHarvestDeps, 'req' | 'updateToolCallResult'>,
|
||||
|
|
@ -218,6 +309,9 @@ export function createBackgroundCodeResultHandler(deps: CodeHarvestDeps): CodeHa
|
|||
preflightCodeOutputBatch,
|
||||
processCodeOutput,
|
||||
runPreviewFinalize,
|
||||
waitForGenerationSettled,
|
||||
generationCreatedAt,
|
||||
generationStreamId,
|
||||
} = deps;
|
||||
return async ({
|
||||
toolCallId,
|
||||
|
|
@ -233,6 +327,7 @@ export function createBackgroundCodeResultHandler(deps: CodeHarvestDeps): CodeHa
|
|||
reapply,
|
||||
backgroundTask,
|
||||
resolveBackgroundTask,
|
||||
onFilesPersisted,
|
||||
}) => {
|
||||
const userId = req.user?.id;
|
||||
if (!userId || !messageId || !conversationId) {
|
||||
|
|
@ -311,22 +406,30 @@ export function createBackgroundCodeResultHandler(deps: CodeHarvestDeps): CodeHa
|
|||
}
|
||||
}
|
||||
|
||||
const deliveryReady = await persistBackgroundToolResultRow(updateToolCallResult, {
|
||||
userId,
|
||||
messageId,
|
||||
conversationId,
|
||||
toolCallId,
|
||||
stepId,
|
||||
agentId,
|
||||
output,
|
||||
attachments,
|
||||
...(backgroundTask != null ? { backgroundTask } : {}),
|
||||
...(resolveBackgroundTask != null ? { resolveBackgroundTask } : {}),
|
||||
});
|
||||
onFilesPersisted?.(attachments);
|
||||
const deliveryReady = await persistBackgroundToolResultRow(
|
||||
updateToolCallResult,
|
||||
{
|
||||
userId,
|
||||
messageId,
|
||||
conversationId,
|
||||
toolCallId,
|
||||
stepId,
|
||||
agentId,
|
||||
output,
|
||||
attachments,
|
||||
...(backgroundTask != null ? { backgroundTask } : {}),
|
||||
...(resolveBackgroundTask != null ? { resolveBackgroundTask } : {}),
|
||||
},
|
||||
waitForGenerationSettled,
|
||||
generationStreamId == null || generationStreamId === conversationId
|
||||
? generationCreatedAt
|
||||
: undefined,
|
||||
);
|
||||
if (!deliveryReady) {
|
||||
logger.warn(
|
||||
`[background] Could not anchor code result onto message ${messageId} (tool call ${toolCallId}); ` +
|
||||
'the dispatch turn never persisted. Poll delivery still returns the result.',
|
||||
'the dispatch turn ended without saving that tool call. Poll delivery still returns the result.',
|
||||
);
|
||||
}
|
||||
return { attachments, ...(backgroundTask != null ? { deliveryReady } : {}) };
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import type {
|
|||
import type { StandardGraph } from '@librechat/agents';
|
||||
import type {
|
||||
SerializableJobData,
|
||||
GenerationSettlementState,
|
||||
CreatedJobData,
|
||||
IEventTransport,
|
||||
UsageMetadata,
|
||||
|
|
@ -3772,9 +3773,21 @@ class GenerationJobManagerClass {
|
|||
await this.jobStore.releaseIdempotencyKey(legacyKey, expectedClaim);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get job status.
|
||||
*/
|
||||
/** Observes identity and final-save ownership without attaching a runtime or
|
||||
* promoting a slow terminal writer to stale-owner recovery. */
|
||||
async getGenerationSettlementState(
|
||||
streamId: string,
|
||||
): Promise<GenerationSettlementState | undefined> {
|
||||
const job = await this.jobStore.getJob(streamId);
|
||||
if (job == null) return undefined;
|
||||
return {
|
||||
createdAt: job.createdAt,
|
||||
status: job.status,
|
||||
terminalPersistencePending: job.terminalPersistencePending,
|
||||
};
|
||||
}
|
||||
|
||||
/** Get job status. */
|
||||
async getJobStatus(streamId: string): Promise<t.GenerationJobStatus | undefined> {
|
||||
const jobData = await this.jobStore.getJob(streamId);
|
||||
return jobData?.status as t.GenerationJobStatus | undefined;
|
||||
|
|
|
|||
276
packages/api/src/stream/__tests__/settled.spec.ts
Normal file
276
packages/api/src/stream/__tests__/settled.spec.ts
Normal file
|
|
@ -0,0 +1,276 @@
|
|||
import type { GenerationSettledEvent, GenerationSettledListener } from '../GenerationJobManager';
|
||||
import type { GenerationJobStatus } from '../../types/stream';
|
||||
import { waitForGenerationSettled } from '../settled';
|
||||
|
||||
function createSource(initial: Record<string, GenerationJobStatus>) {
|
||||
const statuses = new Map(Object.entries(initial));
|
||||
const listeners = new Set<GenerationSettledListener>();
|
||||
const epochs = new Map<string, number>();
|
||||
const pending = new Set<string>();
|
||||
return {
|
||||
statuses,
|
||||
epochs,
|
||||
pending,
|
||||
listeners,
|
||||
getGenerationSettlementState: jest.fn(async (streamId: string) => {
|
||||
const status = statuses.get(streamId);
|
||||
return status == null
|
||||
? undefined
|
||||
: {
|
||||
createdAt: epochs.get(streamId) ?? 1,
|
||||
status,
|
||||
terminalPersistencePending: pending.has(streamId),
|
||||
};
|
||||
}),
|
||||
onGenerationSettled: (listener: GenerationSettledListener) => {
|
||||
listeners.add(listener);
|
||||
return () => {
|
||||
listeners.delete(listener);
|
||||
};
|
||||
},
|
||||
settle(event: Omit<GenerationSettledEvent, 'streamId' | 'userId'>) {
|
||||
statuses.set(event.conversationId, event.status as GenerationJobStatus);
|
||||
for (const listener of listeners) {
|
||||
listener({ ...event, streamId: event.conversationId, userId: 'user-1' });
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe('waitForGenerationSettled', () => {
|
||||
beforeEach(() => {
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
it.each([
|
||||
['no generation', {}],
|
||||
['a completed generation', { 'conversation-1': 'complete' as const }],
|
||||
])('resolves false at once for %s', async (_label, initial) => {
|
||||
const source = createSource(initial);
|
||||
|
||||
await expect(waitForGenerationSettled(source, 'conversation-1')).resolves.toBe(false);
|
||||
expect(source.listeners.size).toBe(0);
|
||||
});
|
||||
|
||||
it('resolves when this conversation settles, ignoring others', async () => {
|
||||
const source = createSource({ 'conversation-1': 'running', 'conversation-2': 'running' });
|
||||
let resolved: boolean | undefined;
|
||||
void waitForGenerationSettled(source, 'conversation-1').then((settled) => {
|
||||
resolved = settled;
|
||||
});
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
|
||||
source.settle({ conversationId: 'conversation-2', status: 'complete' });
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
expect(resolved).toBeUndefined();
|
||||
|
||||
source.settle({ conversationId: 'conversation-1', status: 'complete' });
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
expect(resolved).toBe(true);
|
||||
expect(source.listeners.size).toBe(0);
|
||||
});
|
||||
|
||||
it('waits through an approval pause', async () => {
|
||||
const source = createSource({ 'conversation-1': 'requires_action' });
|
||||
let resolved: boolean | undefined;
|
||||
void waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 }).then(
|
||||
(settled) => {
|
||||
resolved = settled;
|
||||
},
|
||||
);
|
||||
|
||||
await jest.advanceTimersByTimeAsync(5_000);
|
||||
expect(resolved).toBeUndefined();
|
||||
});
|
||||
|
||||
it('observes a generation that settles on another replica', async () => {
|
||||
const source = createSource({ 'conversation-1': 'running' });
|
||||
const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 });
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
|
||||
source.statuses.set('conversation-1', 'complete');
|
||||
await jest.advanceTimersByTimeAsync(1_000);
|
||||
|
||||
await expect(waiting).resolves.toBe(true);
|
||||
expect(source.listeners.size).toBe(0);
|
||||
});
|
||||
|
||||
it('keeps waiting for as long as the generation stays active by default', async () => {
|
||||
const source = createSource({ 'conversation-1': 'running' });
|
||||
let resolved: boolean | undefined;
|
||||
void waitForGenerationSettled(source, 'conversation-1', { recheckMs: 60 * 60 * 1_000 }).then(
|
||||
(settled) => {
|
||||
resolved = settled;
|
||||
},
|
||||
);
|
||||
|
||||
await jest.advanceTimersByTimeAsync(48 * 60 * 60 * 1_000);
|
||||
expect(resolved).toBeUndefined();
|
||||
|
||||
source.settle({ conversationId: 'conversation-1', status: 'complete' });
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
expect(resolved).toBe(true);
|
||||
});
|
||||
|
||||
it('gives up after the maximum wait', async () => {
|
||||
const source = createSource({ 'conversation-1': 'running' });
|
||||
const waiting = waitForGenerationSettled(source, 'conversation-1', {
|
||||
recheckMs: 1_000,
|
||||
maxWaitMs: 10_000,
|
||||
});
|
||||
|
||||
await jest.advanceTimersByTimeAsync(10_000);
|
||||
|
||||
await expect(waiting).resolves.toBe(false);
|
||||
expect(source.listeners.size).toBe(0);
|
||||
});
|
||||
|
||||
it('stops waiting when the caller aborts', async () => {
|
||||
const source = createSource({ 'conversation-1': 'running' });
|
||||
const controller = new AbortController();
|
||||
const waiting = waitForGenerationSettled(source, 'conversation-1', {
|
||||
signal: controller.signal,
|
||||
});
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
|
||||
controller.abort();
|
||||
|
||||
await expect(waiting).resolves.toBe(false);
|
||||
expect(source.listeners.size).toBe(0);
|
||||
});
|
||||
|
||||
it('recovers from initial and later read failures without unsubscribing', async () => {
|
||||
const source = createSource({ 'conversation-1': 'running' });
|
||||
source.getGenerationSettlementState.mockRejectedValueOnce(new Error('job store unavailable'));
|
||||
const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 });
|
||||
await jest.advanceTimersByTimeAsync(1_000);
|
||||
expect(source.getGenerationSettlementState).toHaveBeenCalledTimes(2);
|
||||
expect(source.listeners.size).toBe(1);
|
||||
source.getGenerationSettlementState.mockRejectedValueOnce(new Error('later outage'));
|
||||
await jest.advanceTimersByTimeAsync(1_000);
|
||||
expect(source.listeners.size).toBe(1);
|
||||
source.statuses.set('conversation-1', 'complete');
|
||||
await jest.advanceTimersByTimeAsync(1_000);
|
||||
await expect(waiting).resolves.toBe(true);
|
||||
expect(source.listeners.size).toBe(0);
|
||||
});
|
||||
|
||||
it.each(['running', 'requires_action'] as const)(
|
||||
'does not follow a %s successor on another replica',
|
||||
async (status) => {
|
||||
const source = createSource({ 'conversation-1': 'running' });
|
||||
const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 });
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
source.statuses.set('conversation-1', status);
|
||||
source.epochs.set('conversation-1', 2);
|
||||
await jest.advanceTimersByTimeAsync(1_000);
|
||||
await expect(waiting).resolves.toBe(true);
|
||||
expect(source.listeners.size).toBe(0);
|
||||
},
|
||||
);
|
||||
|
||||
it('uses the dispatch epoch even when the first successful read sees its successor', async () => {
|
||||
const source = createSource({ 'conversation-1': 'requires_action' });
|
||||
source.epochs.set('conversation-1', 2);
|
||||
source.getGenerationSettlementState.mockRejectedValueOnce(new Error('outage'));
|
||||
const waiting = waitForGenerationSettled(source, 'conversation-1', {
|
||||
generationCreatedAt: 1,
|
||||
recheckMs: 1_000,
|
||||
});
|
||||
await jest.advanceTimersByTimeAsync(1_000);
|
||||
await expect(waiting).resolves.toBe(true);
|
||||
});
|
||||
|
||||
it.each(['complete', 'error', 'aborted'] as const)(
|
||||
'waits for %s terminal persistence even after a stale settlement event',
|
||||
async (status) => {
|
||||
const source = createSource({ 'conversation-1': status });
|
||||
source.pending.add('conversation-1');
|
||||
let resolved = false;
|
||||
const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 }).then(
|
||||
(value) => {
|
||||
resolved = true;
|
||||
return value;
|
||||
},
|
||||
);
|
||||
await jest.advanceTimersByTimeAsync(120_000);
|
||||
source.settle({ conversationId: 'conversation-1', status });
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
expect(resolved).toBe(false);
|
||||
source.pending.delete('conversation-1');
|
||||
await jest.advanceTimersByTimeAsync(1_000);
|
||||
await expect(waiting).resolves.toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it('finishes when the observed job disappears', async () => {
|
||||
const source = createSource({ 'conversation-1': 'running' });
|
||||
const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 });
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
source.statuses.delete('conversation-1');
|
||||
await jest.advanceTimersByTimeAsync(1_000);
|
||||
await expect(waiting).resolves.toBe(true);
|
||||
});
|
||||
|
||||
it('resolves a pre-aborted wait without subscribing or creating timers', async () => {
|
||||
const source = createSource({ 'conversation-1': 'running' });
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
await expect(
|
||||
waitForGenerationSettled(source, 'conversation-1', { signal: controller.signal }),
|
||||
).resolves.toBe(false);
|
||||
expect(source.getGenerationSettlementState).not.toHaveBeenCalled();
|
||||
expect(source.listeners.size).toBe(0);
|
||||
expect(jest.getTimerCount()).toBe(0);
|
||||
});
|
||||
|
||||
it('serializes reads and remembers a settlement event during an in-flight read', async () => {
|
||||
const source = createSource({ 'conversation-1': 'running' });
|
||||
let release!: () => void;
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
source.getGenerationSettlementState.mockImplementationOnce(async () => {
|
||||
await gate;
|
||||
return { createdAt: 1, status: 'running', terminalPersistencePending: false };
|
||||
});
|
||||
const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 });
|
||||
await jest.advanceTimersByTimeAsync(10_000);
|
||||
source.settle({ conversationId: 'conversation-1', status: 'complete' });
|
||||
source.settle({ conversationId: 'conversation-1', status: 'complete' });
|
||||
expect(source.getGenerationSettlementState).toHaveBeenCalledTimes(1);
|
||||
release();
|
||||
await expect(waiting).resolves.toBe(true);
|
||||
expect(source.getGenerationSettlementState).toHaveBeenCalledTimes(2);
|
||||
expect(jest.getTimerCount()).toBe(0);
|
||||
});
|
||||
|
||||
it('removes abort listeners and ignores reads completing after cancellation', async () => {
|
||||
const source = createSource({ 'conversation-1': 'running' });
|
||||
const controller = new AbortController();
|
||||
const remove = jest.spyOn(controller.signal, 'removeEventListener');
|
||||
let release!: () => void;
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
source.getGenerationSettlementState.mockImplementationOnce(async () => {
|
||||
await gate;
|
||||
return undefined;
|
||||
});
|
||||
const waiting = waitForGenerationSettled(source, 'conversation-1', {
|
||||
signal: controller.signal,
|
||||
maxWaitMs: 500,
|
||||
});
|
||||
controller.abort();
|
||||
await expect(waiting).resolves.toBe(false);
|
||||
release();
|
||||
await jest.advanceTimersByTimeAsync(1_000);
|
||||
expect(remove).toHaveBeenCalledWith('abort', expect.any(Function));
|
||||
expect(source.listeners.size).toBe(0);
|
||||
expect(jest.getTimerCount()).toBe(0);
|
||||
});
|
||||
});
|
||||
|
|
@ -85,3 +85,7 @@ export { RedisJobStore } from './implementations/RedisJobStore';
|
|||
export { RedisEventTransport } from './implementations/RedisEventTransport';
|
||||
|
||||
export { waitForGenerationPersistence } from './persistence';
|
||||
export { waitForGenerationSettled } from './settled';
|
||||
export type { GenerationSettledSource, GenerationSettledWaitOptions } from './settled';
|
||||
|
||||
export type { GenerationSettlementState } from './interfaces/IJobStore';
|
||||
|
|
|
|||
|
|
@ -103,9 +103,13 @@ export type JobStatus = 'running' | 'complete' | 'error' | 'aborted' | 'requires
|
|||
* Missing markers on pre-rollout records are interpreted as protocol v1. */
|
||||
export type GenerationProtocolVersion = 1 | 2;
|
||||
|
||||
/**
|
||||
* Serializable job data - no object references, suitable for Redis/external storage
|
||||
*/
|
||||
/** Read-only durable evidence, without attaching a runtime or recovering a slow save. */
|
||||
export type GenerationSettlementState = Pick<
|
||||
SerializableJobData,
|
||||
'createdAt' | 'status' | 'terminalPersistencePending'
|
||||
>;
|
||||
|
||||
/** Serializable job data without object references, suitable for external storage. */
|
||||
export interface SerializableJobData {
|
||||
streamId: string;
|
||||
userId: string;
|
||||
|
|
|
|||
114
packages/api/src/stream/settled.ts
Normal file
114
packages/api/src/stream/settled.ts
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
import { logger } from '@librechat/data-schemas';
|
||||
import type { GenerationSettledListener } from './GenerationJobManager';
|
||||
import type { GenerationSettlementState } from './interfaces/IJobStore';
|
||||
|
||||
const DEFAULT_RECHECK_MS = 60_000;
|
||||
|
||||
export interface GenerationSettledSource {
|
||||
getGenerationSettlementState: (
|
||||
streamId: string,
|
||||
) => Promise<GenerationSettlementState | undefined>;
|
||||
onGenerationSettled: (listener: GenerationSettledListener) => () => void;
|
||||
}
|
||||
|
||||
export interface GenerationSettledWaitOptions {
|
||||
recheckMs?: number;
|
||||
/** Pin the dispatch generation even if it was replaced before the first read. */
|
||||
generationCreatedAt?: number;
|
||||
/** Optional caller deadline. Store errors are not evidence of settlement. */
|
||||
maxWaitMs?: number;
|
||||
/** Stops waiting (resolving `false`) once the caller no longer needs the answer. */
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
/** Waits for the dispatch generation, including terminal persistence. Events request
|
||||
* a fresh read; they cannot settle an unrelated epoch or bypass an unfinished save.
|
||||
* The first successful read pins the epoch if the caller did not supply it. Reads
|
||||
* are single-flight, with a bounded periodic fallback for remote events and outages. */
|
||||
export function waitForGenerationSettled(
|
||||
source: GenerationSettledSource,
|
||||
conversationId: string,
|
||||
{
|
||||
recheckMs = DEFAULT_RECHECK_MS,
|
||||
generationCreatedAt,
|
||||
maxWaitMs,
|
||||
signal,
|
||||
}: GenerationSettledWaitOptions = {},
|
||||
): Promise<boolean> {
|
||||
if (signal?.aborted) {
|
||||
return Promise.resolve(false);
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
let done = false;
|
||||
let reading = false;
|
||||
let readRequested = false;
|
||||
let epoch = generationCreatedAt;
|
||||
let observedActive = epoch != null;
|
||||
let unsubscribe: (() => void) | undefined = undefined;
|
||||
let recheck: NodeJS.Timeout | undefined;
|
||||
let deadline: NodeJS.Timeout | undefined;
|
||||
const abort = (): void => finish(false);
|
||||
const finish = (settled: boolean): void => {
|
||||
if (done) return;
|
||||
done = true;
|
||||
unsubscribe?.();
|
||||
signal?.removeEventListener('abort', abort);
|
||||
clearTimeout(recheck);
|
||||
clearTimeout(deadline);
|
||||
resolve(settled);
|
||||
};
|
||||
const readState = async (): Promise<void> => {
|
||||
if (done) return;
|
||||
if (reading) {
|
||||
readRequested = true;
|
||||
return;
|
||||
}
|
||||
reading = true;
|
||||
clearTimeout(recheck);
|
||||
try {
|
||||
const job = await source.getGenerationSettlementState(conversationId);
|
||||
if (done) return;
|
||||
if (job == null || (epoch != null && epoch !== job.createdAt)) {
|
||||
finish(observedActive);
|
||||
return;
|
||||
}
|
||||
epoch = job.createdAt;
|
||||
const active =
|
||||
job.status === 'running' ||
|
||||
job.status === 'requires_action' ||
|
||||
job.terminalPersistencePending === true;
|
||||
if (!active) {
|
||||
finish(observedActive);
|
||||
return;
|
||||
}
|
||||
observedActive = true;
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
`[GenerationSettled] Failed to read generation state for ${conversationId}:`,
|
||||
error,
|
||||
);
|
||||
} finally {
|
||||
reading = false;
|
||||
if (!done) {
|
||||
if (readRequested) {
|
||||
readRequested = false;
|
||||
void readState();
|
||||
} else {
|
||||
recheck = setTimeout(() => void readState(), recheckMs);
|
||||
recheck.unref?.();
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
if (maxWaitMs != null) {
|
||||
deadline = setTimeout(() => finish(false), maxWaitMs);
|
||||
deadline.unref?.();
|
||||
}
|
||||
signal?.addEventListener('abort', abort, { once: true });
|
||||
unsubscribe = source.onGenerationSettled((event) => {
|
||||
if (event.conversationId === conversationId) void readState();
|
||||
});
|
||||
if (done) unsubscribe();
|
||||
else void readState();
|
||||
});
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue