🫂 fix: Keep Legacy Background Results Until Durable (#16405)

Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
lia-by-librechat[bot] 2026-09-26 21:59:33 -04:00 • committed by GitHub
parent d7209efa21
commit 9ffcd28909
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 290 additions and 14 deletions

View file

@ -194,6 +194,39 @@ describe('background task shutdown', () => {
expect(summary).toEqual({ tracked: 1, interrupted: 1, flushed: 1, unsettled: 1 });
});
it('does not retain an unconfirmed result after its task expires', async () => {
const registry = new BackgroundTaskRegistryClass();
const task = createTask(registry, 'unconfirmed-expired');
const controlled = controlledHandle();
registry.trackShutdown(task, controlled.handle);
registry.fail('shutdown-owner', 'shutdown-conversation', task.id, 'no durable write');
const now = jest.spyOn(Date, 'now').mockReturnValue(task.updatedAt + 60 * 60 * 1000 + 1);
try {
expect(registry.get('shutdown-owner', 'shutdown-conversation', task.id)).toBeUndefined();
} finally {
now.mockRestore();
}
expect((await registry.drainForShutdown(drainOptions())).tracked).toBe(0);
});
it('does not retain an unconfirmed result after capacity pressure evicts its task', async () => {
const registry = new BackgroundTaskRegistryClass();
const task = createTask(registry, 'unconfirmed-evicted');
const controlled = controlledHandle();
registry.trackShutdown(task, controlled.handle);
registry.fail('shutdown-owner', 'shutdown-conversation', task.id, 'no durable write');
task.updatedAt = Date.now() - 1_000;
for (let i = 0; i < 199; i++) {
const settled = createTask(registry, `completed-${i}`);
registry.complete('shutdown-owner', 'shutdown-conversation', settled.id, { content: 'ok' });
}
createTask(registry, 'trigger-eviction');
expect(registry.get('shutdown-owner', 'shutdown-conversation', task.id)).toBeUndefined();
expect((await registry.drainForShutdown(drainOptions())).tracked).toBe(0);
});
it('stops tracking a task once its result is durable', async () => {
const registry = new BackgroundTaskRegistryClass();
const controlled = controlledHandle();

View file

@ -887,6 +887,12 @@ export class BackgroundTaskRegistryClass {
return `${userId}::${conversationId}`;
}
private releaseEvictedShutdownHandle(task: BackgroundTask): void {
if (this.shutdownHandles.delete(task)) {
logger.warn(`[background] Unconfirmed shutdown result for evicted task ${task.id}.`);
}
}
private sweepBucketTasks(bucket: TaskBucket, now: number): void {
for (const [taskId, task] of bucket.tasks) {
if (
@ -895,6 +901,7 @@ export class BackgroundTaskRegistryClass {
now - task.updatedAt > COMPLETED_TASK_TTL_MS
) {
bucket.tasks.delete(taskId);
this.releaseEvictedShutdownHandle(task);
}
}
/** Drop dedupe mappings whose task was evicted (keys are
@ -925,6 +932,9 @@ export class BackgroundTaskRegistryClass {
(task) => task.status === 'running' || task.completionPersistencePending === true,
)
) {
for (const task of bucket.tasks.values()) {
this.releaseEvictedShutdownHandle(task);
}
this.buckets.delete(bucketKey);
continue;
}
@ -1079,6 +1089,7 @@ export class BackgroundTaskRegistryClass {
const touched = new Set<TaskBucket>();
for (const [task, bucket] of selected) {
bucket.tasks.delete(task.id);
this.releaseEvictedShutdownHandle(task);
touched.add(bucket);
}
const now = Date.now();

View file

@ -234,6 +234,221 @@ describe('createToolExecuteHandler — background tasks at shutdown', () => {
);
});
it('waits for a legacy pre-admitted completion projection before declaring shutdown durable', async () => {
const { createToolExecuteHandler, registry } = loadModules();
let notifyFlush: () => void = () => undefined;
const flushStarted = new Promise<void>((resolve) => {
notifyFlush = resolve;
});
const trackShutdown = registry.trackShutdown.bind(registry);
jest.spyOn(registry, 'trackShutdown').mockImplementation((task, handle) =>
trackShutdown(task, {
...handle,
flush: (reason) => {
notifyFlush();
return handle.flush(reason);
},
}),
);
let resolveProjection: (persisted: boolean) => void = () => undefined;
const persist = jest.fn(
() =>
new Promise<boolean>((resolve) => {
resolveProjection = resolve;
}),
);
const retire = jest.fn(async () => true);
const tool = {
name: 'search_mcp_docs',
description: 'search docs',
schema: z.object({ q: z.string() }),
invoke: jest.fn(async () => ({ content: 'legacy projection result' })),
} as unknown as StructuredToolInterface;
const handler = createToolExecuteHandler({
loadTools: async () => ({ loadedTools: [tool] }),
backgroundToolCompletion: {
preregister: jest.fn(async () => ({ renew: async () => true, retire })),
persist,
claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
});
await runBatch(handler, {
toolCalls: [
{
id: 'call-legacy-projection',
name: tool.name,
args: { q: 'legacy', run_in_background: true },
stepId: 'step-legacy',
},
],
configurable: buildConfig([tool.name]),
metadata: { thread_id: 'shutdown_convo', run_id: 'response-legacy-projection' },
});
await flushMicrotasks();
expect(persist).toHaveBeenCalledTimes(1);
const draining = registry.drainForShutdown({
...drainOptions(),
deadlineAt: Date.now() + 1_750,
flushReserveMs: 1_400,
});
let drained = false;
void draining.then(() => {
drained = true;
});
await flushStarted;
await flushMicrotasks();
expect(drained).toBe(false);
resolveProjection(true);
expect(await draining).toEqual({ tracked: 1, interrupted: 0, flushed: 1, unsettled: 0 });
expect(retire).not.toHaveBeenCalled();
});
it('reports a pre-admitted legacy completion with no durable projection as unsettled', async () => {
const { createToolExecuteHandler, registry } = loadModules();
const persist = jest.fn(async () => false);
const tool = {
name: 'search_mcp_docs',
description: 'search docs',
schema: z.object({ q: z.string() }),
invoke: jest.fn(async () => ({ content: 'missing legacy result' })),
} as unknown as StructuredToolInterface;
const handler = createToolExecuteHandler({
loadTools: async () => ({ loadedTools: [tool] }),
backgroundToolCompletion: {
preregister: jest.fn(async () => ({ renew: async () => true, retire: async () => true })),
persist,
claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
});
await runBatch(handler, {
toolCalls: [
{
id: 'call-legacy-failed',
name: tool.name,
args: { q: 'failed', run_in_background: true },
stepId: 'step-legacy-failed',
},
],
configurable: buildConfig([tool.name]),
metadata: { thread_id: 'shutdown_convo', run_id: 'response-legacy-failed' },
});
await flushMicrotasks();
expect(persist).toHaveBeenCalledTimes(1);
expect(
await registry.drainForShutdown({
...drainOptions(),
deadlineAt: Date.now() + 190,
interruptGraceMs: 75,
flushReserveMs: 40,
}),
).toEqual({
tracked: 1,
interrupted: 0,
flushed: 1,
unsettled: 1,
});
});
it('projects an interrupted result for an uncooperative legacy completion without a receipt', async () => {
const { createToolExecuteHandler, registry } = loadModules();
const persist = jest.fn(async () => true);
const controlled = controlledTool({ honorAbort: false });
const handler = createToolExecuteHandler({
loadTools: async () => ({ loadedTools: [controlled.tool] }),
backgroundToolCompletion: {
preregister: jest.fn(async () => ({ renew: async () => true, retire: async () => true })),
persist,
claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
});
await runBatch(handler, {
toolCalls: [
{
id: 'call-legacy-stubborn',
name: controlled.tool.name,
args: { q: 'stubborn', run_in_background: true },
stepId: 'step-legacy-stubborn',
},
],
configurable: buildConfig([controlled.tool.name]),
metadata: { thread_id: 'shutdown_convo', run_id: 'response-legacy-stubborn' },
});
expect(
await registry.drainForShutdown({
...drainOptions(),
deadlineAt: Date.now() + 280,
interruptGraceMs: 90,
flushReserveMs: 70,
}),
).toEqual({
tracked: 1,
interrupted: 1,
flushed: 1,
unsettled: 0,
});
expect(persist).toHaveBeenCalledWith(
expect.objectContaining({
output: interruptedOutput,
backgroundTask: expect.objectContaining({ status: 'error' }),
}),
);
controlled.resolve('late success');
await flushMicrotasks();
await flushMicrotasks();
expect(persist).toHaveBeenCalledTimes(1);
});
it('reports a failed legacy interruption projection as unconfirmed', async () => {
const { createToolExecuteHandler, registry } = loadModules();
const persist = jest.fn(async () => false);
const controlled = controlledTool({ honorAbort: false });
const handler = createToolExecuteHandler({
loadTools: async () => ({ loadedTools: [controlled.tool] }),
backgroundToolCompletion: {
preregister: jest.fn(async () => ({ renew: async () => true, retire: async () => true })),
persist,
claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
});
await runBatch(handler, {
toolCalls: [
{
id: 'call-legacy-interruption-failed',
name: controlled.tool.name,
args: { q: 'stubborn', run_in_background: true },
stepId: 'step-legacy-interruption-failed',
},
],
configurable: buildConfig([controlled.tool.name]),
metadata: { thread_id: 'shutdown_convo', run_id: 'response-legacy-interruption-failed' },
});
expect(
await registry.drainForShutdown({
...drainOptions(),
deadlineAt: Date.now() + 280,
interruptGraceMs: 90,
flushReserveMs: 70,
}),
).toEqual({
tracked: 1,
interrupted: 1,
flushed: 1,
unsettled: 1,
});
expect(persist).toHaveBeenCalledTimes(1);
controlled.resolve('late success');
await flushMicrotasks();
await flushMicrotasks();
});
it('waits for the projected result when an independent receipt write returns false', async () => {
const { createToolExecuteHandler, registry } = loadModules();
const adapter = completionAdapter();

View file

@ -5963,6 +5963,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
let durableReceiptWrite: Promise<boolean> | undefined;
let durableReceiptAmbiguous = false;
let durableResultConfirmed = false;
let forcedShutdownResult = false;
let resolveDurableReceipt: () => void = () => undefined;
const durableReceiptSettled = new Promise<void>((resolve) => {
resolveDurableReceipt = resolve;
@ -6306,11 +6307,17 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
);
}
};
const persistSettledBackgroundResult = async (params: {
output?: string;
artifact?: unknown;
status: 'completed' | 'error' | 'cancelled';
}): Promise<void> => {
const persistSettledBackgroundResult = async (
params: {
output?: string;
artifact?: unknown;
status: 'completed' | 'error' | 'cancelled';
},
forced = false,
): Promise<void> => {
if (forcedShutdownResult && !forced) {
return;
}
settledReceipt = { status: params.status, output: params.output };
/** Held for the whole persist, including a code harvest that waits for
* a long dispatch turn, so retention pressure cannot evict the task. */
@ -6671,7 +6678,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
})();
void settlement.then(
() => {
if (completionAdmission?.persistResult == null || !completionPreregistered) {
if (completionAdmission == null || !completionPreregistered) {
resolveDurableReceipt();
}
},
@ -6693,6 +6700,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
} else if (settledReceipt != null) {
await writeDurableReceipt({ ...settledReceipt, settledAt: new Date() });
} else {
forcedShutdownResult = true;
const failure = toBackgroundToolFailure(tc.name, reason);
backgroundTaskRegistry.fail(
backgroundUserId,
@ -6709,22 +6717,31 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
detachedError,
);
}
await writeDurableReceipt({
status: 'error',
output: failure,
settledAt: new Date(),
});
if (completionAdmission?.persistResult != null) {
await writeDurableReceipt({
status: 'error',
output: failure,
settledAt: new Date(),
});
} else if (completionAdmission != null) {
await persistSettledBackgroundResult(
{ status: 'error', output: failure },
true,
);
}
}
if (durableResultConfirmed || completionAdmission?.persistResult == null) {
if (durableResultConfirmed || completionAdmission == null) {
return;
}
if (settledReceipt != null) {
if (settledReceipt != null && !forcedShutdownResult) {
await settlement;
if (durableResultConfirmed) {
return;
}
}
throw new Error(`Background task ${task.id} has no durable shutdown result`);
if (completionPreregistered) {
throw new Error(`Background task ${task.id} has no durable shutdown result`);
}
},
});
if (