CERBERUS/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2026-10-22 01:53:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

🫀 fix: Expire Aborted Background Tool Leases (#15428)

Browse source 
* fix: Bound Background Tool Producer Heartbeats

* fix: fence timed-out delivery retirement
This commit is contained in:
Danny Avila 2026-08-31 14:13:00 -04:00 • committed by GitHub
parent 6be3f798fc
commit 2e2c29e10d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 75 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

3
packages/api/src/agents/backgroundCompletion.ts
View file

@ -1,5 +1,8 @@
/** Deadline after which an invocation owner requests cancellation. */
export const BACKGROUND_TASK_TIMEOUT_MS: number = 30 * 60 * 1000;
/** Gives a cooperative tool a short window to settle after cancellation before
* its automatic completion delivery is retired. */
export const BACKGROUND_TASK_ABORT_GRACE_MS: number = 60 * 1000;
/** Three missed heartbeats prove the process-local executor has been lost. */
export const BACKGROUND_TOOL_PRODUCER_LEASE_MS: number = 30_000;
export const BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS: number = 10_000;

22
packages/api/src/agents/handlers.background.spec.ts
View file

@ -2,6 +2,7 @@ import { z } from 'zod';
import { logger } from '@librechat/data-schemas';
import type { StructuredToolInterface } from '@librechat/agents/langchain/tools';
import type { FiltersConfig } from 'librechat-data-provider';
import { BACKGROUND_TASK_ABORT_GRACE_MS, BACKGROUND_TASK_TIMEOUT_MS } from './backgroundCompletion';
import { backgroundTaskRegistry, CHECK_BACKGROUND_TASK_NAME } from './background';
import { ContentFilterError } from '../middleware/contentFilter';
import { createToolExecuteHandler } from './handlers';
@ -540,7 +541,7 @@ describe('createToolExecuteHandler — background tool calls', () => {
}
});
it('keeps an abort-resistant invocation running without a false timeout receipt', async () => {
it('retires the automatic wakeup after an abort-resistant invocation exceeds its grace period', async () => {
jest.useFakeTimers({ doNotFake: ['setImmediate'] });
try {
const tool = {
@ -550,12 +551,14 @@ describe('createToolExecuteHandler — background tool calls', () => {
invoke: jest.fn(() => new Promise(() => undefined)),
} as unknown as StructuredToolInterface;
const persist = jest.fn(async () => true);
const renew = jest.fn(async () => true);
const retire = jest.fn(async () => true);
const handler = createToolExecuteHandler({
loadTools: async () => ({ loadedTools: [tool] }),
backgroundToolCompletion: {
preregister: jest.fn(async () => ({
renew: jest.fn(async () => true),
retire: jest.fn(async () => true),
renew,
retire,
})),
persist,
claim: jest.fn(async () => ({ status: 'acquired' as const })),
@ -576,12 +579,23 @@ describe('createToolExecuteHandler — background tool calls', () => {
metadata: { thread_id: 'exec_convo', run_id: 'response-timeout-resistant' },
});
jest.advanceTimersByTime(31 * 60 * 1000);
jest.advanceTimersByTime(BACKGROUND_TASK_TIMEOUT_MS + BACKGROUND_TASK_ABORT_GRACE_MS);
await flushMicrotasks();
await flushMicrotasks();
const taskId = JSON.parse(dispatch.content).background_task_id as string;
expect(backgroundTaskRegistry.get('exec_user', 'exec_convo', taskId)?.status).toBe('running');
expect(persist).not.toHaveBeenCalled();
expect(renew).toHaveBeenCalled();
expect(retire).toHaveBeenCalledWith(
'background task did not settle after its abort grace period',
{ onlyIfUnclaimed: true },
);
const renewalsBefore = renew.mock.calls.length;
jest.advanceTimersByTime(3 * 60 * 1000);
await flushMicrotasks();
expect(renew).toHaveBeenCalledTimes(renewalsBefore);
} finally {
jest.useRealTimers();
}

62
packages/api/src/agents/handlers.ts
View file

@ -67,6 +67,10 @@ import {
contentFilterModelBoundBlockResponse,
isContentFilterError,
} from '~/middleware/contentFilter';
import {
BACKGROUND_TASK_ABORT_GRACE_MS,
BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS,
} from './backgroundCompletion';
import {
hasIntentArg,
stripIntentArg,
@ -75,7 +79,6 @@ import {
} from './intent';
import { getSafeErrorMetadata, logAxiosError, runOutsideTracing, truncateMiddle } from '~/utils';
import { resolveCallerCapabilityProjectionSnapshot } from './callerCapabilities';
import { BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS } from './backgroundCompletion';
import { buildSkillPrimeMessage, SKILL_FILE_PREFIX } from './skills';
import { createSkillContentDigest } from './compatibility';
import { parseFrontmatter } from '../skills/import';
@ -4918,11 +4921,15 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
}
};
let producerHeartbeatInFlight: Promise<void> | undefined;
let producerHeartbeatStopped = false;
const producerAdmission = completionAdmission;
const producerHeartbeat =
producerAdmission == null
? undefined
: setInterval(() => {
if (producerHeartbeatStopped) {
return;
}
if (producerHeartbeatInFlight != null) {
return;
}
@ -4946,12 +4953,51 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
});
}, BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS);
(producerHeartbeat as { unref?: () => void } | undefined)?.unref?.();
const stopProducerHeartbeat = async (retireReason?: string): Promise<void> => {
if (!producerHeartbeatStopped) {
producerHeartbeatStopped = true;
if (producerHeartbeat != null) {
clearInterval(producerHeartbeat);
}
}
await producerHeartbeatInFlight;
if (retireReason == null || producerAdmission == null) {
return;
}
try {
const retired = await producerAdmission.retire(retireReason, {
onlyIfUnclaimed: true,
});
if (!retired) {
logger.warn(
`[background] Could not retire timed-out completion delivery for task ${task.id}.`,
);
}
} catch (retireError) {
logger.warn(
`[background] Failed to retire timed-out completion delivery for task ${task.id}:`,
retireError,
);
}
};
let producerRetirementTimeout: ReturnType<typeof setTimeout> | undefined;
const requestBackgroundAbort = (): void => {
backgroundAbortController.abort(
new DOMException('Background task timed out', 'AbortError'),
);
producerRetirementTimeout = setTimeout(() => {
producerRetirementTimeout = undefined;
void stopProducerHeartbeat(
'background task did not settle after its abort grace period',
);
}, BACKGROUND_TASK_ABORT_GRACE_MS);
producerRetirementTimeout.unref?.();
};
void (async () => {
try {
const result = await withBackgroundTaskTimeout(invokePromise, () =>
backgroundAbortController.abort(
new DOMException('Background task timed out', 'AbortError'),
),
const result = await withBackgroundTaskTimeout(
invokePromise,
requestBackgroundAbort,
);
if (isCodeCall) {
markCodeSandboxWarm();
@ -5093,10 +5139,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
await persistBackgroundResult({ output: deliveredError, status: 'error' });
await wakeDetachedActor();
} finally {
if (producerHeartbeat != null) {
clearInterval(producerHeartbeat);
if (producerRetirementTimeout != null) {
clearTimeout(producerRetirementTimeout);
}
await producerHeartbeatInFlight;
await stopProducerHeartbeat();
}
})();
if (