⛽ feat: Negotiate Background Command Timeouts (#16100)

* ⏳ feat: Negotiate BYOM Background Command Timeouts

* ⏱️ fix: Preserve Foreground Programmatic Timeout
This commit is contained in:
Danny Avila 2026-09-19 15:01:41 -04:00 • committed by GitHub
parent 61d034c20f
commit 13c74c9574
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 147 additions and 12 deletions

View file

@ -2300,6 +2300,7 @@ async function loadToolsForExecution({
gitIdentity: agent?.git_identity,
maxTimeoutMs: resolveAttachedWorkspaceCommandTimeoutMax(
codeExecutionContext.codeEnvironmentConfigSchema,
codeExecutionContext.codeWorkspace?.maxCommandTimeoutMs,
),
maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs(
codeExecutionContext.codeEnvironmentConfigSchema,

View file

@ -41,6 +41,8 @@ export interface CodeExecutionContext {
/** Live, server-validated directory selection. Never derive session reuse from this field. */
codeWorkspace?: CodeWorkspaceSelection & {
operations: CodeWorkspaceOperation[];
/** Live Code API execution ceiling. Omitted by older deployments. */
maxCommandTimeoutMs?: number;
instructions?: CodeWorkspaceDescriptor['instructions'];
environment?: CodeWorkspaceDescriptor['environment'];
};

View file

@ -10,6 +10,7 @@ import {
CHECK_BACKGROUND_TASK_NAME,
} from './background';
import { BACKGROUND_TASK_ABORT_GRACE_MS, BACKGROUND_TASK_TIMEOUT_MS } from './backgroundCompletion';
import { BACKGROUND_TOOL_INVOCATION_CONFIG_KEY } from './invocation';
import { ContentFilterError } from '../middleware/contentFilter';
import { createToolExecuteHandler } from './handlers';
@ -183,6 +184,14 @@ describe('createToolExecuteHandler — background tool calls', () => {
});
expect(events.slice(0, 2)).toEqual(['preregister', 'invoke']);
expect(tool.invoke).toHaveBeenCalledWith(
{ q: 'continuations' },
expect.objectContaining({
configurable: expect.objectContaining({
[BACKGROUND_TOOL_INVOCATION_CONFIG_KEY]: true,
}),
}),
);
expect(JSON.parse(dispatch.content).message).toContain('host will resume you');
await flushMicrotasks();
expect(events).toEqual(['preregister', 'invoke', 'persist']);

View file

@ -121,6 +121,7 @@ import {
import { buildSkillPrimeMessage, isSkillFilePath, SKILL_FILE_PREFIX } from './skills';
import { resolveCallerCapabilityProjectionSnapshot } from './callerCapabilities';
import { resolveAttachedWorkspaceQueueWaitMs } from '~/code/command';
import { BACKGROUND_TOOL_INVOCATION_CONFIG_KEY } from './invocation';
import { mergeCodeFilesIntoContext } from './codeFilesSession';
import { toolValidationFeedback } from './validationFeedback';
import { createSkillContentDigest } from './compatibility';
@ -6010,6 +6011,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
signal: backgroundAbortController.signal,
configurable: {
...mergedConfigurable,
[BACKGROUND_TOOL_INVOCATION_CONFIG_KEY]: true,
...(detachedReservation?.status === 'reserved'
? {
eventActorDetachedAction: {

View file

@ -1993,6 +1993,7 @@ export async function initializeAgent(
trustedCodeExecutionContext.environmentType === 'attached'
? resolveAttachedWorkspaceCommandTimeoutMax(
trustedCodeExecutionContext.codeEnvironmentConfigSchema,
trustedCodeExecutionContext.codeWorkspace?.maxCommandTimeoutMs,
)
: undefined;
if (

View file

@ -0,0 +1,2 @@
/** Host-only marker added when an ordinary tool is executing as a detached background task. */
export const BACKGROUND_TOOL_INVOCATION_CONFIG_KEY = '__librechatBackgroundToolInvocation';

View file

@ -84,6 +84,7 @@ describe('getCodeBridgeWorkerStatus', () => {
online: true,
ready: true,
leaseExpiresInMs: 45_000,
maxCommandTimeoutMs: 120_000,
capabilities: {
statefulWorkspace: true,
sandboxProfile: 'native-srt',
@ -114,6 +115,7 @@ describe('getCodeBridgeWorkerStatus', () => {
status: 'ready',
statefulWorkspace: true,
leaseExpiresInMs: 45_000,
maxCommandTimeoutMs: 120_000,
sandboxProfile: 'native-srt',
runtimes: ['bash'],
operations: ['read_file', 'execute_command'],
@ -172,6 +174,8 @@ describe('getCodeBridgeWorkerStatus', () => {
{ online: true, ready: false },
{ online: false, ready: false, leaseExpiresInMs: 5_000 },
{ online: true, ready: true, leaseExpiresInMs: 60_001 },
{ online: true, ready: true, maxCommandTimeoutMs: 0 },
{ online: true, ready: true, maxCommandTimeoutMs: 300_001 },
{
online: true,
ready: true,

View file

@ -1,5 +1,6 @@
import { createHash } from 'node:crypto';
import {
CODE_ENVIRONMENT_COMMAND_TIMEOUT_HARD_MAX_MS,
CODE_WORKSPACE_ID_PATTERN,
CODE_WORKSPACE_MAX_COUNT,
CODE_WORKSPACE_OPERATIONS,
@ -38,6 +39,7 @@ export type CodeBridgeWorkerStatus = {
operations?: CodeWorkspaceOperation[];
workspaces?: CodeWorkspaceDescriptor[];
programmaticLanguages?: ['bash'];
maxCommandTimeoutMs?: number;
};
export type CodeBridgeFetch = (
@ -292,6 +294,7 @@ export async function getCodeBridgeWorkerStatus({
online?: unknown;
ready?: unknown;
leaseExpiresInMs?: unknown;
maxCommandTimeoutMs?: unknown;
capabilities?: {
statefulWorkspace?: unknown;
sandboxProfile?: unknown;
@ -306,6 +309,12 @@ export async function getCodeBridgeWorkerStatus({
Number.isSafeInteger(status.leaseExpiresInMs) &&
status.leaseExpiresInMs > 0 &&
status.leaseExpiresInMs <= 60_000);
const validCommandTimeout =
status.maxCommandTimeoutMs == null ||
(typeof status.maxCommandTimeoutMs === 'number' &&
Number.isSafeInteger(status.maxCommandTimeoutMs) &&
status.maxCommandTimeoutMs >= 1 &&
status.maxCommandTimeoutMs <= CODE_ENVIRONMENT_COMMAND_TIMEOUT_HARD_MAX_MS);
const validCapabilities =
capabilities == null || validStatusString(capabilities.sandboxProfile);
const validRuntimes = capabilities == null || validStatusStringArray(capabilities.runtimes);
@ -322,6 +331,7 @@ export async function getCodeBridgeWorkerStatus({
(status.online && (status.leaseExpiresInMs == null || capabilities == null)) ||
(!status.online && (status.leaseExpiresInMs != null || capabilities != null)) ||
!validLease ||
!validCommandTimeout ||
!validCapabilities ||
!validRuntimes ||
!validWorkspaceTools ||
@ -366,6 +376,9 @@ export async function getCodeBridgeWorkerStatus({
...(typeof status.leaseExpiresInMs !== 'number'
? {}
: { leaseExpiresInMs: status.leaseExpiresInMs }),
...(typeof status.maxCommandTimeoutMs !== 'number'
? {}
: { maxCommandTimeoutMs: status.maxCommandTimeoutMs }),
...(typeof capabilities?.sandboxProfile !== 'string'
? {}
: { sandboxProfile: capabilities.sandboxProfile }),

View file

@ -351,7 +351,11 @@ describe('resolveCodeExecutionWorkspaceContext', () => {
delete process.env.TEST_CODE_CAPABILITY_TOKEN;
});
function workspaceStatus(workspaces: unknown[], statefulWorkspace: boolean = true): Response {
function workspaceStatus(
workspaces: unknown[],
statefulWorkspace: boolean = true,
maxCommandTimeoutMs?: number,
): Response {
return new Response(
JSON.stringify({
protocolVersion: 1,
@ -359,6 +363,7 @@ describe('resolveCodeExecutionWorkspaceContext', () => {
online: true,
ready: true,
leaseExpiresInMs: 45_000,
maxCommandTimeoutMs,
capabilities: {
statefulWorkspace,
sandboxProfile: 'native-srt',
@ -375,10 +380,14 @@ describe('resolveCodeExecutionWorkspaceContext', () => {
it('binds the exact advertised workspace and its operation ceiling', async () => {
jest.spyOn(globalThis, 'fetch').mockResolvedValue(
workspaceStatus([
{ id: 'project-a', name: 'Project A' },
{ id: 'docs', operations: ['read_file', 'list_files'] },
]),
workspaceStatus(
[
{ id: 'project-a', name: 'Project A' },
{ id: 'docs', operations: ['read_file', 'list_files'] },
],
true,
120_000,
),
);
await expect(
@ -397,6 +406,7 @@ describe('resolveCodeExecutionWorkspaceContext', () => {
environmentId: 'personal',
workspaceId: 'docs',
operations: ['read_file', 'list_files'],
maxCommandTimeoutMs: 120_000,
},
});
});

View file

@ -178,6 +178,9 @@ export async function resolveCodeExecutionWorkspaceContext({
codeWorkspace: {
...selection,
operations: [...(workspace.operations ?? status.operations)],
...(status.maxCommandTimeoutMs == null
? {}
: { maxCommandTimeoutMs: status.maxCommandTimeoutMs }),
...(workspace.instructions ? { instructions: workspace.instructions } : {}),
...(workspace.environment ? { environment: workspace.environment } : {}),
},

View file

@ -8,8 +8,10 @@ import {
createAttachedWorkspaceBashTool,
createGitIdentityProgrammaticBashTool,
resolveAttachedWorkspaceCommandTimeoutMax,
resolveAttachedWorkspaceProgrammaticTimeout,
resolveAttachedWorkspaceQueueWaitMs,
} from './command';
import { BACKGROUND_TOOL_INVOCATION_CONFIG_KEY } from '~/agents/invocation';
describe('programmatic Bash Git identity', () => {
test('applies authorship before the SDK sends a programmatic script', async () => {
@ -88,6 +90,22 @@ describe('programmatic Bash Git identity', () => {
await once(server, 'close');
}
});
test('preserves the foreground timeout while bounding an explicit admin override', () => {
expect(resolveAttachedWorkspaceProgrammaticTimeout(undefined, 90_000)).toBe(30_000);
expect(
resolveAttachedWorkspaceProgrammaticTimeout(
{ limits: { maxCommandTimeoutMs: 120_000 } },
90_000,
),
).toBe(90_000);
expect(
resolveAttachedWorkspaceProgrammaticTimeout(
{ limits: { maxCommandTimeoutMs: 60_000 } },
90_000,
),
).toBe(60_000);
});
});
function commandResponse(overrides: Record<string, unknown> = {}): Response {
@ -264,6 +282,41 @@ describe('createAttachedWorkspaceBashTool', () => {
properties: { timeoutMs: { type: 'integer', minimum: 1, maximum: 120_000 } },
});
expect(resolveAttachedWorkspaceCommandTimeoutMax()).toBe(30_000);
expect(resolveAttachedWorkspaceCommandTimeoutMax(undefined, 90_000)).toBe(90_000);
expect(
resolveAttachedWorkspaceCommandTimeoutMax(
{ limits: { maxCommandTimeoutMs: 120_000 } },
90_000,
),
).toBe(90_000);
expect(
resolveAttachedWorkspaceCommandTimeoutMax(
{ limits: { maxCommandTimeoutMs: 60_000 } },
90_000,
),
).toBe(60_000);
});
test('uses the negotiated ceiling only for omitted detached background timeouts', async () => {
const fetchImpl: CodeBridgeFetch = jest.fn(async () => commandResponse());
const bashTool = createAttachedWorkspaceBashTool({
baseUrl: 'https://code.example.com/v1',
authHeaders: () => ({}),
workspaceId: 'project-a',
maxTimeoutMs: 90_000,
fetchImpl,
});
await bashTool.invoke(
{ command: 'npm test' },
{ configurable: { [BACKGROUND_TOOL_INVOCATION_CONFIG_KEY]: true } },
);
await bashTool.invoke({ command: 'npm test' });
const backgroundRequest = JSON.parse(String((fetchImpl as jest.Mock).mock.calls[0][1]?.body));
const foregroundRequest = JSON.parse(String((fetchImpl as jest.Mock).mock.calls[1][1]?.body));
expect(backgroundRequest).toMatchObject({ timeoutMs: 90_000 });
expect(foregroundRequest).toMatchObject({ timeoutMs: 30_000 });
});
test('resolves the administrator-configured admission budget', () => {

View file

@ -21,6 +21,7 @@ import {
WORKSPACE_COMMAND_MAX_TIMEOUT_MS,
WORKSPACE_QUEUE_MAX_WAIT_MS,
} from './workspace';
import { BACKGROUND_TOOL_INVOCATION_CONFIG_KEY } from '~/agents/invocation';
const DEFAULT_OUTPUT_BYTES = 256 * 1024;
@ -74,7 +75,7 @@ function buildAttachedTimeoutSchema(maxTimeoutMs: number): BoundedTimeoutSchema
type: 'integer',
minimum: 1,
maximum: maxTimeoutMs,
description: `Optional execution timeout in milliseconds, from 1 through ${maxTimeoutMs}. Defaults to ${defaultTimeoutMs}. Waiting for an available worker does not consume this execution budget.`,
description: `Optional execution timeout in milliseconds, from 1 through ${maxTimeoutMs}. Defaults to ${defaultTimeoutMs} for foreground calls and ${maxTimeoutMs} for detached background calls. Waiting for an available worker does not consume this execution budget.`,
};
}
@ -87,11 +88,41 @@ function normalizeAttachedWorkspaceCommandTimeoutMax(maxTimeoutMs: number): numb
export function resolveAttachedWorkspaceCommandTimeoutMax(
configSchema?: CodeEnvironmentUserConfigSchema,
upstreamMaxTimeoutMs?: number,
): number {
const configured = configSchema?.limits?.maxCommandTimeoutMs;
return configured == null
? WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS
: normalizeAttachedWorkspaceCommandTimeoutMax(configured);
const upstream =
upstreamMaxTimeoutMs == null
? WORKSPACE_COMMAND_MAX_TIMEOUT_MS
: normalizeAttachedWorkspaceCommandTimeoutMax(upstreamMaxTimeoutMs);
let requested = WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS;
if (configured != null) {
requested = normalizeAttachedWorkspaceCommandTimeoutMax(configured);
} else if (upstreamMaxTimeoutMs != null) {
requested = upstream;
}
return Math.min(requested, upstream);
}
/**
* Programmatic calls do not currently carry the detached-invocation marker.
* Preserve their historical foreground default while still enforcing both an
* explicit administrator override and the live upstream ceiling.
*/
export function resolveAttachedWorkspaceProgrammaticTimeout(
configSchema?: CodeEnvironmentUserConfigSchema,
upstreamMaxTimeoutMs?: number,
): number {
const configured = configSchema?.limits?.maxCommandTimeoutMs;
const requested =
configured == null
? WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS
: normalizeAttachedWorkspaceCommandTimeoutMax(configured);
const upstream =
upstreamMaxTimeoutMs == null
? WORKSPACE_COMMAND_MAX_TIMEOUT_MS
: normalizeAttachedWorkspaceCommandTimeoutMax(upstreamMaxTimeoutMs);
return Math.min(requested, upstream);
}
/**
@ -207,8 +238,9 @@ export function createContextProgrammaticBashTool(
...(attached
? {
workspaceId: context.codeWorkspace?.workspaceId,
runTimeoutMs: resolveAttachedWorkspaceCommandTimeoutMax(
runTimeoutMs: resolveAttachedWorkspaceProgrammaticTimeout(
context.codeEnvironmentConfigSchema,
context.codeWorkspace?.maxCommandTimeoutMs,
),
}
: {}),
@ -259,7 +291,7 @@ export function createAttachedWorkspaceBashTool({
workspaceId: string;
environment?: CodeWorkspaceDescriptor['environment'];
gitIdentity?: AgentGitIdentity | null;
/** Deployment ceiling already intersected with the protocol hard cap. */
/** Effective admin/upstream ceiling already intersected with the protocol hard cap. */
maxTimeoutMs?: number;
/** Deployment admission budget; omitted keeps the built-in default. */
maxQueueWaitMs?: number;
@ -305,7 +337,10 @@ export function createAttachedWorkspaceBashTool({
action ??
commandWithGitIdentity(commandWithArguments(rawInput.command!, rawInput.args), gitIdentity);
const timeoutMs =
rawInput.timeoutMs ?? Math.min(WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, effectiveMaxTimeoutMs);
rawInput.timeoutMs ??
(config?.configurable?.[BACKGROUND_TOOL_INVOCATION_CONFIG_KEY] === true
? effectiveMaxTimeoutMs
: Math.min(WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, effectiveMaxTimeoutMs));
const signal = config?.signal;
const trace = {
runId: config?.metadata?.run_id,