From 13c74c9574fb207ca25bd591b63168b8cf9f8cd4 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 19 Sep 2026 15:01:41 -0400 Subject: [PATCH] =?UTF-8?q?=E2=9B=BD=20feat:=20Negotiate=20Background=20Co?= =?UTF-8?q?mmand=20Timeouts=20(#16100)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ⏳ feat: Negotiate BYOM Background Command Timeouts * ⏱️ fix: Preserve Foreground Programmatic Timeout --- api/server/services/ToolService.js | 1 + packages/api/src/agents/execution.ts | 2 + .../src/agents/handlers.background.spec.ts | 9 ++++ packages/api/src/agents/handlers.ts | 2 + packages/api/src/agents/initialize.ts | 1 + packages/api/src/agents/invocation.ts | 2 + packages/api/src/code/bridge.spec.ts | 4 ++ packages/api/src/code/bridge.ts | 13 +++++ packages/api/src/code/capabilities.spec.ts | 20 +++++-- packages/api/src/code/capabilities.ts | 3 ++ packages/api/src/code/command.spec.ts | 53 +++++++++++++++++++ packages/api/src/code/command.ts | 49 ++++++++++++++--- 12 files changed, 147 insertions(+), 12 deletions(-) create mode 100644 packages/api/src/agents/invocation.ts diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.js index faf785e2b1..8231bac696 100644 --- a/api/server/services/ToolService.js +++ b/api/server/services/ToolService.js @@ -2300,6 +2300,7 @@ async function loadToolsForExecution({ gitIdentity: agent?.git_identity, maxTimeoutMs: resolveAttachedWorkspaceCommandTimeoutMax( codeExecutionContext.codeEnvironmentConfigSchema, + codeExecutionContext.codeWorkspace?.maxCommandTimeoutMs, ), maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs( codeExecutionContext.codeEnvironmentConfigSchema, diff --git a/packages/api/src/agents/execution.ts b/packages/api/src/agents/execution.ts index 5e55e9ff3b..dc6eef1b77 100644 --- a/packages/api/src/agents/execution.ts +++ b/packages/api/src/agents/execution.ts @@ -41,6 +41,8 @@ export interface CodeExecutionContext { /** Live, server-validated directory selection. Never derive session reuse from this field. */ codeWorkspace?: CodeWorkspaceSelection & { operations: CodeWorkspaceOperation[]; + /** Live Code API execution ceiling. Omitted by older deployments. */ + maxCommandTimeoutMs?: number; instructions?: CodeWorkspaceDescriptor['instructions']; environment?: CodeWorkspaceDescriptor['environment']; }; diff --git a/packages/api/src/agents/handlers.background.spec.ts b/packages/api/src/agents/handlers.background.spec.ts index dc41cb8464..2bac0dacf7 100644 --- a/packages/api/src/agents/handlers.background.spec.ts +++ b/packages/api/src/agents/handlers.background.spec.ts @@ -10,6 +10,7 @@ import { CHECK_BACKGROUND_TASK_NAME, } from './background'; import { BACKGROUND_TASK_ABORT_GRACE_MS, BACKGROUND_TASK_TIMEOUT_MS } from './backgroundCompletion'; +import { BACKGROUND_TOOL_INVOCATION_CONFIG_KEY } from './invocation'; import { ContentFilterError } from '../middleware/contentFilter'; import { createToolExecuteHandler } from './handlers'; @@ -183,6 +184,14 @@ describe('createToolExecuteHandler — background tool calls', () => { }); expect(events.slice(0, 2)).toEqual(['preregister', 'invoke']); + expect(tool.invoke).toHaveBeenCalledWith( + { q: 'continuations' }, + expect.objectContaining({ + configurable: expect.objectContaining({ + [BACKGROUND_TOOL_INVOCATION_CONFIG_KEY]: true, + }), + }), + ); expect(JSON.parse(dispatch.content).message).toContain('host will resume you'); await flushMicrotasks(); expect(events).toEqual(['preregister', 'invoke', 'persist']); diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts index 2e5982bc2d..1f7cb845d5 100644 --- a/packages/api/src/agents/handlers.ts +++ b/packages/api/src/agents/handlers.ts @@ -121,6 +121,7 @@ import { import { buildSkillPrimeMessage, isSkillFilePath, SKILL_FILE_PREFIX } from './skills'; import { resolveCallerCapabilityProjectionSnapshot } from './callerCapabilities'; import { resolveAttachedWorkspaceQueueWaitMs } from '~/code/command'; +import { BACKGROUND_TOOL_INVOCATION_CONFIG_KEY } from './invocation'; import { mergeCodeFilesIntoContext } from './codeFilesSession'; import { toolValidationFeedback } from './validationFeedback'; import { createSkillContentDigest } from './compatibility'; @@ -6010,6 +6011,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand signal: backgroundAbortController.signal, configurable: { ...mergedConfigurable, + [BACKGROUND_TOOL_INVOCATION_CONFIG_KEY]: true, ...(detachedReservation?.status === 'reserved' ? { eventActorDetachedAction: { diff --git a/packages/api/src/agents/initialize.ts b/packages/api/src/agents/initialize.ts index 94f1a15bd3..79ae24e74a 100644 --- a/packages/api/src/agents/initialize.ts +++ b/packages/api/src/agents/initialize.ts @@ -1993,6 +1993,7 @@ export async function initializeAgent( trustedCodeExecutionContext.environmentType === 'attached' ? resolveAttachedWorkspaceCommandTimeoutMax( trustedCodeExecutionContext.codeEnvironmentConfigSchema, + trustedCodeExecutionContext.codeWorkspace?.maxCommandTimeoutMs, ) : undefined; if ( diff --git a/packages/api/src/agents/invocation.ts b/packages/api/src/agents/invocation.ts new file mode 100644 index 0000000000..e5c355d2f9 --- /dev/null +++ b/packages/api/src/agents/invocation.ts @@ -0,0 +1,2 @@ +/** Host-only marker added when an ordinary tool is executing as a detached background task. */ +export const BACKGROUND_TOOL_INVOCATION_CONFIG_KEY = '__librechatBackgroundToolInvocation'; diff --git a/packages/api/src/code/bridge.spec.ts b/packages/api/src/code/bridge.spec.ts index db1af81812..51d237e2bd 100644 --- a/packages/api/src/code/bridge.spec.ts +++ b/packages/api/src/code/bridge.spec.ts @@ -84,6 +84,7 @@ describe('getCodeBridgeWorkerStatus', () => { online: true, ready: true, leaseExpiresInMs: 45_000, + maxCommandTimeoutMs: 120_000, capabilities: { statefulWorkspace: true, sandboxProfile: 'native-srt', @@ -114,6 +115,7 @@ describe('getCodeBridgeWorkerStatus', () => { status: 'ready', statefulWorkspace: true, leaseExpiresInMs: 45_000, + maxCommandTimeoutMs: 120_000, sandboxProfile: 'native-srt', runtimes: ['bash'], operations: ['read_file', 'execute_command'], @@ -172,6 +174,8 @@ describe('getCodeBridgeWorkerStatus', () => { { online: true, ready: false }, { online: false, ready: false, leaseExpiresInMs: 5_000 }, { online: true, ready: true, leaseExpiresInMs: 60_001 }, + { online: true, ready: true, maxCommandTimeoutMs: 0 }, + { online: true, ready: true, maxCommandTimeoutMs: 300_001 }, { online: true, ready: true, diff --git a/packages/api/src/code/bridge.ts b/packages/api/src/code/bridge.ts index b44701804a..2ece18cadf 100644 --- a/packages/api/src/code/bridge.ts +++ b/packages/api/src/code/bridge.ts @@ -1,5 +1,6 @@ import { createHash } from 'node:crypto'; import { + CODE_ENVIRONMENT_COMMAND_TIMEOUT_HARD_MAX_MS, CODE_WORKSPACE_ID_PATTERN, CODE_WORKSPACE_MAX_COUNT, CODE_WORKSPACE_OPERATIONS, @@ -38,6 +39,7 @@ export type CodeBridgeWorkerStatus = { operations?: CodeWorkspaceOperation[]; workspaces?: CodeWorkspaceDescriptor[]; programmaticLanguages?: ['bash']; + maxCommandTimeoutMs?: number; }; export type CodeBridgeFetch = ( @@ -292,6 +294,7 @@ export async function getCodeBridgeWorkerStatus({ online?: unknown; ready?: unknown; leaseExpiresInMs?: unknown; + maxCommandTimeoutMs?: unknown; capabilities?: { statefulWorkspace?: unknown; sandboxProfile?: unknown; @@ -306,6 +309,12 @@ export async function getCodeBridgeWorkerStatus({ Number.isSafeInteger(status.leaseExpiresInMs) && status.leaseExpiresInMs > 0 && status.leaseExpiresInMs <= 60_000); + const validCommandTimeout = + status.maxCommandTimeoutMs == null || + (typeof status.maxCommandTimeoutMs === 'number' && + Number.isSafeInteger(status.maxCommandTimeoutMs) && + status.maxCommandTimeoutMs >= 1 && + status.maxCommandTimeoutMs <= CODE_ENVIRONMENT_COMMAND_TIMEOUT_HARD_MAX_MS); const validCapabilities = capabilities == null || validStatusString(capabilities.sandboxProfile); const validRuntimes = capabilities == null || validStatusStringArray(capabilities.runtimes); @@ -322,6 +331,7 @@ export async function getCodeBridgeWorkerStatus({ (status.online && (status.leaseExpiresInMs == null || capabilities == null)) || (!status.online && (status.leaseExpiresInMs != null || capabilities != null)) || !validLease || + !validCommandTimeout || !validCapabilities || !validRuntimes || !validWorkspaceTools || @@ -366,6 +376,9 @@ export async function getCodeBridgeWorkerStatus({ ...(typeof status.leaseExpiresInMs !== 'number' ? {} : { leaseExpiresInMs: status.leaseExpiresInMs }), + ...(typeof status.maxCommandTimeoutMs !== 'number' + ? {} + : { maxCommandTimeoutMs: status.maxCommandTimeoutMs }), ...(typeof capabilities?.sandboxProfile !== 'string' ? {} : { sandboxProfile: capabilities.sandboxProfile }), diff --git a/packages/api/src/code/capabilities.spec.ts b/packages/api/src/code/capabilities.spec.ts index a473389ca2..94dcdcc200 100644 --- a/packages/api/src/code/capabilities.spec.ts +++ b/packages/api/src/code/capabilities.spec.ts @@ -351,7 +351,11 @@ describe('resolveCodeExecutionWorkspaceContext', () => { delete process.env.TEST_CODE_CAPABILITY_TOKEN; }); - function workspaceStatus(workspaces: unknown[], statefulWorkspace: boolean = true): Response { + function workspaceStatus( + workspaces: unknown[], + statefulWorkspace: boolean = true, + maxCommandTimeoutMs?: number, + ): Response { return new Response( JSON.stringify({ protocolVersion: 1, @@ -359,6 +363,7 @@ describe('resolveCodeExecutionWorkspaceContext', () => { online: true, ready: true, leaseExpiresInMs: 45_000, + maxCommandTimeoutMs, capabilities: { statefulWorkspace, sandboxProfile: 'native-srt', @@ -375,10 +380,14 @@ describe('resolveCodeExecutionWorkspaceContext', () => { it('binds the exact advertised workspace and its operation ceiling', async () => { jest.spyOn(globalThis, 'fetch').mockResolvedValue( - workspaceStatus([ - { id: 'project-a', name: 'Project A' }, - { id: 'docs', operations: ['read_file', 'list_files'] }, - ]), + workspaceStatus( + [ + { id: 'project-a', name: 'Project A' }, + { id: 'docs', operations: ['read_file', 'list_files'] }, + ], + true, + 120_000, + ), ); await expect( @@ -397,6 +406,7 @@ describe('resolveCodeExecutionWorkspaceContext', () => { environmentId: 'personal', workspaceId: 'docs', operations: ['read_file', 'list_files'], + maxCommandTimeoutMs: 120_000, }, }); }); diff --git a/packages/api/src/code/capabilities.ts b/packages/api/src/code/capabilities.ts index 4f76a2797e..7ae391fd3f 100644 --- a/packages/api/src/code/capabilities.ts +++ b/packages/api/src/code/capabilities.ts @@ -178,6 +178,9 @@ export async function resolveCodeExecutionWorkspaceContext({ codeWorkspace: { ...selection, operations: [...(workspace.operations ?? status.operations)], + ...(status.maxCommandTimeoutMs == null + ? {} + : { maxCommandTimeoutMs: status.maxCommandTimeoutMs }), ...(workspace.instructions ? { instructions: workspace.instructions } : {}), ...(workspace.environment ? { environment: workspace.environment } : {}), }, diff --git a/packages/api/src/code/command.spec.ts b/packages/api/src/code/command.spec.ts index 769878928b..583990bd66 100644 --- a/packages/api/src/code/command.spec.ts +++ b/packages/api/src/code/command.spec.ts @@ -8,8 +8,10 @@ import { createAttachedWorkspaceBashTool, createGitIdentityProgrammaticBashTool, resolveAttachedWorkspaceCommandTimeoutMax, + resolveAttachedWorkspaceProgrammaticTimeout, resolveAttachedWorkspaceQueueWaitMs, } from './command'; +import { BACKGROUND_TOOL_INVOCATION_CONFIG_KEY } from '~/agents/invocation'; describe('programmatic Bash Git identity', () => { test('applies authorship before the SDK sends a programmatic script', async () => { @@ -88,6 +90,22 @@ describe('programmatic Bash Git identity', () => { await once(server, 'close'); } }); + + test('preserves the foreground timeout while bounding an explicit admin override', () => { + expect(resolveAttachedWorkspaceProgrammaticTimeout(undefined, 90_000)).toBe(30_000); + expect( + resolveAttachedWorkspaceProgrammaticTimeout( + { limits: { maxCommandTimeoutMs: 120_000 } }, + 90_000, + ), + ).toBe(90_000); + expect( + resolveAttachedWorkspaceProgrammaticTimeout( + { limits: { maxCommandTimeoutMs: 60_000 } }, + 90_000, + ), + ).toBe(60_000); + }); }); function commandResponse(overrides: Record = {}): Response { @@ -264,6 +282,41 @@ describe('createAttachedWorkspaceBashTool', () => { properties: { timeoutMs: { type: 'integer', minimum: 1, maximum: 120_000 } }, }); expect(resolveAttachedWorkspaceCommandTimeoutMax()).toBe(30_000); + expect(resolveAttachedWorkspaceCommandTimeoutMax(undefined, 90_000)).toBe(90_000); + expect( + resolveAttachedWorkspaceCommandTimeoutMax( + { limits: { maxCommandTimeoutMs: 120_000 } }, + 90_000, + ), + ).toBe(90_000); + expect( + resolveAttachedWorkspaceCommandTimeoutMax( + { limits: { maxCommandTimeoutMs: 60_000 } }, + 90_000, + ), + ).toBe(60_000); + }); + + test('uses the negotiated ceiling only for omitted detached background timeouts', async () => { + const fetchImpl: CodeBridgeFetch = jest.fn(async () => commandResponse()); + const bashTool = createAttachedWorkspaceBashTool({ + baseUrl: 'https://code.example.com/v1', + authHeaders: () => ({}), + workspaceId: 'project-a', + maxTimeoutMs: 90_000, + fetchImpl, + }); + + await bashTool.invoke( + { command: 'npm test' }, + { configurable: { [BACKGROUND_TOOL_INVOCATION_CONFIG_KEY]: true } }, + ); + await bashTool.invoke({ command: 'npm test' }); + + const backgroundRequest = JSON.parse(String((fetchImpl as jest.Mock).mock.calls[0][1]?.body)); + const foregroundRequest = JSON.parse(String((fetchImpl as jest.Mock).mock.calls[1][1]?.body)); + expect(backgroundRequest).toMatchObject({ timeoutMs: 90_000 }); + expect(foregroundRequest).toMatchObject({ timeoutMs: 30_000 }); }); test('resolves the administrator-configured admission budget', () => { diff --git a/packages/api/src/code/command.ts b/packages/api/src/code/command.ts index 6b1f90d661..5d7a5ba0c4 100644 --- a/packages/api/src/code/command.ts +++ b/packages/api/src/code/command.ts @@ -21,6 +21,7 @@ import { WORKSPACE_COMMAND_MAX_TIMEOUT_MS, WORKSPACE_QUEUE_MAX_WAIT_MS, } from './workspace'; +import { BACKGROUND_TOOL_INVOCATION_CONFIG_KEY } from '~/agents/invocation'; const DEFAULT_OUTPUT_BYTES = 256 * 1024; @@ -74,7 +75,7 @@ function buildAttachedTimeoutSchema(maxTimeoutMs: number): BoundedTimeoutSchema type: 'integer', minimum: 1, maximum: maxTimeoutMs, - description: `Optional execution timeout in milliseconds, from 1 through ${maxTimeoutMs}. Defaults to ${defaultTimeoutMs}. Waiting for an available worker does not consume this execution budget.`, + description: `Optional execution timeout in milliseconds, from 1 through ${maxTimeoutMs}. Defaults to ${defaultTimeoutMs} for foreground calls and ${maxTimeoutMs} for detached background calls. Waiting for an available worker does not consume this execution budget.`, }; } @@ -87,11 +88,41 @@ function normalizeAttachedWorkspaceCommandTimeoutMax(maxTimeoutMs: number): numb export function resolveAttachedWorkspaceCommandTimeoutMax( configSchema?: CodeEnvironmentUserConfigSchema, + upstreamMaxTimeoutMs?: number, ): number { const configured = configSchema?.limits?.maxCommandTimeoutMs; - return configured == null - ? WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS - : normalizeAttachedWorkspaceCommandTimeoutMax(configured); + const upstream = + upstreamMaxTimeoutMs == null + ? WORKSPACE_COMMAND_MAX_TIMEOUT_MS + : normalizeAttachedWorkspaceCommandTimeoutMax(upstreamMaxTimeoutMs); + let requested = WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS; + if (configured != null) { + requested = normalizeAttachedWorkspaceCommandTimeoutMax(configured); + } else if (upstreamMaxTimeoutMs != null) { + requested = upstream; + } + return Math.min(requested, upstream); +} + +/** + * Programmatic calls do not currently carry the detached-invocation marker. + * Preserve their historical foreground default while still enforcing both an + * explicit administrator override and the live upstream ceiling. + */ +export function resolveAttachedWorkspaceProgrammaticTimeout( + configSchema?: CodeEnvironmentUserConfigSchema, + upstreamMaxTimeoutMs?: number, +): number { + const configured = configSchema?.limits?.maxCommandTimeoutMs; + const requested = + configured == null + ? WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS + : normalizeAttachedWorkspaceCommandTimeoutMax(configured); + const upstream = + upstreamMaxTimeoutMs == null + ? WORKSPACE_COMMAND_MAX_TIMEOUT_MS + : normalizeAttachedWorkspaceCommandTimeoutMax(upstreamMaxTimeoutMs); + return Math.min(requested, upstream); } /** @@ -207,8 +238,9 @@ export function createContextProgrammaticBashTool( ...(attached ? { workspaceId: context.codeWorkspace?.workspaceId, - runTimeoutMs: resolveAttachedWorkspaceCommandTimeoutMax( + runTimeoutMs: resolveAttachedWorkspaceProgrammaticTimeout( context.codeEnvironmentConfigSchema, + context.codeWorkspace?.maxCommandTimeoutMs, ), } : {}), @@ -259,7 +291,7 @@ export function createAttachedWorkspaceBashTool({ workspaceId: string; environment?: CodeWorkspaceDescriptor['environment']; gitIdentity?: AgentGitIdentity | null; - /** Deployment ceiling already intersected with the protocol hard cap. */ + /** Effective admin/upstream ceiling already intersected with the protocol hard cap. */ maxTimeoutMs?: number; /** Deployment admission budget; omitted keeps the built-in default. */ maxQueueWaitMs?: number; @@ -305,7 +337,10 @@ export function createAttachedWorkspaceBashTool({ action ?? commandWithGitIdentity(commandWithArguments(rawInput.command!, rawInput.args), gitIdentity); const timeoutMs = - rawInput.timeoutMs ?? Math.min(WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, effectiveMaxTimeoutMs); + rawInput.timeoutMs ?? + (config?.configurable?.[BACKGROUND_TOOL_INVOCATION_CONFIG_KEY] === true + ? effectiveMaxTimeoutMs + : Math.min(WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, effectiveMaxTimeoutMs)); const signal = config?.signal; const trace = { runId: config?.metadata?.run_id,