From bf91d2b59c9f5a656b6b86d4cf42d2a16e4e1c8c Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Sun, 27 Sep 2026 20:19:54 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=A5=A7=20fix:=20Reserve=20Execution=20Tim?= =?UTF-8?q?e=20in=20Workspace=20Admission=20(#16421)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🥧 fix: Reserve Execution Time in Workspace Admission * 🥧 fix: Pass Workspace Admission Static Checks --------- Co-authored-by: Lia --- api/server/services/Files/Code/process.js | 24 ++ .../services/Files/Code/process.spec.js | 18 + api/server/services/ToolService.js | 4 + librechat.example.yaml | 11 + packages/api/src/agents/handlers.spec.ts | 30 +- packages/api/src/agents/handlers.ts | 51 ++- packages/api/src/code/command.spec.ts | 34 +- packages/api/src/code/command.ts | 22 +- packages/api/src/code/workspace.spec.ts | 312 ++++++++++++++++++ packages/api/src/code/workspace.ts | 148 +++++++-- packages/data-provider/src/config.spec.ts | 33 +- packages/data-provider/src/config.ts | 16 + 12 files changed, 656 insertions(+), 47 deletions(-) diff --git a/api/server/services/Files/Code/process.js b/api/server/services/Files/Code/process.js index 10348d71af..6371b1c40f 100644 --- a/api/server/services/Files/Code/process.js +++ b/api/server/services/Files/Code/process.js @@ -1088,10 +1088,14 @@ async function readWorkspaceFile({ req, signal, maxQueueWaitMs, + maxRequestTimeoutMs, + deadlineAtMs, }) { return executeWorkspaceTool({ baseURL: codeApiBaseUrl, maxQueueWaitMs, + maxRequestTimeoutMs, + deadlineAtMs, /** Minted per admission attempt: a queued call outlives one token TTL. */ authHeaders: async () => ({ ...(await getCodeApiAuthHeaders(req, bridgeWorkerId)), @@ -1137,10 +1141,14 @@ async function searchWorkspace({ req, signal, maxQueueWaitMs, + maxRequestTimeoutMs, + deadlineAtMs, }) { return executeWorkspaceTool({ baseURL: codeApiBaseUrl, maxQueueWaitMs, + maxRequestTimeoutMs, + deadlineAtMs, /** Minted per admission attempt: a queued call outlives one token TTL. */ authHeaders: async () => ({ ...(await getCodeApiAuthHeaders(req, bridgeWorkerId)), @@ -1186,10 +1194,14 @@ async function listWorkspaceFiles({ req, signal, maxQueueWaitMs, + maxRequestTimeoutMs, + deadlineAtMs, }) { return executeWorkspaceTool({ baseURL: codeApiBaseUrl, maxQueueWaitMs, + maxRequestTimeoutMs, + deadlineAtMs, /** Minted per admission attempt: a queued call outlives one token TTL. */ authHeaders: async () => ({ ...(await getCodeApiAuthHeaders(req, bridgeWorkerId)), @@ -1221,10 +1233,14 @@ async function writeWorkspaceFile({ req, signal, maxQueueWaitMs, + maxRequestTimeoutMs, + deadlineAtMs, }) { return executeWorkspaceTool({ baseURL: codeApiBaseUrl, maxQueueWaitMs, + maxRequestTimeoutMs, + deadlineAtMs, /** Minted per admission attempt: a queued call outlives one token TTL. */ authHeaders: async () => ({ ...(await getCodeApiAuthHeaders(req, bridgeWorkerId)), @@ -1256,10 +1272,14 @@ async function editWorkspaceFile({ req, signal, maxQueueWaitMs, + maxRequestTimeoutMs, + deadlineAtMs, }) { return executeWorkspaceTool({ baseURL: codeApiBaseUrl, maxQueueWaitMs, + maxRequestTimeoutMs, + deadlineAtMs, /** Minted per admission attempt: a queued call outlives one token TTL. */ authHeaders: async () => ({ ...(await getCodeApiAuthHeaders(req, bridgeWorkerId)), @@ -1290,10 +1310,14 @@ async function previewWorkspaceEdit({ req, signal, maxQueueWaitMs, + maxRequestTimeoutMs, + deadlineAtMs, }) { return executeWorkspaceTool({ baseURL: codeApiBaseUrl, maxQueueWaitMs, + maxRequestTimeoutMs, + deadlineAtMs, /** Minted per admission attempt: a queued call outlives one token TTL. */ authHeaders: async () => ({ ...(await getCodeApiAuthHeaders(req, bridgeWorkerId)), diff --git a/api/server/services/Files/Code/process.spec.js b/api/server/services/Files/Code/process.spec.js index 4714f570a1..875368196f 100644 --- a/api/server/services/Files/Code/process.spec.js +++ b/api/server/services/Files/Code/process.spec.js @@ -2061,6 +2061,8 @@ describe('Code Process', () => { req: mockReq, signal: controller.signal, maxQueueWaitMs: 0, + maxRequestTimeoutMs: 125_000, + deadlineAtMs: 160_000, }), ).resolves.toBe(result); @@ -2081,6 +2083,8 @@ describe('Code Process', () => { baseURL: 'https://attached-code.example.com/v1', authHeaders: expect.any(Function), maxQueueWaitMs: 0, + maxRequestTimeoutMs: 125_000, + deadlineAtMs: 160_000, request: { protocolVersion: 1, operation: 'read_file', @@ -2140,6 +2144,8 @@ describe('Code Process', () => { baseURL: 'https://attached-code.example.com/v1', authHeaders: expect.any(Function), maxQueueWaitMs: 0, + maxRequestTimeoutMs: undefined, + deadlineAtMs: undefined, request: { protocolVersion: 1, operation: 'search_text', @@ -2198,6 +2204,8 @@ describe('Code Process', () => { baseURL: 'https://attached-code.example.com/v1', authHeaders: expect.any(Function), maxQueueWaitMs: 0, + maxRequestTimeoutMs: undefined, + deadlineAtMs: undefined, request: { protocolVersion: 1, operation: 'list_files', @@ -2258,6 +2266,8 @@ describe('Code Process', () => { baseURL: 'https://attached-code.example.com/v1', authHeaders: expect.any(Function), maxQueueWaitMs: 0, + maxRequestTimeoutMs: undefined, + deadlineAtMs: undefined, request: { protocolVersion: 1, operation: 'write_file', @@ -2296,11 +2306,15 @@ describe('Code Process', () => { bridgeWorkerId: 'worker-user-1', req: mockReq, expected_base_sha256: 'a'.repeat(64), + maxRequestTimeoutMs: 125_000, + deadlineAtMs: 160_000, }), ).resolves.toBe(result); expect(mockExecuteWorkspaceTool).toHaveBeenCalledWith( expect.objectContaining({ + maxRequestTimeoutMs: 125_000, + deadlineAtMs: 160_000, request: { protocolVersion: 1, operation: 'edit_file', @@ -2338,11 +2352,15 @@ describe('Code Process', () => { codeApiBaseUrl: 'https://attached-code.example.com/v1', executionProfile: 'stateful', req: mockReq, + maxRequestTimeoutMs: 125_000, + deadlineAtMs: 160_000, }), ).resolves.toBe(result); expect(mockExecuteWorkspaceTool).toHaveBeenCalledWith( expect.objectContaining({ + maxRequestTimeoutMs: 125_000, + deadlineAtMs: 160_000, request: { protocolVersion: 1, operation: 'preview_edit', diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.js index 62a1cb0420..a64713f75f 100644 --- a/api/server/services/ToolService.js +++ b/api/server/services/ToolService.js @@ -53,6 +53,7 @@ const { createRepositoryInstructionLoader, resolveAttachedWorkspaceCommandTimeoutMax, resolveAttachedWorkspaceQueueWaitMs, + resolveAttachedWorkspaceRequestTimeoutMs, createContextProgrammaticBashTool, resolveCodeExecutionContext, resolveCodeExecutionWorkspaceContext, @@ -2308,6 +2309,9 @@ async function loadToolsForExecution({ maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs( codeExecutionContext.codeEnvironmentConfigSchema, ), + maxRequestTimeoutMs: resolveAttachedWorkspaceRequestTimeoutMs( + codeExecutionContext.codeEnvironmentConfigSchema, + ), }) : createBashExecutionTool({ authHeaders, diff --git a/librechat.example.yaml b/librechat.example.yaml index 7836c55405..2f8caedec6 100644 --- a/librechat.example.yaml +++ b/librechat.example.yaml @@ -720,6 +720,17 @@ endpoints: # # Maximum Bash execution budget the model may request per call. # # Omit to preserve the historical 30-second limit. Hard cap: 300000. # maxCommandTimeoutMs: 120000 + # # Optional total HTTP budget per workspace tool call, including + # # admission, retries, worst-case execution, settlement, and delivery. + # # Do NOT enable until every LibreChat API replica recognizes this + # # strict config field, Code API honors X-LibreChat-Workspace-Queue-Wait-Ms, + # # and a non-mutating timed canary verifies the live CDN, load balancer, + # # ingress, and caller can all outlast this value. The Nginx example + # # (120s) cannot establish the live limit. Omission keeps 30s admission. + # # With 125s total, a read can queue <=90s, a default command <=85s, + # # a 90s command <=25s. Commands >115s cannot fit; the 120s command + # # ceiling above requires a larger VERIFIED total budget to be usable. + # # maxRequestTimeoutMs: 125000 # permissions: # fileWrite: # allowed: [allow, ask, deny] diff --git a/packages/api/src/agents/handlers.spec.ts b/packages/api/src/agents/handlers.spec.ts index c79e694fd6..afb806d0f8 100644 --- a/packages/api/src/agents/handlers.spec.ts +++ b/packages/api/src/agents/handlers.spec.ts @@ -5610,7 +5610,9 @@ describe('createToolExecuteHandler', () => { workspaceInstanceId: 'a'.repeat(64), operations: TEST_ATTACHED_WORKSPACE_OPERATIONS, }, - codeEnvironmentConfigSchema: { limits: { maxQueueWaitMs: 0 } }, + codeEnvironmentConfigSchema: { + limits: { maxQueueWaitMs: 0, maxRequestTimeoutMs: 125_000 }, + }, bridgeWorkerId: 'user-worker', }, }, @@ -5642,6 +5644,8 @@ describe('createToolExecuteHandler', () => { workspace_id: 'project-a', workspace_instance_id: 'a'.repeat(64), maxQueueWaitMs: 0, + maxRequestTimeoutMs: 125_000, + deadlineAtMs: expect.any(Number), codeApiBaseUrl: 'https://code.example.com', executionProfile: 'stateful', bridgeWorkerId: 'user-worker', @@ -5845,7 +5849,8 @@ describe('createToolExecuteHandler', () => { ])( 'shares a protected edit retry horizon ($budget ms, preview $elapsed ms)', async ({ budget, elapsed, remaining }) => { - let nowMs = Date.now(); + const startedAt = Date.now(); + let nowMs = startedAt; jest.spyOn(Date, 'now').mockImplementation(() => nowMs); const previewWorkspaceEdit = jest.fn(async () => { nowMs += elapsed; @@ -5899,7 +5904,9 @@ describe('createToolExecuteHandler', () => { executionProfile: 'stateful', statefulSessions: true, environmentType: 'attached', - codeEnvironmentConfigSchema: { limits: { maxQueueWaitMs: budget } }, + codeEnvironmentConfigSchema: { + limits: { maxQueueWaitMs: budget, maxRequestTimeoutMs: 125_000 }, + }, bridgeWorkerId: 'user-worker', }, }, @@ -5918,7 +5925,11 @@ describe('createToolExecuteHandler', () => { ]); expect(previewWorkspaceEdit).toHaveBeenCalledWith( - expect.objectContaining({ maxQueueWaitMs: budget }), + expect.objectContaining({ + maxQueueWaitMs: budget, + maxRequestTimeoutMs: 125_000, + deadlineAtMs: startedAt + 125_000, + }), ); if (remaining == null) { expect(result.status).toBe('error'); @@ -5931,6 +5942,8 @@ describe('createToolExecuteHandler', () => { expect.objectContaining({ expected_base_sha256: 'b'.repeat(64), maxQueueWaitMs: remaining, + maxRequestTimeoutMs: 125_000, + deadlineAtMs: startedAt + 125_000, }), ); }, @@ -6626,7 +6639,9 @@ describe('createToolExecuteHandler', () => { workspaceInstanceId: 'b'.repeat(64), operations: TEST_ATTACHED_WORKSPACE_OPERATIONS, }, - codeEnvironmentConfigSchema: { limits: { maxQueueWaitMs: 0 } }, + codeEnvironmentConfigSchema: { + limits: { maxQueueWaitMs: 0, maxRequestTimeoutMs: 125_000 }, + }, bridgeWorkerId: 'personal-worker-1', statefulSessions: true, }, @@ -6647,6 +6662,7 @@ describe('createToolExecuteHandler', () => { workspace_id: 'project-a', workspace_instance_id: 'b'.repeat(64), maxQueueWaitMs: 0, + maxRequestTimeoutMs: 125_000, start_line: 1, max_lines: 200, codeApiBaseUrl: 'https://code.example.com/v1', @@ -6930,6 +6946,7 @@ describe('createToolExecuteHandler', () => { executionProfile: 'stateful', environmentType: 'attached', bridgeWorkerId: 'personal-worker-1', + codeEnvironmentConfigSchema: { limits: { maxRequestTimeoutMs: 125_000 } }, statefulSessions: true, }, searchWorkspace, @@ -6954,6 +6971,7 @@ describe('createToolExecuteHandler', () => { query: 'needle', workspace_id: 'project-a', maxQueueWaitMs: 300000, + maxRequestTimeoutMs: 125_000, path: 'src', max_results: 20, codeApiBaseUrl: 'https://code.example.com/v1', @@ -7084,6 +7102,7 @@ describe('createToolExecuteHandler', () => { executionProfile: 'stateful', environmentType: 'attached', bridgeWorkerId: 'personal-worker-1', + codeEnvironmentConfigSchema: { limits: { maxRequestTimeoutMs: 125_000 } }, statefulSessions: true, }, listWorkspaceFiles, @@ -7107,6 +7126,7 @@ describe('createToolExecuteHandler', () => { expect(listWorkspaceFiles).toHaveBeenCalledWith({ workspace_id: 'project-a', maxQueueWaitMs: 300000, + maxRequestTimeoutMs: 125_000, path: 'src', after_path: 'src/app.ts', max_results: 20, diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts index 4438b0d0c4..7eb99e4e4d 100644 --- a/packages/api/src/agents/handlers.ts +++ b/packages/api/src/agents/handlers.ts @@ -119,6 +119,10 @@ import { WORKSPACE_EDIT_MAX_COUNT, WORKSPACE_WRITE_MAX_BYTES, } from '~/code/workspace'; +import { + resolveAttachedWorkspaceQueueWaitMs, + resolveAttachedWorkspaceRequestTimeoutMs, +} from '~/code/command'; import { hasIntentArg, stripIntentArg, @@ -127,7 +131,6 @@ import { } from './intent'; import { buildSkillPrimeMessage, isSkillFilePath, SKILL_FILE_PREFIX } from './skills'; import { resolveCallerCapabilityProjectionSnapshot } from './callerCapabilities'; -import { resolveAttachedWorkspaceQueueWaitMs } from '~/code/command'; import { BACKGROUND_TOOL_INVOCATION_CONFIG_KEY } from './invocation'; import { mergeCodeFilesIntoContext } from './codeFilesSession'; import { toolValidationFeedback } from './validationFeedback'; @@ -568,6 +571,8 @@ export interface ToolExecuteOptions { req?: ServerRequest; signal?: AbortSignal; maxQueueWaitMs?: number; + maxRequestTimeoutMs?: number; + deadlineAtMs?: number; }) => Promise; /** Searches literal text within an attached worker's logical workspace. */ searchWorkspace?: (params: { @@ -582,6 +587,8 @@ export interface ToolExecuteOptions { req?: ServerRequest; signal?: AbortSignal; maxQueueWaitMs?: number; + maxRequestTimeoutMs?: number; + deadlineAtMs?: number; }) => Promise; /** Lists relative file paths within an attached worker's logical workspace. */ listWorkspaceFiles?: (params: { @@ -596,6 +603,8 @@ export interface ToolExecuteOptions { req?: ServerRequest; signal?: AbortSignal; maxQueueWaitMs?: number; + maxRequestTimeoutMs?: number; + deadlineAtMs?: number; }) => Promise; /** Writes a UTF-8 file within an attached worker's logical workspace. */ writeWorkspaceFile?: (params: { @@ -610,6 +619,8 @@ export interface ToolExecuteOptions { req?: ServerRequest; signal?: AbortSignal; maxQueueWaitMs?: number; + maxRequestTimeoutMs?: number; + deadlineAtMs?: number; }) => Promise; /** Previews exact replacements without mutating an attached worker workspace. */ previewWorkspaceEdit?: (params: { @@ -623,6 +634,8 @@ export interface ToolExecuteOptions { req?: ServerRequest; signal?: AbortSignal; maxQueueWaitMs?: number; + maxRequestTimeoutMs?: number; + deadlineAtMs?: number; }) => Promise; /** Applies exact replacements atomically within an attached worker workspace. */ editWorkspaceFile?: (params: { @@ -637,6 +650,8 @@ export interface ToolExecuteOptions { req?: ServerRequest; signal?: AbortSignal; maxQueueWaitMs?: number; + maxRequestTimeoutMs?: number; + deadlineAtMs?: number; }) => Promise; /** * Reads a code-execution sandbox file by shelling `cat` through the @@ -2462,9 +2477,7 @@ async function handleWorkspaceFileRead( start_line: startLine, max_lines: maxLines, codeApiBaseUrl: codeExecutionContext.baseUrl, - maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs( - codeExecutionContext.codeEnvironmentConfigSchema, - ), + ...attachedWorkspaceRequestLimits(codeExecutionContext), executionProfile: codeExecutionContext.executionProfile, ...(codeExecutionContext.bridgeWorkerId ? { bridgeWorkerId: codeExecutionContext.bridgeWorkerId } @@ -2570,9 +2583,7 @@ async function handleWorkspaceSearchCall( ...(typeof args.path === 'string' && args.path.length > 0 ? { path: args.path } : {}), max_results: Number(maxResults), codeApiBaseUrl: codeExecutionContext.baseUrl, - maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs( - codeExecutionContext.codeEnvironmentConfigSchema, - ), + ...attachedWorkspaceRequestLimits(codeExecutionContext), executionProfile: codeExecutionContext.executionProfile, ...(codeExecutionContext.bridgeWorkerId ? { bridgeWorkerId: codeExecutionContext.bridgeWorkerId } @@ -2663,9 +2674,7 @@ async function handleWorkspaceListCall( : {}), max_results: Number(maxResults), codeApiBaseUrl: codeExecutionContext.baseUrl, - maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs( - codeExecutionContext.codeEnvironmentConfigSchema, - ), + ...attachedWorkspaceRequestLimits(codeExecutionContext), executionProfile: codeExecutionContext.executionProfile, ...(codeExecutionContext.bridgeWorkerId ? { bridgeWorkerId: codeExecutionContext.bridgeWorkerId } @@ -3910,6 +3919,18 @@ function attachedWorkspaceAuthoringPath( return pathError ? errorResult(tc, pathError) : { filePath: relativePath }; } +function attachedWorkspaceRequestLimits(codeExecutionContext: CodeExecutionContext): { + maxQueueWaitMs: number; + maxRequestTimeoutMs?: number; +} { + const config = codeExecutionContext.codeEnvironmentConfigSchema; + const maxRequestTimeoutMs = resolveAttachedWorkspaceRequestTimeoutMs(config); + return { + maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs(config), + ...(maxRequestTimeoutMs == null ? {} : { maxRequestTimeoutMs }), + }; +} + function attachedWorkspaceMutationParams( codeExecutionContext: CodeExecutionContext, workspaceId: string, @@ -3924,16 +3945,20 @@ function attachedWorkspaceMutationParams( req?: ServerRequest; signal?: AbortSignal; maxQueueWaitMs: number; + maxRequestTimeoutMs?: number; + deadlineAtMs?: number; } { + const limits = attachedWorkspaceRequestLimits(codeExecutionContext); return { workspace_id: workspaceId, ...(codeExecutionContext.codeWorkspace?.workspaceInstanceId ? { workspace_instance_id: codeExecutionContext.codeWorkspace.workspaceInstanceId } : {}), codeApiBaseUrl: codeExecutionContext.baseUrl, - maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs( - codeExecutionContext.codeEnvironmentConfigSchema, - ), + ...limits, + ...(limits.maxRequestTimeoutMs == null + ? {} + : { deadlineAtMs: Date.now() + limits.maxRequestTimeoutMs }), executionProfile: codeExecutionContext.executionProfile, ...(codeExecutionContext.bridgeWorkerId ? { bridgeWorkerId: codeExecutionContext.bridgeWorkerId } diff --git a/packages/api/src/code/command.spec.ts b/packages/api/src/code/command.spec.ts index 0cd2dd12f8..4ba032bbde 100644 --- a/packages/api/src/code/command.spec.ts +++ b/packages/api/src/code/command.spec.ts @@ -11,6 +11,7 @@ import { resolveAttachedWorkspaceCommandTimeoutMax, resolveAttachedWorkspaceProgrammaticTimeout, resolveAttachedWorkspaceQueueWaitMs, + resolveAttachedWorkspaceRequestTimeoutMs, } from './command'; import { BACKGROUND_TOOL_INVOCATION_CONFIG_KEY } from '~/agents/invocation'; @@ -376,7 +377,7 @@ describe('createAttachedWorkspaceBashTool', () => { expect(foregroundRequest).toMatchObject({ timeoutMs: 30_000 }); }); - test('resolves the administrator-configured admission budget', () => { + test('resolves the administrator-configured retry horizon', () => { expect(resolveAttachedWorkspaceQueueWaitMs()).toBe(5 * 60_000); expect(resolveAttachedWorkspaceQueueWaitMs({ limits: { maxQueueWaitMs: 30_000 } })).toBe( 30_000, @@ -387,6 +388,37 @@ describe('createAttachedWorkspaceBashTool', () => { ); }); + test('opts Bash into a verified HTTP limit without changing execution or disabling the initial attempt', async () => { + jest.spyOn(Date, 'now').mockReturnValue(1_000); + expect(resolveAttachedWorkspaceRequestTimeoutMs()).toBeUndefined(); + expect( + resolveAttachedWorkspaceRequestTimeoutMs({ limits: { maxRequestTimeoutMs: 125_000 } }), + ).toBe(125_000); + expect( + resolveAttachedWorkspaceRequestTimeoutMs({ limits: { maxRequestTimeoutMs: 610_001 } }), + ).toBeUndefined(); + const fetchImpl: CodeBridgeFetch = jest.fn(async () => commandResponse()); + const bashTool = createAttachedWorkspaceBashTool({ + baseUrl: 'https://code.example.com/v1', + authHeaders: () => ({}), + workspaceId: 'project-a', + maxTimeoutMs: 90_000, + maxQueueWaitMs: 0, + maxRequestTimeoutMs: 125_000, + fetchImpl, + }); + + await bashTool.func({ command: 'sleep 90', timeoutMs: 90_000 }, undefined, {}); + await bashTool.func({ command: 'pwd' }, undefined, {}); + + const [, long] = (fetchImpl as jest.Mock).mock.calls[0]; + const [, short] = (fetchImpl as jest.Mock).mock.calls[1]; + expect(JSON.parse(long.body).timeoutMs).toBe(90_000); + expect(JSON.parse(short.body).timeoutMs).toBe(30_000); + expect(long.headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe('25000'); + expect(short.headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe('85000'); + }); + test('lowers the omitted timeout when the deployment ceiling is below 30 seconds', async () => { const fetchImpl: CodeBridgeFetch = jest.fn(async () => commandResponse()); const bashTool = createAttachedWorkspaceBashTool({ diff --git a/packages/api/src/code/command.ts b/packages/api/src/code/command.ts index e3ea3b3fd2..2ec67f9885 100644 --- a/packages/api/src/code/command.ts +++ b/packages/api/src/code/command.ts @@ -1,5 +1,6 @@ import { logger } from '@librechat/data-schemas'; import { tool } from '@librechat/agents/langchain/tools'; +import { CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS } from 'librechat-data-provider'; import { BashExecutionToolDefinition, BashToolOutputReferencesGuide, @@ -140,6 +141,21 @@ export function resolveAttachedWorkspaceQueueWaitMs( return Math.min(WORKSPACE_QUEUE_MAX_WAIT_MS, configured); } +export function resolveAttachedWorkspaceRequestTimeoutMs( + configSchema?: CodeEnvironmentUserConfigSchema, +): number | undefined { + const configured = configSchema?.limits?.maxRequestTimeoutMs; + if ( + configured == null || + !Number.isSafeInteger(configured) || + configured < 1 || + configured > CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS + ) { + return undefined; + } + return configured; +} + export function buildAttachedWorkspaceBashSchema( maxTimeoutMs: number = WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, environment?: CodeWorkspaceDescriptor['environment'], @@ -286,6 +302,7 @@ export function createAttachedWorkspaceBashTool({ gitIdentity, maxTimeoutMs = WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, maxQueueWaitMs, + maxRequestTimeoutMs, fetchImpl, }: { baseUrl: string; @@ -296,8 +313,10 @@ export function createAttachedWorkspaceBashTool({ gitIdentity?: AgentGitIdentity | null; /** Effective admin/upstream ceiling already intersected with the protocol hard cap. */ maxTimeoutMs?: number; - /** Deployment admission budget; omitted keeps the built-in default. */ + /** Retry horizon across typed queue expirations, not an admission budget. */ maxQueueWaitMs?: number; + /** Verified total HTTP budget; omission keeps the legacy per-attempt timeout. */ + maxRequestTimeoutMs?: number; fetchImpl?: CodeBridgeFetch; }): DynamicStructuredTool { const effectiveMaxTimeoutMs = normalizeAttachedWorkspaceCommandTimeoutMax(maxTimeoutMs); @@ -376,6 +395,7 @@ export function createAttachedWorkspaceBashTool({ signal, fetchImpl, ...(maxQueueWaitMs == null ? {} : { maxQueueWaitMs }), + ...(maxRequestTimeoutMs == null ? {} : { maxRequestTimeoutMs }), }); if (result.operation !== 'execute_command') { throw new Error('Attached workspace returned an unexpected command result.'); diff --git a/packages/api/src/code/workspace.spec.ts b/packages/api/src/code/workspace.spec.ts index 84b02735a7..149f5bd4da 100644 --- a/packages/api/src/code/workspace.spec.ts +++ b/packages/api/src/code/workspace.spec.ts @@ -242,6 +242,37 @@ describe('workspace admission feedback', () => { }, ); + test('cancels a stalled credential refresh without opening a workspace admission window', async () => { + const controller = new AbortController(); + const reason = new DOMException('Chat stopped', 'AbortError'); + const fetchImpl = jest.fn(); + let entered!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + const operation = executeWorkspaceTool({ + baseURL: 'https://code.example.com/v1', + authHeaders: async () => { + entered(); + return new Promise>(() => {}); + }, + request: { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'src/app.ts', + edits: [{ oldText: 'a', newText: 'b' }], + }, + signal: controller.signal, + maxRequestTimeoutMs: 125_000, + fetchImpl, + }); + await started; + controller.abort(reason); + await expect(operation).rejects.toBe(reason); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + test('does not dispatch a retry if credential refresh spends the remaining queue budget', async () => { const now = jest.spyOn(Date, 'now').mockReturnValue(1000); const authHeaders = jest @@ -438,6 +469,287 @@ describe('executeWorkspaceTool', () => { ).rejects.toMatchObject({ upstreamStatus: 504 }); expect(timeout).toHaveBeenCalledWith(budget); expect(fetchImpl.mock.calls[0][1].body).toBe(JSON.stringify(request)); + const admissionMs = Number( + fetchImpl.mock.calls[0][1].headers['X-LibreChat-Workspace-Queue-Wait-Ms'], + ); + expect(admissionMs).toBeGreaterThan(0); + expect(admissionMs).toBeLessThanOrEqual(30_000); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + test.each<[WorkspaceToolRequest, number, number]>([ + [ + { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, + 125_000, + 90_000, + ], + [ + { protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'pwd' }, + 125_000, + 85_000, + ], + [ + { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: 'pwd', + timeoutMs: 90_000, + }, + 125_000, + 25_000, + ], + [ + { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, + 60_000, + 25_000, + ], + [ + { protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'pwd' }, + 60_000, + 20_000, + ], + [ + { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: 'pwd', + timeoutMs: 300_000, + }, + 610_000, + 300_000, + ], + ])( + 'advertises only the remaining queue time for %j under a %i ms limit', + async (request, maxRequestTimeoutMs, allowanceMs) => { + jest.spyOn(Date, 'now').mockReturnValue(1_000); + const timeout = jest.spyOn(AbortSignal, 'timeout'); + const fetchImpl = jest + .fn() + .mockResolvedValue( + new Response(JSON.stringify({ code: 'ASSIGNMENT_EXPIRED' }), { status: 504 }), + ); + await expect( + executeWorkspaceTool({ + baseURL: 'https://code.example.com/v1', + authHeaders: {}, + request, + maxRequestTimeoutMs, + fetchImpl, + }), + ).rejects.toMatchObject({ upstreamStatus: 504 }); + expect(timeout).toHaveBeenCalledWith(maxRequestTimeoutMs); + expect(fetchImpl.mock.calls[0][1].headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe( + String(allowanceMs), + ); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }, + ); + + test('accepts a result after 31 simulated seconds of admission with an explicit longer budget', async () => { + const now = jest.spyOn(Date, 'now').mockReturnValue(1_000); + const fetchImpl: CodeBridgeFetch = jest.fn(async (_url, init) => { + expect((init?.headers as Record)['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe( + '90000', + ); + now.mockReturnValue(32_000); + expect(init?.signal?.aborted).toBe(false); + return Response.json({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + content: 'ready', + startLine: 1, + endLine: 1, + truncated: false, + }); + }); + await expect( + executeWorkspaceTool({ + baseURL: 'https://code.example.com/v1', + authHeaders: {}, + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + }, + maxRequestTimeoutMs: 125_000, + fetchImpl, + }), + ).resolves.toMatchObject({ content: 'ready' }); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + test('uses the earlier caller deadline rather than assuming the abort signal reveals it', async () => { + jest.spyOn(Date, 'now').mockReturnValue(1_000); + const timeout = jest.spyOn(AbortSignal, 'timeout'); + const fetchImpl = jest.fn().mockResolvedValue(new Response('{}', { status: 504 })); + await expect( + executeWorkspaceTool({ + baseURL: 'https://code.example.com/v1', + authHeaders: {}, + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + }, + maxRequestTimeoutMs: 125_000, + deadlineAtMs: 61_000, + fetchImpl, + }), + ).rejects.toMatchObject({ upstreamStatus: 504 }); + expect(timeout).toHaveBeenCalledWith(60_000); + expect(fetchImpl.mock.calls[0][1].headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe('25000'); + }); + + test('subtracts credential acquisition from the allowance before dispatching', async () => { + const now = jest.spyOn(Date, 'now').mockReturnValue(1_000); + const fetchImpl = jest.fn().mockResolvedValue(new Response('{}', { status: 504 })); + await expect( + executeWorkspaceTool({ + baseURL: 'https://code.example.com/v1', + authHeaders: async () => { + now.mockReturnValue(20_000); + return {}; + }, + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + }, + maxRequestTimeoutMs: 125_000, + fetchImpl, + }), + ).rejects.toMatchObject({ upstreamStatus: 504 }); + expect(fetchImpl.mock.calls[0][1].headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe('71000'); + }); + + test.each<[WorkspaceToolRequest, number]>([ + [ + { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: 'pwd', + timeoutMs: 300_000, + }, + 125_000, + ], + [ + { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: 'pwd', + timeoutMs: 90_000, + }, + 60_000, + ], + ])( + 'does not dispatch when %j cannot fit within the %i ms ingress limit', + async (request, maxRequestTimeoutMs) => { + const fetchImpl = jest.fn(); + await expect( + executeWorkspaceTool({ + baseURL: 'https://code.example.com/v1', + authHeaders: {}, + request, + maxRequestTimeoutMs, + fetchImpl, + }), + ).rejects.toMatchObject({ + reason: 'insufficient_time', + message: expect.stringContaining('not started'), + }); + expect(fetchImpl).not.toHaveBeenCalled(); + }, + ); + + test('keeps the retry horizon separate from one longer admission attempt', async () => { + jest.spyOn(Date, 'now').mockReturnValue(1_000); + const fetchImpl = jest + .fn() + .mockResolvedValue( + new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), { status: 503 }), + ); + await expect( + executeWorkspaceTool({ + baseURL: 'https://code.example.com/v1', + authHeaders: {}, + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + }, + maxRequestTimeoutMs: 125_000, + maxQueueWaitMs: 0, + fetchImpl, + }), + ).rejects.toThrow('The operation was not started'); + expect(fetchImpl.mock.calls[0][1].headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe('90000'); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + test('keeps Retry-After capped at 30 seconds even when one admission may wait longer', async () => { + jest.useFakeTimers(); + try { + const fetchImpl = jest + .fn() + .mockResolvedValueOnce( + new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), { + status: 503, + headers: { 'Retry-After': '300' }, + }), + ) + .mockResolvedValueOnce(new Response('{}', { status: 504 })); + const result = executeWorkspaceTool({ + baseURL: 'https://code.example.com/v1', + authHeaders: {}, + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + }, + maxRequestTimeoutMs: 125_000, + fetchImpl, + }).catch((error: WorkspaceToolHttpError) => error); + await jest.advanceTimersByTimeAsync(29_999); + expect(fetchImpl).toHaveBeenCalledTimes(1); + await jest.advanceTimersByTimeAsync(1); + await expect(result).resolves.toMatchObject({ upstreamStatus: 504 }); + expect(fetchImpl.mock.calls[1][1].headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe( + '60000', + ); + } finally { + jest.useRealTimers(); + } + }); + + test('never retries a post-dispatch timeout with an unknown mutation outcome', async () => { + const fetchImpl = jest + .fn() + .mockRejectedValue(new DOMException('response deadline', 'TimeoutError')); + await expect( + executeWorkspaceTool({ + baseURL: 'https://code.example.com/v1', + authHeaders: {}, + request: { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: 'touch marker', + }, + maxRequestTimeoutMs: 125_000, + fetchImpl, + }), + ).rejects.toMatchObject({ reason: 'timeout' }); expect(fetchImpl).toHaveBeenCalledTimes(1); }); diff --git a/packages/api/src/code/workspace.ts b/packages/api/src/code/workspace.ts index 3de4d8ab5f..c39d4002f9 100644 --- a/packages/api/src/code/workspace.ts +++ b/packages/api/src/code/workspace.ts @@ -1,4 +1,8 @@ -import { CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS } from 'librechat-data-provider'; +import { + CODE_ENVIRONMENT_ADMISSION_MAX_MS, + CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS, + CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS, +} from 'librechat-data-provider'; import type { CodeBridgeFetch } from './bridge'; const WORKSPACE_TOOL_TIMEOUT_MS = 30_000; @@ -19,8 +23,9 @@ const DEFAULT_COMMAND_OUTPUT_BYTES = 256 * 1024; const MAX_COMMAND_OUTPUT_BYTES = 1024 * 1024; const MAX_COMMAND_SIGNAL_LENGTH = 32; const WORKSPACE_COMMAND_TRANSPORT_GRACE_MS = 5_000; -/** Matches Code API's bounded admission wait and command settlement allowance. */ +/** Legacy admission window and Retry-After cap, independent of the retry horizon. */ const WORKSPACE_QUEUE_TIMEOUT_MS = 30_000; +const WORKSPACE_QUEUE_WAIT_HEADER = 'X-LibreChat-Workspace-Queue-Wait-Ms'; /** Compatibility export; the deployment schema owns the default and hard cap. */ export const WORKSPACE_QUEUE_MAX_WAIT_MS: number = CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS; const WORKSPACE_QUEUE_RETRY_DELAY_MS = 1_000; @@ -263,16 +268,25 @@ export type WorkspaceToolResult = export class WorkspaceToolHttpError extends Error { constructor( - public readonly reason: 'rejected' | 'invalid' | 'timeout' | 'failed', + public readonly reason: 'rejected' | 'invalid' | 'timeout' | 'failed' | 'insufficient_time', public readonly upstreamStatus?: number, public readonly upstreamBody?: string, public readonly upstreamBodyTruncated = false, ) { - super( - (reason === 'rejected' && + let message = `Workspace tool request ${reason}`; + if ( + reason === 'rejected' && isWorkspaceAdmissionTimeout(upstreamStatus, upstreamBody, upstreamBodyTruncated) - ? 'Workspace capacity was unavailable before the queue deadline. The operation was not started. Wait for active work to finish or select an independent workspace on a machine with available capacity.' - : `Workspace tool request ${reason}`) + + ) { + message = + 'Workspace capacity was unavailable before the queue deadline. The operation was not started. Wait for active work to finish or select an independent workspace on a machine with available capacity.'; + } + if (reason === 'insufficient_time') { + message = + 'Workspace execution cannot fit within the remaining HTTP budget. The operation was not started.'; + } + super( + message + (upstreamStatus == null ? '' : ` (upstreamStatus: ${upstreamStatus})`) + (upstreamBody ? `; upstreamBody: ${JSON.stringify(upstreamBody)}` : '') + (upstreamBodyTruncated ? ' [body truncated or incomplete]' : ''), @@ -731,13 +745,19 @@ function isValidResult( ); } -function getWorkspaceToolTimeoutMs(request: WorkspaceToolRequest): number { - const executionBudgetMs = - request.operation === 'execute_command' - ? (request.timeoutMs ?? WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS) + +function getWorkspaceExecutionBudgetMs(request: WorkspaceToolRequest): number { + return request.operation === 'execute_command' + ? (request.timeoutMs ?? WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS) + WORKSPACE_COMMAND_SETTLEMENT_GRACE_MS - : WORKSPACE_TOOL_TIMEOUT_MS; - return WORKSPACE_QUEUE_TIMEOUT_MS + executionBudgetMs + WORKSPACE_COMMAND_TRANSPORT_GRACE_MS; + : WORKSPACE_TOOL_TIMEOUT_MS; +} + +function getWorkspaceToolTimeoutMs(request: WorkspaceToolRequest): number { + return ( + WORKSPACE_QUEUE_TIMEOUT_MS + + getWorkspaceExecutionBudgetMs(request) + + WORKSPACE_COMMAND_TRANSPORT_GRACE_MS + ); } /** @@ -749,6 +769,31 @@ export type WorkspaceToolAuthHeaders = | Record | (() => Promise> | Record); +function getWorkspaceAuthHeaders( + supplier: () => Promise> | Record, + signal: AbortSignal, +): Promise> { + signal.throwIfAborted(); + return new Promise((resolve, reject) => { + const abort = () => reject(signal.reason); + signal.addEventListener('abort', abort, { once: true }); + if (signal.aborted) { + abort(); + return; + } + try { + void Promise.resolve(supplier()) + .then(resolve, reject) + .finally(() => { + signal.removeEventListener('abort', abort); + }); + } catch (error) { + signal.removeEventListener('abort', abort); + reject(error); + } + }); +} + export async function executeWorkspaceTool({ baseURL, authHeaders, @@ -756,6 +801,8 @@ export async function executeWorkspaceTool({ signal, fetchImpl = fetch, maxQueueWaitMs = WORKSPACE_QUEUE_MAX_WAIT_MS, + maxRequestTimeoutMs, + deadlineAtMs, }: { baseURL: string; authHeaders: WorkspaceToolAuthHeaders; @@ -763,34 +810,81 @@ export async function executeWorkspaceTool({ signal?: AbortSignal; fetchImpl?: CodeBridgeFetch; maxQueueWaitMs?: number; + /** End-to-end HTTP budget for this call. Omission keeps legacy per-attempt timeouts. */ + maxRequestTimeoutMs?: number; + /** Optional earlier caller deadline; a signal alone has no remaining-time value. */ + deadlineAtMs?: number; }): Promise { if ( !isValidRequest(request) || !Number.isSafeInteger(maxQueueWaitMs) || maxQueueWaitMs < 0 || - maxQueueWaitMs > WORKSPACE_QUEUE_MAX_WAIT_MS + maxQueueWaitMs > WORKSPACE_QUEUE_MAX_WAIT_MS || + (maxRequestTimeoutMs !== undefined && + (!Number.isSafeInteger(maxRequestTimeoutMs) || + maxRequestTimeoutMs < 1 || + maxRequestTimeoutMs > CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS)) || + (deadlineAtMs !== undefined && (!Number.isSafeInteger(deadlineAtMs) || deadlineAtMs < 1)) ) { throw new WorkspaceToolHttpError('invalid'); } + const executionBudgetMs = getWorkspaceExecutionBudgetMs(request); + const completionReserveMs = executionBudgetMs + WORKSPACE_COMMAND_TRANSPORT_GRACE_MS; + const perAttemptTimeoutMs = maxRequestTimeoutMs ?? getWorkspaceToolTimeoutMs(request); + const callerDeadlineAt = Math.min( + deadlineAtMs ?? Infinity, + maxRequestTimeoutMs == null ? Infinity : Date.now() + maxRequestTimeoutMs, + ); const queueDeadlineAt = Date.now() + maxQueueWaitMs; + const retryDeadlineAt = Math.min(queueDeadlineAt, callerDeadlineAt - completionReserveMs); const body = JSON.stringify(request); let lastAdmissionRejection: WorkspaceToolHttpError | undefined; while (true) { try { signal?.throwIfAborted(); - /** The endpoint replies only after execution settles. Capping this by the - * retry horizon would also abort already-admitted commands, with an unknown - * mutation outcome. Server admission and execution retain separate budgets. */ - const timeoutSignal = AbortSignal.timeout(getWorkspaceToolTimeoutMs(request)); + /** Admission and retries cannot consume the reserved execution or delivery time. */ + const attemptTimeoutMs = Math.floor( + Math.min(perAttemptTimeoutMs, callerDeadlineAt - Date.now()), + ); + if (attemptTimeoutMs <= completionReserveMs) { + throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time'); + } + const attemptStartedAt = Date.now(); + const timeoutSignal = AbortSignal.timeout(attemptTimeoutMs); const requestSignal = signal != null && typeof AbortSignal.any === 'function' ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal; - const attemptHeaders = typeof authHeaders === 'function' ? await authHeaders() : authHeaders; - requestSignal.throwIfAborted(); - if (lastAdmissionRejection && Date.now() >= queueDeadlineAt) { + let attemptHeaders: Record; + try { + attemptHeaders = + typeof authHeaders === 'function' + ? await getWorkspaceAuthHeaders(authHeaders, requestSignal) + : authHeaders; + } catch (error) { + if (timeoutSignal.aborted && signal?.aborted !== true) { + throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time'); + } + throw error; + } + signal?.throwIfAborted(); + if (timeoutSignal.aborted) { + throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time'); + } + if (lastAdmissionRejection && Date.now() >= retryDeadlineAt) { throw lastAdmissionRejection; } + const remainingMs = Math.min( + attemptTimeoutMs - (Date.now() - attemptStartedAt), + callerDeadlineAt - Date.now(), + ); + const queueAllowanceMs = Math.min( + CODE_ENVIRONMENT_ADMISSION_MAX_MS, + Math.floor(remainingMs - completionReserveMs), + ); + if (queueAllowanceMs < 1) { + throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time'); + } const response = await fetchImpl( `${baseURL.trim().replace(/\/+$/, '')}/workspace-tools/execute`, { @@ -798,6 +892,7 @@ export async function executeWorkspaceTool({ headers: { ...attemptHeaders, 'Content-Type': 'application/json', + [WORKSPACE_QUEUE_WAIT_HEADER]: String(queueAllowanceMs), }, body, redirect: 'error', @@ -816,13 +911,14 @@ export async function executeWorkspaceTool({ } const delayMs = Math.min( workspaceAdmissionRetryDelay(response.headers.get('Retry-After')), - queueDeadlineAt - Date.now(), + retryDeadlineAt - Date.now(), ); lastAdmissionRejection = rejection; + if (delayMs <= 0) throw rejection; await waitForWorkspaceAdmission(delayMs, signal); - /** A clamped delay can land exactly on the deadline: the budget is spent, - * so never open another admission window that could still be admitted. */ - if (Date.now() >= queueDeadlineAt) { + /** A clamped delay can land exactly on either deadline. Never open + * another admission window without the full execution reserve. */ + if (Date.now() >= retryDeadlineAt) { throw rejection; } continue; @@ -840,7 +936,7 @@ export async function executeWorkspaceTool({ ) { throw error; } - if (error instanceof Error && error.name === 'TimeoutError') { + if (isRecord(error) && error.name === 'TimeoutError') { throw new WorkspaceToolHttpError('timeout'); } throw new WorkspaceToolHttpError('failed'); diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts index 5e0eead7ce..2dd82eaa88 100644 --- a/packages/data-provider/src/config.spec.ts +++ b/packages/data-provider/src/config.spec.ts @@ -561,6 +561,24 @@ describe('attached code environment user config schema', () => { }, ); + it.each([60_000, 125_000, 610_000])( + 'accepts a bounded %i ms workspace HTTP limit', + (maxRequestTimeoutMs) => { + expect(codeEnvironmentUserConfigSchema.parse({ limits: { maxRequestTimeoutMs } })).toEqual({ + limits: { maxRequestTimeoutMs }, + }); + }, + ); + + it.each([0, -1, 0.5, 610_001, NaN, Infinity])( + 'rejects an invalid workspace HTTP limit of %s', + (maxRequestTimeoutMs) => { + expect( + codeEnvironmentUserConfigSchema.safeParse({ limits: { maxRequestTimeoutMs } }).success, + ).toBe(false); + }, + ); + it('keeps an omitted admission budget backward compatible', () => { expect(codeEnvironmentUserConfigSchema.parse({ limits: {} })).toEqual({ limits: {} }); }); @@ -584,7 +602,7 @@ describe('attached code environment user config schema', () => { fileWrite: { allowed: ['allow', 'ask', 'deny'], default: 'ask' }, commandExecution: { allowed: ['ask', 'deny'], default: 'ask' }, }, - limits: { maxCommandTimeoutMs: 120000 }, + limits: { maxCommandTimeoutMs: 120000, maxRequestTimeoutMs: 125_000 }, }, }, ], @@ -597,6 +615,19 @@ describe('attached code environment user config schema', () => { throw new Error(result.error.toString()); } expect(result.success).toBe(true); + expect(result.data).toMatchObject({ + endpoints: { + agents: { + statefulCodeSessions: { + environments: [ + { + configSchema: { limits: { maxRequestTimeoutMs: 125_000 } }, + }, + ], + }, + }, + }, + }); }); it('rejects an attached command timeout above the protocol hard cap', () => { diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index dfb835647e..21288b3dda 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -1205,6 +1205,11 @@ export const CODE_ENVIRONMENT_COMMAND_TIMEOUT_HARD_MAX_MS = 5 * 60_000; * initial admission wait or an already-admitted operation's execution budget. */ export const CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS = 5 * 60_000; +/** Code API's per-request admission ceiling, independent of the retry horizon. */ +export const CODE_ENVIRONMENT_ADMISSION_MAX_MS = 5 * 60_000; +/** Maximum opt-in HTTP budget: five minutes of admission and execution plus ten seconds for settlement and delivery. */ +export const CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS = + CODE_ENVIRONMENT_ADMISSION_MAX_MS + CODE_ENVIRONMENT_COMMAND_TIMEOUT_HARD_MAX_MS + 10_000; /** * Typed user-tunable surface for one attached code environment. Omitted fields @@ -1240,6 +1245,17 @@ export const codeEnvironmentUserConfigSchema = z .min(0) .max(CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS) .optional(), + /** Total HTTP budget for one workspace tool call, including retries, + * execution, settlement, and delivery. Only set this after verifying + * the shortest timeout on the actual Code API path and updating Code API + * to honor per-request queue allowances. Omission keeps the 30-second + * per-attempt admission budget. */ + maxRequestTimeoutMs: z + .number() + .int() + .min(1) + .max(CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS) + .optional(), }) .strict() .optional(),