From 5674faaa01cb4eac3d51232ef9c007823d63462b Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:22:30 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=AA=80=20fix:=20Separate=20Workspace=20Ad?= =?UTF-8?q?mission=20and=20Transport=20Budgets=20(#16728)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: Separate workspace admission and transport budgets * fix: Validate workspace run reserves and retry test types --------- Co-authored-by: Lia --- api/server/services/Files/Code/process.js | 24 ++ .../services/Files/Code/process.spec.js | 16 + api/server/services/ToolService.js | 4 + .../services/__tests__/ToolService.spec.js | 16 + packages/api/src/agents/handlers.spec.ts | 34 +- packages/api/src/agents/handlers.ts | 24 +- packages/api/src/code/command.spec.ts | 52 +++ packages/api/src/code/command.ts | 17 +- packages/api/src/code/workspace.spec.ts | 233 +++++++++++- packages/api/src/code/workspace.ts | 355 ++++++++++++------ packages/data-provider/src/config.spec.ts | 45 +++ packages/data-provider/src/config.ts | 52 ++- 12 files changed, 712 insertions(+), 160 deletions(-) diff --git a/api/server/services/Files/Code/process.js b/api/server/services/Files/Code/process.js index a50987755c..594e1aba38 100644 --- a/api/server/services/Files/Code/process.js +++ b/api/server/services/Files/Code/process.js @@ -1092,6 +1092,8 @@ async function readWorkspaceFile({ signal, maxQueueWaitMs, maxRequestTimeoutMs, + maxRunTimeoutMs, + admission, deadlineAtMs, }) { return executeWorkspaceTool({ @@ -1100,6 +1102,8 @@ async function readWorkspaceFile({ maxQueueWaitMs, codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs, maxRequestTimeoutMs, + ...(maxRunTimeoutMs == null ? {} : { maxRunTimeoutMs }), + ...(admission == null ? {} : { admission }), deadlineAtMs, /** Minted per admission attempt: a queued call outlives one token TTL. */ authHeaders: async () => ({ @@ -1149,6 +1153,8 @@ async function searchWorkspace({ signal, maxQueueWaitMs, maxRequestTimeoutMs, + maxRunTimeoutMs, + admission, deadlineAtMs, }) { return executeWorkspaceTool({ @@ -1157,6 +1163,8 @@ async function searchWorkspace({ maxQueueWaitMs, codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs, maxRequestTimeoutMs, + ...(maxRunTimeoutMs == null ? {} : { maxRunTimeoutMs }), + ...(admission == null ? {} : { admission }), deadlineAtMs, /** Minted per admission attempt: a queued call outlives one token TTL. */ authHeaders: async () => ({ @@ -1206,6 +1214,8 @@ async function listWorkspaceFiles({ signal, maxQueueWaitMs, maxRequestTimeoutMs, + maxRunTimeoutMs, + admission, deadlineAtMs, }) { return executeWorkspaceTool({ @@ -1214,6 +1224,8 @@ async function listWorkspaceFiles({ maxQueueWaitMs, codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs, maxRequestTimeoutMs, + ...(maxRunTimeoutMs == null ? {} : { maxRunTimeoutMs }), + ...(admission == null ? {} : { admission }), deadlineAtMs, /** Minted per admission attempt: a queued call outlives one token TTL. */ authHeaders: async () => ({ @@ -1248,6 +1260,8 @@ async function writeWorkspaceFile({ signal, maxQueueWaitMs, maxRequestTimeoutMs, + maxRunTimeoutMs, + admission, deadlineAtMs, }) { return executeWorkspaceTool({ @@ -1256,6 +1270,8 @@ async function writeWorkspaceFile({ maxQueueWaitMs, codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs, maxRequestTimeoutMs, + ...(maxRunTimeoutMs == null ? {} : { maxRunTimeoutMs }), + ...(admission == null ? {} : { admission }), deadlineAtMs, /** Minted per admission attempt: a queued call outlives one token TTL. */ authHeaders: async () => ({ @@ -1291,6 +1307,8 @@ async function editWorkspaceFile({ signal, maxQueueWaitMs, maxRequestTimeoutMs, + maxRunTimeoutMs, + admission, deadlineAtMs, }) { return executeWorkspaceTool({ @@ -1299,6 +1317,8 @@ async function editWorkspaceFile({ maxQueueWaitMs, codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs, maxRequestTimeoutMs, + ...(maxRunTimeoutMs == null ? {} : { maxRunTimeoutMs }), + ...(admission == null ? {} : { admission }), deadlineAtMs, /** Minted per admission attempt: a queued call outlives one token TTL. */ authHeaders: async () => ({ @@ -1334,6 +1354,8 @@ async function previewWorkspaceEdit({ signal, maxQueueWaitMs, maxRequestTimeoutMs, + maxRunTimeoutMs, + admission, deadlineAtMs, }) { return executeWorkspaceTool({ @@ -1342,6 +1364,8 @@ async function previewWorkspaceEdit({ maxQueueWaitMs, codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs, maxRequestTimeoutMs, + ...(maxRunTimeoutMs == null ? {} : { maxRunTimeoutMs }), + ...(admission == null ? {} : { admission }), deadlineAtMs, /** Minted per admission attempt: a queued call outlives one token TTL. */ authHeaders: async () => ({ diff --git a/api/server/services/Files/Code/process.spec.js b/api/server/services/Files/Code/process.spec.js index 4257741ae9..cce7a4ba29 100644 --- a/api/server/services/Files/Code/process.spec.js +++ b/api/server/services/Files/Code/process.spec.js @@ -2085,6 +2085,14 @@ describe('Code Process', () => { signal: controller.signal, maxQueueWaitMs: 0, maxRequestTimeoutMs: 125_000, + maxRunTimeoutMs: 180_000, + admission: { + queueWaitMs: 60_000, + initialDelayMs: 1_000, + maxDelayMs: 30_000, + multiplier: 2, + jitterRatio: 0.2, + }, deadlineAtMs: 160_000, }), ).resolves.toBe(result); @@ -2108,6 +2116,14 @@ describe('Code Process', () => { maxQueueWaitMs: 0, codeApiMaxRetryWaitMs: undefined, maxRequestTimeoutMs: 125_000, + maxRunTimeoutMs: 180_000, + admission: { + queueWaitMs: 60_000, + initialDelayMs: 1_000, + maxDelayMs: 30_000, + multiplier: 2, + jitterRatio: 0.2, + }, deadlineAtMs: 160_000, request: { protocolVersion: 1, diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.js index 048cd0c4ea..bb7002a2c5 100644 --- a/api/server/services/ToolService.js +++ b/api/server/services/ToolService.js @@ -53,6 +53,7 @@ const { createRepositoryInstructionLoader, resolveAttachedWorkspaceCommandTimeoutMax, resolveAttachedWorkspaceQueueWaitMs, + resolveAttachedWorkspaceAdmissionOptions, resolveAttachedWorkspaceRequestTimeoutMs, createContextProgrammaticBashTool, resolveCodeExecutionContext, @@ -2383,6 +2384,9 @@ async function loadToolsForExecution({ maxRequestTimeoutMs: resolveAttachedWorkspaceRequestTimeoutMs( codeExecutionContext.codeEnvironmentConfigSchema, ), + ...resolveAttachedWorkspaceAdmissionOptions( + codeExecutionContext.codeEnvironmentConfigSchema, + ), minCommandAdmissionMs: codeExecutionContext.codeEnvironmentConfigSchema?.limits?.minCommandAdmissionMs, }) diff --git a/api/server/services/__tests__/ToolService.spec.js b/api/server/services/__tests__/ToolService.spec.js index 5516ce25f5..d851f4075a 100644 --- a/api/server/services/__tests__/ToolService.spec.js +++ b/api/server/services/__tests__/ToolService.spec.js @@ -3062,8 +3062,16 @@ describe('ToolService - Action Capability Gating', () => { defaultCommandTimeoutMs: 60_000, maxQueueWaitMs: 0, maxRequestTimeoutMs: 90_000, + maxRunTimeoutMs: 180_000, minCommandAdmissionMs: 15_000, }, + admission: { + queueWaitMs: 60_000, + initialDelayMs: 1_000, + maxDelayMs: 30_000, + multiplier: 2, + jitterRatio: 0.2, + }, }, }); const toolRegistry = new Map([ @@ -3095,6 +3103,14 @@ describe('ToolService - Action Capability Gating', () => { maxQueueWaitMs: 0, codeApiMaxRetryWaitMs: 0, maxRequestTimeoutMs: 90_000, + maxRunTimeoutMs: 180_000, + admission: { + queueWaitMs: 60_000, + initialDelayMs: 1_000, + maxDelayMs: 30_000, + multiplier: 2, + jitterRatio: 0.2, + }, minCommandAdmissionMs: 15_000, }); expect(mockResolveCodeExecutionWorkspaceContext).toHaveBeenCalledWith( diff --git a/packages/api/src/agents/handlers.spec.ts b/packages/api/src/agents/handlers.spec.ts index 0e4609741d..e6e0e83b25 100644 --- a/packages/api/src/agents/handlers.spec.ts +++ b/packages/api/src/agents/handlers.spec.ts @@ -7394,7 +7394,18 @@ describe('createToolExecuteHandler', () => { statefulSessions: true, environmentType: 'attached', codeEnvironmentConfigSchema: { - limits: { maxQueueWaitMs: budget, maxRequestTimeoutMs: 125_000 }, + limits: { + maxQueueWaitMs: budget, + maxRequestTimeoutMs: 125_000, + maxRunTimeoutMs: 180_000, + }, + admission: { + queueWaitMs: 60_000, + initialDelayMs: 1_000, + maxDelayMs: 30_000, + multiplier: 2, + jitterRatio: 0.2, + }, }, bridgeWorkerId: 'user-worker', }, @@ -7417,7 +7428,13 @@ describe('createToolExecuteHandler', () => { expect.objectContaining({ maxQueueWaitMs: budget, maxRequestTimeoutMs: 125_000, - deadlineAtMs: startedAt + 125_000, + maxRunTimeoutMs: 180_000, + admission: expect.objectContaining({ + queueWaitMs: 60_000, + multiplier: 2, + jitterRatio: 0.2, + }), + deadlineAtMs: startedAt + 180_000, }), ); expect(result.status).toBe('success'); @@ -7426,7 +7443,13 @@ describe('createToolExecuteHandler', () => { expected_base_sha256: 'b'.repeat(64), maxQueueWaitMs: remaining, maxRequestTimeoutMs: 125_000, - deadlineAtMs: startedAt + 125_000, + maxRunTimeoutMs: 180_000, + admission: expect.objectContaining({ + queueWaitMs: 60_000, + multiplier: 2, + jitterRatio: 0.2, + }), + deadlineAtMs: startedAt + 180_000, }), ); }, @@ -9052,8 +9075,9 @@ describe('createToolExecuteHandler', () => { async (name) => { const message = await reject(name, legacyBody); - expect(message).toContain('Invalid workspace path'); - expect(message.endsWith('"}"')).toBe(true); + expect(message).toContain('INVALID_PATH'); + expect(message).not.toContain('Invalid workspace path'); + expect(message).not.toContain('upstreamBody'); }, ); }); diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts index fb40c62e8e..e64f4d2b14 100644 --- a/packages/api/src/agents/handlers.ts +++ b/packages/api/src/agents/handlers.ts @@ -38,6 +38,7 @@ import type { CodeEnvRef, CodeWorkspaceOperation, PtcToolCallEvent } from 'libre import type { StructuredToolInterface } from '@librechat/agents/langchain/tools'; import type { CodeEnvFile, CodeSessionContext } from '@librechat/agents'; import type { + WorkspaceAdmissionOptions, WorkspaceEditResult, WorkspaceTextEdit, WorkspacePreviewEditResult, @@ -106,6 +107,7 @@ import { import { resolveAttachedWorkspaceReadFileLines, WorkspaceToolHttpError, + resolveAttachedWorkspaceAdmissionOptions, WORKSPACE_EDIT_MAX_COUNT, WORKSPACE_WRITE_MAX_BYTES, } from '~/code/workspace'; @@ -605,6 +607,8 @@ export interface ToolExecuteOptions { signal?: AbortSignal; maxQueueWaitMs?: number; maxRequestTimeoutMs?: number; + maxRunTimeoutMs?: number; + admission?: WorkspaceAdmissionOptions['admission']; deadlineAtMs?: number; }) => Promise; /** Searches literal text within an attached worker's logical workspace. */ @@ -622,6 +626,8 @@ export interface ToolExecuteOptions { signal?: AbortSignal; maxQueueWaitMs?: number; maxRequestTimeoutMs?: number; + maxRunTimeoutMs?: number; + admission?: WorkspaceAdmissionOptions['admission']; deadlineAtMs?: number; }) => Promise; /** Lists relative file paths within an attached worker's logical workspace. */ @@ -639,6 +645,8 @@ export interface ToolExecuteOptions { signal?: AbortSignal; maxQueueWaitMs?: number; maxRequestTimeoutMs?: number; + maxRunTimeoutMs?: number; + admission?: WorkspaceAdmissionOptions['admission']; deadlineAtMs?: number; }) => Promise; /** Writes a UTF-8 file within an attached worker's logical workspace. */ @@ -656,6 +664,8 @@ export interface ToolExecuteOptions { signal?: AbortSignal; maxQueueWaitMs?: number; maxRequestTimeoutMs?: number; + maxRunTimeoutMs?: number; + admission?: WorkspaceAdmissionOptions['admission']; deadlineAtMs?: number; }) => Promise; /** Previews exact replacements without mutating an attached worker workspace. */ @@ -674,6 +684,8 @@ export interface ToolExecuteOptions { signal?: AbortSignal; maxQueueWaitMs?: number; maxRequestTimeoutMs?: number; + maxRunTimeoutMs?: number; + admission?: WorkspaceAdmissionOptions['admission']; deadlineAtMs?: number; }) => Promise; /** Applies exact replacements atomically within an attached worker workspace. */ @@ -693,6 +705,8 @@ export interface ToolExecuteOptions { signal?: AbortSignal; maxQueueWaitMs?: number; maxRequestTimeoutMs?: number; + maxRunTimeoutMs?: number; + admission?: WorkspaceAdmissionOptions['admission']; deadlineAtMs?: number; }) => Promise; /** Bounded reads return complete text; omitted maxBytes retains the legacy stdout path. */ @@ -3842,10 +3856,11 @@ function attachedWorkspaceAuthoringPath( function attachedWorkspaceRequestLimits(codeExecutionContext: CodeExecutionContext): { maxQueueWaitMs: number; maxRequestTimeoutMs?: number; -} { +} & WorkspaceAdmissionOptions { const config = codeExecutionContext.codeEnvironmentConfigSchema; const maxRequestTimeoutMs = resolveAttachedWorkspaceRequestTimeoutMs(config); return { + ...resolveAttachedWorkspaceAdmissionOptions(config), maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs(config), ...(maxRequestTimeoutMs == null ? {} : { maxRequestTimeoutMs }), }; @@ -3868,8 +3883,9 @@ function attachedWorkspaceMutationParams( maxQueueWaitMs: number; maxRequestTimeoutMs?: number; deadlineAtMs?: number; -} { +} & WorkspaceAdmissionOptions { const limits = attachedWorkspaceRequestLimits(codeExecutionContext); + const runTimeoutMs = limits.maxRunTimeoutMs ?? limits.maxRequestTimeoutMs; return { workspace_id: workspaceId, ...(codeExecutionContext.codeWorkspace?.workspaceInstanceId @@ -3878,9 +3894,7 @@ function attachedWorkspaceMutationParams( ...(codeExecutionContext.codeWorkspace?.linkedWorktrees ? { linked_worktrees: true } : {}), codeApiBaseUrl: codeExecutionContext.baseUrl, ...limits, - ...(limits.maxRequestTimeoutMs == null - ? {} - : { deadlineAtMs: Date.now() + limits.maxRequestTimeoutMs }), + ...(runTimeoutMs == null ? {} : { deadlineAtMs: Date.now() + runTimeoutMs }), executionProfile: codeExecutionContext.executionProfile, ...(codeExecutionContext.bridgeWorkerId ? { bridgeWorkerId: codeExecutionContext.bridgeWorkerId } diff --git a/packages/api/src/code/command.spec.ts b/packages/api/src/code/command.spec.ts index 39c1caacb3..5b55ff0a0d 100644 --- a/packages/api/src/code/command.spec.ts +++ b/packages/api/src/code/command.spec.ts @@ -1308,3 +1308,55 @@ describe('createAttachedWorkspaceBashTool', () => { ]); }); }); + +describe('attached command admission configuration', () => { + test('bounds the advertised command ceiling by the shorter overall run budget', () => { + expect( + resolveAttachedWorkspaceCommandTimeoutMax({ + limits: { + maxCommandTimeoutMs: 100_000, + maxRequestTimeoutMs: 100_000, + maxRunTimeoutMs: 65_000, + }, + }), + ).toBe(45_000); + }); + + test('passes separate run and retry policies through the actual tool invocation', async () => { + const admission = { + queueWaitMs: 60_000, + initialDelayMs: 1_000, + maxDelayMs: 30_000, + multiplier: 2, + jitterRatio: 0.2, + }; + const fetchImpl = jest.fn, Parameters>( + async () => + new Response( + JSON.stringify({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + exitCode: 0, + stdout: 'ready', + stderr: '', + timedOut: false, + truncated: false, + }), + ), + ); + const bash = createAttachedWorkspaceBashTool({ + baseUrl: 'https://code.example/v1', + authHeaders: () => ({}), + workspaceId: 'primary', + fetchImpl, + maxRequestTimeoutMs: 100_000, + maxRunTimeoutMs: 180_000, + admission, + }); + await bash.invoke({ command: 'echo ready' }); + expect( + new Headers(fetchImpl.mock.calls[0][1]?.headers).get('X-LibreChat-Workspace-Queue-Wait-Ms'), + ).toBe('60000'); + }); +}); diff --git a/packages/api/src/code/command.ts b/packages/api/src/code/command.ts index cdf53ee132..72e6e8a405 100644 --- a/packages/api/src/code/command.ts +++ b/packages/api/src/code/command.ts @@ -14,7 +14,7 @@ import type { import type { createBashProgrammaticToolCallingSchema } from '@librechat/agents'; import type { DynamicStructuredTool } from '@librechat/agents/langchain/tools'; import type { LCTool } from '@librechat/agents'; -import type { WorkspaceExecuteCommandResult } from './workspace'; +import type { WorkspaceAdmissionOptions, WorkspaceExecuteCommandResult } from './workspace'; import type { CodeExecutionContext } from '~/agents/execution'; import type { CodeBridgeFetch } from './bridge'; import { @@ -173,7 +173,10 @@ export function resolveAttachedWorkspaceCommandTimeoutMax( } return fitCommandTimeoutMaxToBudget( Math.min(requested, upstream), - resolveAttachedWorkspaceRequestTimeoutMs(configSchema), + Math.min( + resolveAttachedWorkspaceRequestTimeoutMs(configSchema) ?? Infinity, + configSchema?.limits?.maxRunTimeoutMs ?? Infinity, + ), configSchema?.limits?.minCommandAdmissionMs, ); } @@ -183,7 +186,7 @@ function fitCommandTimeoutMaxToBudget( maxRequestTimeoutMs?: number, minCommandAdmissionMs?: number, ): number { - if (maxRequestTimeoutMs == null) return maxTimeoutMs; + if (maxRequestTimeoutMs == null || !Number.isFinite(maxRequestTimeoutMs)) return maxTimeoutMs; return Math.min( maxTimeoutMs, fitWorkspaceCommandTimeoutToBudget(maxRequestTimeoutMs, minCommandAdmissionMs), @@ -450,6 +453,8 @@ export function createAttachedWorkspaceBashTool({ maxQueueWaitMs, codeApiMaxRetryWaitMs, maxRequestTimeoutMs, + maxRunTimeoutMs, + admission, minCommandAdmissionMs, linkedWorktrees = false, nativeSandbox = false, @@ -477,10 +482,10 @@ export function createAttachedWorkspaceBashTool({ /** Minimum time for command admission inside an opted-in HTTP budget. */ minCommandAdmissionMs?: number; fetchImpl?: CodeBridgeFetch; -}): DynamicStructuredTool { +} & WorkspaceAdmissionOptions): DynamicStructuredTool { const effectiveMaxTimeoutMs = fitCommandTimeoutMaxToBudget( normalizeAttachedWorkspaceCommandTimeoutMax(maxTimeoutMs), - maxRequestTimeoutMs, + Math.min(maxRequestTimeoutMs ?? Infinity, maxRunTimeoutMs ?? Infinity), minCommandAdmissionMs, ); const effectiveDefaultTimeoutMs = resolveAttachedWorkspaceCommandTimeoutDefault( @@ -570,6 +575,8 @@ export function createAttachedWorkspaceBashTool({ ...(maxQueueWaitMs == null ? {} : { maxQueueWaitMs }), ...(codeApiMaxRetryWaitMs == null ? {} : { codeApiMaxRetryWaitMs }), ...(maxRequestTimeoutMs == null ? {} : { maxRequestTimeoutMs }), + ...(maxRunTimeoutMs == null ? {} : { maxRunTimeoutMs }), + ...(admission == null ? {} : { admission }), }); if (result.operation !== 'execute_command') { throw new Error('Attached workspace returned an unexpected command result.'); diff --git a/packages/api/src/code/workspace.spec.ts b/packages/api/src/code/workspace.spec.ts index 11ed3cce2d..2abcf3e18a 100644 --- a/packages/api/src/code/workspace.spec.ts +++ b/packages/api/src/code/workspace.spec.ts @@ -1,3 +1,5 @@ +import { logger } from '@librechat/data-schemas'; +import { codeEnvironmentAdmissionSchema } from 'librechat-data-provider'; import type { WorkspaceToolRequest } from './workspace'; import type { CodeBridgeFetch } from './bridge'; import { @@ -1137,7 +1139,9 @@ describe('executeWorkspaceTool', () => { }), ).rejects.toMatchObject({ upstreamStatus: 504 }); expect(timeout).toHaveBeenCalledWith(60_000); - expect(fetchImpl.mock.calls[0][1].headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe('25000'); + expect( + new Headers(fetchImpl.mock.calls[0][1]?.headers).get('X-LibreChat-Workspace-Queue-Wait-Ms'), + ).toBe('25000'); }); test('subtracts credential acquisition from the allowance before dispatching', async () => { @@ -1230,7 +1234,7 @@ describe('executeWorkspaceTool', () => { expect(fetchImpl).toHaveBeenCalledTimes(1); }); - test('keeps Retry-After capped at 30 seconds even when one admission may wait longer', async () => { + test('does not shorten a Retry-After hint to fit a shrinking total budget', async () => { jest.useFakeTimers(); try { const fetchImpl = jest @@ -1254,13 +1258,8 @@ describe('executeWorkspaceTool', () => { maxRequestTimeoutMs: 125_000, fetchImpl, }).catch((error: WorkspaceToolHttpError) => error); - await jest.advanceTimersByTimeAsync(29_999); + await expect(result).resolves.toMatchObject({ upstreamStatus: 503 }); expect(fetchImpl).toHaveBeenCalledTimes(1); - await jest.advanceTimersByTimeAsync(1); - await expect(result).resolves.toMatchObject({ upstreamStatus: 504 }); - expect(fetchImpl.mock.calls[1][1].headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe( - '60000', - ); } finally { jest.useRealTimers(); } @@ -2425,3 +2424,221 @@ describe('linked worktree lanes', () => { ).rejects.toMatchObject({ reason: 'invalid' }); }); }); + +describe('workspace admission policy and independent budgets', () => { + const command: WorkspaceToolRequest = { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: 'echo ready', + timeoutMs: 30_000, + }; + const success = () => + new Response( + JSON.stringify({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + exitCode: 0, + stdout: 'ready', + stderr: '', + timedOut: false, + truncated: false, + }), + ); + const queueTimeout = () => + new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), { + status: 503, + headers: { 'Retry-After': '0' }, + }); + + beforeEach(() => { + jest.useFakeTimers(); + jest.setSystemTime(new Date('2026-10-04T00:00:00.000Z')); + }); + afterEach(() => jest.useRealTimers()); + + test('keeps the full 30-second execution budget after 60 seconds waiting', async () => { + let attempts = 0; + const fetchImpl = jest.fn, Parameters>( + async (_url, init) => { + if (init == null) throw new Error('Missing request init'); + attempts++; + if (attempts === 1) { + await new Promise((resolve) => setTimeout(resolve, 30_000)); + return queueTimeout(); + } + expect(Number(new Headers(init.headers).get('X-LibreChat-Workspace-Queue-Wait-Ms'))).toBe( + 60_000, + ); + await new Promise((resolve) => setTimeout(resolve, 59_900)); + expect(init.signal?.aborted).toBe(false); + return success(); + }, + ); + const result = executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + request: command, + maxRequestTimeoutMs: 100_000, + maxRunTimeoutMs: 180_000, + admission: codeEnvironmentAdmissionSchema.parse({ queueWaitMs: 60_000 }), + }); + await jest.advanceTimersByTimeAsync(90_000); + await expect(result).resolves.toMatchObject({ stdout: 'ready' }); + expect(fetchImpl).toHaveBeenCalledTimes(2); + expect(fetchImpl.mock.calls[1][1]?.body).toBe(fetchImpl.mock.calls[0][1]?.body); + }); + + test('bounds queue allowance by the transport reserve, not the larger run deadline', async () => { + const fetchImpl = jest.fn, Parameters>(async () => + success(), + ); + await executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + request: command, + maxRequestTimeoutMs: 65_000, + maxRunTimeoutMs: 180_000, + admission: codeEnvironmentAdmissionSchema.parse({ queueWaitMs: 180_000 }), + }); + expect( + new Headers(fetchImpl.mock.calls[0][1]?.headers).get('X-LibreChat-Workspace-Queue-Wait-Ms'), + ).toBe('25000'); + }); + + test('honors an earlier caller deadline even when overall and transport budgets are larger', async () => { + const fetchImpl = jest.fn, Parameters>(async () => + success(), + ); + await expect( + executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + request: command, + maxRequestTimeoutMs: 100_000, + maxRunTimeoutMs: 180_000, + deadlineAtMs: Date.now() + 40_000, + }), + ).rejects.toMatchObject({ reason: 'insufficient_time' }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + test('adds exponential jitter only to whitelisted rejections and caps local backoff', async () => { + jest.spyOn(Math, 'random').mockReturnValue(1 - Number.EPSILON); + const fetchImpl = jest + .fn() + .mockResolvedValueOnce(new Response('{"code":"WORKSPACE_QUEUE_TIMEOUT"}', { status: 503 })) + .mockResolvedValueOnce(new Response('{"code":"WORKSPACE_QUEUE_TIMEOUT"}', { status: 503 })) + .mockResolvedValueOnce(new Response('{"code":"WORKSPACE_QUEUE_TIMEOUT"}', { status: 503 })) + .mockResolvedValueOnce(success()); + const result = executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + request: command, + admission: codeEnvironmentAdmissionSchema.parse({ + initialDelayMs: 1_000, + maxDelayMs: 3_000, + multiplier: 2, + jitterRatio: 0.5, + }), + }); + await jest.advanceTimersByTimeAsync(1_499); + expect(fetchImpl).toHaveBeenCalledTimes(1); + await jest.advanceTimersByTimeAsync(1); + expect(fetchImpl).toHaveBeenCalledTimes(2); + await jest.advanceTimersByTimeAsync(3_000); + expect(fetchImpl).toHaveBeenCalledTimes(3); + await jest.advanceTimersByTimeAsync(3_000); + await expect(result).resolves.toMatchObject({ stdout: 'ready' }); + expect(fetchImpl).toHaveBeenCalledTimes(4); + }); + + test.each(['60', 'date', 'body'])( + 'does not retry before a 60-second %s hint with jitter', + async (hint) => { + jest.spyOn(Math, 'random').mockReturnValue(1 - Number.EPSILON); + const fetchImpl = jest + .fn() + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + error: 'rate_limited', + ...(hint === 'body' ? { retry_after_seconds: 60 } : {}), + }), + { + status: 429, + headers: + hint === 'body' + ? {} + : { + 'Retry-After': + hint === 'date' ? new Date(Date.now() + 60_000).toUTCString() : hint, + }, + }, + ), + ) + .mockResolvedValueOnce(success()); + const result = executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + request: command, + maxRequestTimeoutMs: 100_000, + maxRunTimeoutMs: 180_000, + codeApiMaxRetryWaitMs: 180_000, + admission: codeEnvironmentAdmissionSchema.parse({ jitterRatio: 0.5 }), + }); + await jest.advanceTimersByTimeAsync(89_999); + expect(fetchImpl).toHaveBeenCalledTimes(1); + await jest.advanceTimersByTimeAsync(1); + await expect(result).resolves.toMatchObject({ stdout: 'ready' }); + expect(fetchImpl).toHaveBeenCalledTimes(2); + }, + ); + + test('keeps upstream bodies diagnostic-only for every rejection kind', () => { + for (const status of [429, 503, 502, 409]) { + const body = '{"code":"WORKSPACE_QUEUE_TIMEOUT","error":"private upstream diagnostic"}'; + const error = new WorkspaceToolHttpError('rejected', status, body); + expect(error.message).not.toContain('private upstream diagnostic'); + expect(error.upstreamBody).toBe(body); + } + }); + + test('emits one aggregate outcome without calling HTTP latency queue wait', async () => { + const log = jest.spyOn(logger, 'debug'); + const fetchImpl = jest + .fn() + .mockResolvedValueOnce(queueTimeout()) + .mockResolvedValueOnce(success()); + const result = executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + request: command, + }); + await jest.advanceTimersByTimeAsync(100); + await result; + expect(log).toHaveBeenCalledTimes(1); + expect(log).toHaveBeenCalledWith( + '[WorkspaceAdmission] outcome', + expect.objectContaining({ + attempts: 2, + queueRejections: 1, + retryWaitMs: 100, + outcome: 'completed', + }), + ); + expect(log).toHaveBeenCalledWith( + '[WorkspaceAdmission] outcome', + expect.not.objectContaining({ + queueWaitMs: expect.any(Number), + }), + ); + }); +}); diff --git a/packages/api/src/code/workspace.ts b/packages/api/src/code/workspace.ts index 471ba4ed3b..135f683822 100644 --- a/packages/api/src/code/workspace.ts +++ b/packages/api/src/code/workspace.ts @@ -1,4 +1,7 @@ +import { trace } from '@opentelemetry/api'; +import { logger } from '@librechat/data-schemas'; import { + codeEnvironmentAdmissionSchema, CODE_ENVIRONMENT_ADMISSION_MAX_MS, CODE_ENVIRONMENT_COMMAND_TIMEOUT_DEFAULT_MS, CODE_ENVIRONMENT_READ_FILE_DEFAULT_LINES, @@ -32,12 +35,11 @@ const DEFAULT_COMMAND_OUTPUT_BYTES = 256 * 1024; const MAX_COMMAND_OUTPUT_BYTES = 1024 * 1024; const MAX_COMMAND_SIGNAL_LENGTH = 32; const WORKSPACE_COMMAND_TRANSPORT_GRACE_MS = 5_000; -/** Legacy admission window and Retry-After cap, independent of the retry horizon. */ +/** Legacy per-request admission window, independent of the retry horizon. */ const WORKSPACE_QUEUE_TIMEOUT_MS = 30_000; const WORKSPACE_QUEUE_WAIT_HEADER = 'X-LibreChat-Workspace-Queue-Wait-Ms'; /** Compatibility export; the deployment schema owns the default and hard cap. */ export const WORKSPACE_QUEUE_MAX_WAIT_MS: number = CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS; -const WORKSPACE_QUEUE_RETRY_DELAY_MS = 1_000; const WORKSPACE_COMMAND_SETTLEMENT_GRACE_MS = 5_000; /** @@ -319,6 +321,7 @@ export class WorkspaceToolHttpError extends Error { public readonly upstreamBody?: string, public readonly upstreamBodyTruncated = false, ) { + const parsed = upstreamBodyTruncated ? undefined : parseErrorBody(upstreamBody); let message = `Workspace tool request ${reason}`; const admissionRejection = reason === 'rejected' @@ -339,11 +342,9 @@ export class WorkspaceToolHttpError extends Error { super( message + (upstreamStatus == null ? '' : ` (upstreamStatus: ${upstreamStatus})`) + - (upstreamBody ? `; upstreamBody: ${JSON.stringify(upstreamBody)}` : '') + - (upstreamBodyTruncated ? ' [body truncated or incomplete]' : ''), + (parsed?.code == null ? '' : ` (upstreamCode: ${parsed.code})`), ); this.name = 'WorkspaceToolHttpError'; - const parsed = upstreamBodyTruncated ? undefined : parseErrorBody(upstreamBody); this.upstreamCode = parsed?.code; this.editConflict = reason === 'rejected' && upstreamStatus === 409 && parsed?.code === 'EDIT_CONFLICT' @@ -425,25 +426,56 @@ function waitForWorkspaceAdmission(delayMs: number, signal?: AbortSignal): Promi }); } -function workspaceAdmissionRetryDelay(value: string | null, rateLimitBody?: string): number { +const DEFAULT_ADMISSION_POLICY = codeEnvironmentAdmissionSchema.parse({}); + +export interface WorkspaceAdmissionOptions { + maxRunTimeoutMs?: number; + admission?: CodeEnvironmentUserConfigSchema['admission']; +} + +/** Keep omitted policy absent so existing adapters and total-budget semantics remain unchanged. */ +export function resolveAttachedWorkspaceAdmissionOptions( + config?: CodeEnvironmentUserConfigSchema, +): WorkspaceAdmissionOptions { + return { + ...(config?.limits?.maxRunTimeoutMs == null + ? {} + : { maxRunTimeoutMs: config.limits.maxRunTimeoutMs }), + ...(config?.admission == null ? {} : { admission: config.admission }), + }; +} + +function workspaceAdmissionRetryDelay( + value: string | null, + body: string | undefined, + attempt: number, + policy: NonNullable, +): number { + let hintMs: number | undefined; if (value != null && /^\d+$/.test(value)) { - const seconds = Number(value); - if (Number.isFinite(seconds)) { - return Math.max(100, Math.min(seconds * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS)); - } + hintMs = Number(value) * 1_000; + } else if (value != null) { + const date = Date.parse(value); + if (Number.isFinite(date)) hintMs = Math.max(0, date - Date.now()); } - if (rateLimitBody) { + if (hintMs != null && !Number.isFinite(hintMs)) hintMs = undefined; + if (hintMs == null && body) { try { - const parsed: { retry_after_seconds?: number } | null = JSON.parse(rateLimitBody); + const parsed: { retry_after_seconds?: number } | null = JSON.parse(body); const seconds = parsed?.retry_after_seconds; if (typeof seconds === 'number' && Number.isFinite(seconds) && seconds >= 0) { - return Math.max(100, Math.min(seconds * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS)); + hintMs = seconds * 1_000; } } catch { - // Fall back when the delay hint is unusable. + // The rejection whitelist already requires complete JSON. } } - return WORKSPACE_QUEUE_RETRY_DELAY_MS; + const delayMs = + hintMs == null + ? Math.min(policy.maxDelayMs, policy.initialDelayMs * policy.multiplier ** (attempt - 1)) + : Math.max(100, hintMs); + const jitteredMs = Math.ceil(delayMs + delayMs * policy.jitterRatio * Math.random()); + return hintMs == null ? Math.min(policy.maxDelayMs, jitteredMs) : jitteredMs; } /** Keep a received HTTP status even if reading its diagnostic body fails or stalls. */ @@ -1109,6 +1141,8 @@ export async function executeWorkspaceTool({ maxQueueWaitMs = WORKSPACE_QUEUE_MAX_WAIT_MS, codeApiMaxRetryWaitMs = CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS, maxRequestTimeoutMs, + maxRunTimeoutMs, + admission, deadlineAtMs, linkedWorktrees = false, }: { @@ -1120,14 +1154,18 @@ export async function executeWorkspaceTool({ maxQueueWaitMs?: number; /** Maximum time waiting for Code API rate-limit admission, independent of queue retries. */ codeApiMaxRetryWaitMs?: number; - /** End-to-end HTTP budget for this call. Omission keeps legacy per-attempt timeouts. */ + /** Per-attempt transport ceiling; also the total budget unless maxRunTimeoutMs is set. */ maxRequestTimeoutMs?: number; /** Optional earlier caller deadline; a signal alone has no remaining-time value. */ deadlineAtMs?: number; /** The worker runs each `.worktrees/` in its own lane; route matching requests there. */ linkedWorktrees?: boolean; -}): Promise { +} & WorkspaceAdmissionOptions): Promise { + const parsedPolicy = codeEnvironmentAdmissionSchema.safeParse( + admission ?? DEFAULT_ADMISSION_POLICY, + ); if ( + !parsedPolicy.success || !isValidRequest(request) || !Number.isSafeInteger(maxQueueWaitMs) || maxQueueWaitMs < 0 || @@ -1139,141 +1177,210 @@ export async function executeWorkspaceTool({ (!Number.isSafeInteger(maxRequestTimeoutMs) || maxRequestTimeoutMs < 1 || maxRequestTimeoutMs > CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS)) || + (maxRunTimeoutMs !== undefined && + (!Number.isSafeInteger(maxRunTimeoutMs) || + maxRunTimeoutMs < 1 || + maxRunTimeoutMs > CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS)) || (deadlineAtMs !== undefined && (!Number.isSafeInteger(deadlineAtMs) || deadlineAtMs < 1)) ) { throw new WorkspaceToolHttpError('invalid'); } + const policy = parsedPolicy.data; const lane = linkedWorktrees === true ? toLinkedWorktreeRequest(request) : undefined; const wireRequest: WorkspaceToolRequest = lane?.request ?? request; const executionBudgetMs = getWorkspaceExecutionBudgetMs(wireRequest); const completionReserveMs = executionBudgetMs + WORKSPACE_COMMAND_TRANSPORT_GRACE_MS; const perAttemptTimeoutMs = maxRequestTimeoutMs ?? getWorkspaceToolTimeoutMs(wireRequest); + const startedAt = Date.now(); + const runTimeoutMs = maxRunTimeoutMs ?? maxRequestTimeoutMs; const callerDeadlineAt = Math.min( deadlineAtMs ?? Infinity, - maxRequestTimeoutMs == null ? Infinity : Date.now() + maxRequestTimeoutMs, + runTimeoutMs == null ? Infinity : startedAt + runTimeoutMs, ); - const queueDeadlineAt = Date.now() + maxQueueWaitMs; + const queueDeadlineAt = startedAt + maxQueueWaitMs; const callerRetryDeadlineAt = callerDeadlineAt - completionReserveMs; const body = JSON.stringify(wireRequest); let lastAdmissionRejection: WorkspaceToolHttpError | undefined; let lastRetryDeadlineAt = Infinity; let rateLimitWaitedMs = 0; - while (true) { - try { - signal?.throwIfAborted(); - /** Admission and retries cannot consume the reserved execution or delivery time. */ - const attemptTimeoutMs = Math.floor( - Math.min(perAttemptTimeoutMs, callerDeadlineAt - Date.now()), - ); - if (attemptTimeoutMs <= completionReserveMs) { - throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time'); - } - const attemptStartedAt = Date.now(); - const timeoutSignal = AbortSignal.timeout(attemptTimeoutMs); - const requestSignal = - signal != null && typeof AbortSignal.any === 'function' - ? AbortSignal.any([signal, timeoutSignal]) - : timeoutSignal; - let attemptHeaders: Record; + let retryWaitMs = 0; + let attempts = 0; + let queueRejections = 0; + let rateLimitRejections = 0; + let outcome = 'failed'; + const span = trace.getTracer('librechat.workspace').startSpan('workspace.tool', { + attributes: { + 'workspace.operation': wireRequest.operation, + 'workspace.id': wireRequest.workspaceId, + ...(lane == null ? {} : { 'workspace.lane': lane.worktree }), + 'workspace.execution_budget_ms': executionBudgetMs, + 'workspace.transport_budget_ms': perAttemptTimeoutMs, + }, + }); + try { + while (true) { try { - attemptHeaders = - typeof authHeaders === 'function' - ? await getWorkspaceAuthHeaders(authHeaders, requestSignal) - : authHeaders; - } catch (error) { - if (timeoutSignal.aborted && signal?.aborted !== true) { + signal?.throwIfAborted(); + /** Admission and retries cannot consume the reserved execution or delivery time. */ + const attemptTimeoutMs = Math.floor( + Math.min(perAttemptTimeoutMs, callerDeadlineAt - Date.now()), + ); + if (attemptTimeoutMs <= completionReserveMs) { throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time'); } - throw error; - } - signal?.throwIfAborted(); - if (timeoutSignal.aborted) { - throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time'); - } - if (lastAdmissionRejection && Date.now() >= lastRetryDeadlineAt) { - throw lastAdmissionRejection; - } - const remainingMs = Math.min( - attemptTimeoutMs - (Date.now() - attemptStartedAt), - callerDeadlineAt - Date.now(), - ); - const queueAllowanceMs = Math.min( - CODE_ENVIRONMENT_ADMISSION_MAX_MS, - Math.floor(remainingMs - completionReserveMs), - ); - if (queueAllowanceMs < 1) { - throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time'); - } - const response = await fetchImpl( - `${baseURL.trim().replace(/\/+$/, '')}/workspace-tools/execute`, - { - method: 'POST', - headers: { - ...attemptHeaders, - 'Content-Type': 'application/json', - [WORKSPACE_QUEUE_WAIT_HEADER]: String(queueAllowanceMs), - }, - body, - redirect: 'error', - signal: requestSignal, - }, - ); - if (!response.ok) { - const { body, truncated } = await readErrorBody(response, requestSignal); + const attemptStartedAt = Date.now(); + const timeoutSignal = AbortSignal.timeout(attemptTimeoutMs); + const requestSignal = + signal != null && typeof AbortSignal.any === 'function' + ? AbortSignal.any([signal, timeoutSignal]) + : timeoutSignal; + let attemptHeaders: Record; + try { + attemptHeaders = + typeof authHeaders === 'function' + ? await getWorkspaceAuthHeaders(authHeaders, requestSignal) + : authHeaders; + } catch (error) { + if (timeoutSignal.aborted && signal?.aborted !== true) { + throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time'); + } + throw error; + } signal?.throwIfAborted(); - const rejection = new WorkspaceToolHttpError('rejected', response.status, body, truncated); - const admission = getWorkspaceAdmissionRejection(response.status, body, truncated); - if (admission == null) throw rejection; - const retryDeadlineAt = Math.min( - admission === 'queue_timeout' ? queueDeadlineAt : Infinity, - callerRetryDeadlineAt, + if (timeoutSignal.aborted) { + throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time'); + } + if (lastAdmissionRejection && Date.now() >= lastRetryDeadlineAt) { + throw lastAdmissionRejection; + } + const remainingMs = Math.min( + attemptTimeoutMs - (Date.now() - attemptStartedAt), + callerDeadlineAt - Date.now(), ); - if (Date.now() >= retryDeadlineAt) throw rejection; - const retryAfterMs = workspaceAdmissionRetryDelay( - response.headers.get('Retry-After'), - admission === 'rate_limited' ? body : undefined, + const queueAllowanceMs = Math.min( + policy.queueWaitMs ?? CODE_ENVIRONMENT_ADMISSION_MAX_MS, + Math.floor(remainingMs - completionReserveMs), ); + if (queueAllowanceMs < 1) { + throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time'); + } + attempts++; + const response = await fetchImpl( + `${baseURL.trim().replace(/\/+$/, '')}/workspace-tools/execute`, + { + method: 'POST', + headers: { + ...attemptHeaders, + 'Content-Type': 'application/json', + [WORKSPACE_QUEUE_WAIT_HEADER]: String(queueAllowanceMs), + }, + body, + redirect: 'error', + signal: requestSignal, + }, + ); + if (!response.ok) { + const { body, truncated } = await readErrorBody(response, requestSignal); + signal?.throwIfAborted(); + const rejection = new WorkspaceToolHttpError( + 'rejected', + response.status, + body, + truncated, + ); + const admission = getWorkspaceAdmissionRejection(response.status, body, truncated); + if (admission == null) throw rejection; + if (admission === 'queue_timeout') queueRejections++; + else rateLimitRejections++; + const rejectionCount = + admission === 'queue_timeout' ? queueRejections : rateLimitRejections; + if (rejectionCount === 1) { + span.addEvent('workspace.admission.rejected', { kind: admission }); + } + const retryDeadlineAt = Math.min( + admission === 'queue_timeout' ? queueDeadlineAt : Infinity, + callerRetryDeadlineAt, + ); + if (Date.now() >= retryDeadlineAt) throw rejection; + const retryAfterMs = workspaceAdmissionRetryDelay( + response.headers.get('Retry-After'), + admission === 'rate_limited' ? body : undefined, + rejectionCount, + policy, + ); + if ( + admission === 'rate_limited' && + retryAfterMs > codeApiMaxRetryWaitMs - rateLimitWaitedMs + ) { + throw rejection; + } + // Never truncate a server hint into an early retry. + if (!Number.isFinite(retryAfterMs) || retryAfterMs >= retryDeadlineAt - Date.now()) { + throw rejection; + } + const delayMs = retryAfterMs; + lastAdmissionRejection = rejection; + lastRetryDeadlineAt = retryDeadlineAt; + if (delayMs <= 0) throw rejection; + const waitStartedAt = Date.now(); + await waitForWorkspaceAdmission(delayMs, signal); + retryWaitMs += Math.max(delayMs, Date.now() - waitStartedAt); + if (admission === 'rate_limited') { + /** A timer may run late. Charge the actual wait, but honor the retry that + * was authorized before waiting; another 429 cannot exceed the balance. */ + rateLimitWaitedMs += Math.max(retryAfterMs, Date.now() - waitStartedAt); + } + /** Timer oversleep cannot consume the execution reserve. */ + if (Date.now() >= retryDeadlineAt) { + throw rejection; + } + continue; + } + const result = await readBoundedJson(response, requestSignal); + if (!isValidResult(wireRequest, result)) { + throw new WorkspaceToolHttpError('invalid'); + } + outcome = 'completed'; + return lane ? fromLinkedWorktreeResult(result, lane.worktree) : result; + } catch (error) { + outcome = error instanceof WorkspaceToolHttpError ? error.reason : 'failed'; + if (signal?.aborted === true) outcome = 'cancelled'; + if (error instanceof WorkspaceToolHttpError) throw error; if ( - admission === 'rate_limited' && - retryAfterMs > codeApiMaxRetryWaitMs - rateLimitWaitedMs + signal?.aborted === true && + (error === signal.reason || (isRecord(error) && error.name === 'AbortError')) ) { - throw rejection; + throw error; } - const delayMs = Math.min(retryAfterMs, retryDeadlineAt - Date.now()); - lastAdmissionRejection = rejection; - lastRetryDeadlineAt = retryDeadlineAt; - if (delayMs <= 0) throw rejection; - const waitStartedAt = Date.now(); - await waitForWorkspaceAdmission(delayMs, signal); - if (admission === 'rate_limited') { - /** A timer may run late. Charge the actual wait, but honor the retry that - * was authorized before waiting; another 429 cannot exceed the balance. */ - rateLimitWaitedMs += Math.max(retryAfterMs, Date.now() - waitStartedAt); + if (isRecord(error) && error.name === 'TimeoutError') { + outcome = 'timeout'; + throw new WorkspaceToolHttpError('timeout'); } - /** A clamped delay can land exactly on this admission's retry deadline. Never open - * another admission window without the full execution reserve. */ - if (Date.now() >= retryDeadlineAt) { - throw rejection; - } - continue; + throw new WorkspaceToolHttpError('failed'); } - const result = await readBoundedJson(response, requestSignal); - if (!isValidResult(wireRequest, result)) { - throw new WorkspaceToolHttpError('invalid'); - } - return lane ? fromLinkedWorktreeResult(result, lane.worktree) : result; - } catch (error) { - if (error instanceof WorkspaceToolHttpError) throw error; - if ( - signal?.aborted === true && - (error === signal.reason || (isRecord(error) && error.name === 'AbortError')) - ) { - throw error; - } - if (isRecord(error) && error.name === 'TimeoutError') { - throw new WorkspaceToolHttpError('timeout'); - } - throw new WorkspaceToolHttpError('failed'); } + } finally { + const diagnostics = { + operation: wireRequest.operation, + workspaceId: wireRequest.workspaceId, + ...(lane == null ? {} : { lane: lane.worktree }), + outcome, + attempts, + queueRejections, + rateLimitRejections, + retryWaitMs, + rateLimitWaitedMs, + elapsedMs: Math.max(0, Date.now() - startedAt), + }; + span.setAttributes({ + 'workspace.outcome': outcome, + 'workspace.attempts': attempts, + 'workspace.queue_rejections': queueRejections, + 'workspace.rate_limit_rejections': rateLimitRejections, + 'workspace.client_retry_wait_ms': retryWaitMs, + 'workspace.rate_limit_wait_ms': rateLimitWaitedMs, + }); + span.end(); + logger.debug('[WorkspaceAdmission] outcome', diagnostics); } } diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts index 4081c33ffa..addf042c7b 100644 --- a/packages/data-provider/src/config.spec.ts +++ b/packages/data-provider/src/config.spec.ts @@ -2518,3 +2518,48 @@ describe('subagent activity policy', () => { ).toBe(false); }); }); + +describe('workspace admission configuration', () => { + it('preserves absent defaults and resolves an explicit admission policy', () => { + expect(codeEnvironmentUserConfigSchema.parse({})).toEqual({}); + expect(codeEnvironmentUserConfigSchema.parse({ admission: {} }).admission).toEqual({ + initialDelayMs: 1_000, + maxDelayMs: 30_000, + multiplier: 1, + jitterRatio: 0, + }); + expect( + codeEnvironmentUserConfigSchema.parse({ + admission: { queueWaitMs: 180_000, multiplier: 2, jitterRatio: 0.2 }, + limits: { maxRequestTimeoutMs: 220_000, maxRunTimeoutMs: 400_000 }, + }), + ).toMatchObject({ + admission: { queueWaitMs: 180_000, multiplier: 2, jitterRatio: 0.2 }, + limits: { maxRequestTimeoutMs: 220_000, maxRunTimeoutMs: 400_000 }, + }); + }); + + it.each([ + { queueWaitMs: 0 }, + { queueWaitMs: 300_001 }, + { initialDelayMs: 99 }, + { initialDelayMs: 2_000, maxDelayMs: 1_000 }, + { multiplier: 0 }, + { jitterRatio: -0.1 }, + { jitterRatio: 1.1 }, + { maxDelayMs: Infinity }, + { queueWaitMs: 1.5 }, + { unexpected: true }, + ])('rejects an invalid policy %j', (admission) => { + expect(codeEnvironmentUserConfigSchema.safeParse({ admission }).success).toBe(false); + }); + + it.each([0, 1, -1, 1.5, 20_000, 610_001, Infinity])( + 'rejects an invalid run deadline %s', + (maxRunTimeoutMs) => { + expect( + codeEnvironmentUserConfigSchema.safeParse({ limits: { maxRunTimeoutMs } }).success, + ).toBe(false); + }, + ); +}); diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 86be6771e6..bdd91bbffa 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -1281,6 +1281,22 @@ export const CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS = CODE_ENVIRONMENT_COMMAND_TIMEOUT_HARD_MAX_MS + CODE_ENVIRONMENT_COMMAND_BUDGET_GRACE_MS; +export const codeEnvironmentAdmissionSchema = z + .object({ + /** Optional per-request queue ceiling, bounded by transport and execution reserves. */ + queueWaitMs: z.number().int().min(1).max(CODE_ENVIRONMENT_ADMISSION_MAX_MS).optional(), + initialDelayMs: z.number().int().min(100).max(30_000).optional().default(1_000), + maxDelayMs: z.number().int().min(100).max(300_000).optional().default(30_000), + multiplier: z.number().min(1).max(10).optional().default(1), + /** Additive jitter never advances a server's Retry-After hint. */ + jitterRatio: z.number().min(0).max(1).optional().default(0), + }) + .strict() + .refine((policy) => policy.maxDelayMs >= policy.initialDelayMs, { + path: ['maxDelayMs'], + message: 'Maximum retry delay must cover the initial delay', + }); + /** * Typed user-tunable surface for one attached code environment. Omitted fields * remain fixed at LibreChat's safe baseline. Isolation, networking, mounts, @@ -1295,6 +1311,7 @@ export const codeEnvironmentUserConfigSchema = z }) .strict() .optional(), + admission: codeEnvironmentAdmissionSchema.optional(), limits: z .object({ /** Foreground Bash timeout when the call omits timeoutMs. Omission keeps 30 seconds; @@ -1331,19 +1348,25 @@ export const codeEnvironmentUserConfigSchema = z .min(0) .max(CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS) .optional(), - /** Total HTTP budget for one workspace tool call, including retries, - * execution, settlement, and delivery. Only set this after verifying - * the shortest timeout on the actual Code API path and updating Code API - * to honor per-request queue allowances. Omission keeps the 30-second - * per-attempt admission budget. */ + /** Transport ceiling after verifying the shortest timeout on the Code API path. + * Also the overall call budget unless maxRunTimeoutMs is set. Omission keeps + * the legacy per-attempt admission and execution budgets. */ maxRequestTimeoutMs: z .number() .int() .min(1) .max(CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS) .optional(), + /** Separate overall call deadline across credentials, backoff, and HTTP attempts. + * Omission preserves maxRequestTimeoutMs as the total budget. */ + maxRunTimeoutMs: z + .number() + .int() + .min(1) + .max(CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS) + .optional(), /** Admission allowance before local dispatch overhead for a Bash command inside - * maxRequestTimeoutMs. Omission reserves ten seconds; ignored without a total HTTP budget. */ + * the transport/run budgets. Omission reserves ten seconds. */ minCommandAdmissionMs: z .number() .int() @@ -1353,19 +1376,22 @@ export const codeEnvironmentUserConfigSchema = z }) .strict() .superRefine((limits, context) => { + const budgetMs = Math.min( + limits.maxRequestTimeoutMs ?? Infinity, + limits.maxRunTimeoutMs ?? Infinity, + ); if ( - limits.maxRequestTimeoutMs == null || - limits.maxRequestTimeoutMs > - (limits.minCommandAdmissionMs ?? CODE_ENVIRONMENT_COMMAND_ADMISSION_DEFAULT_MS) + - CODE_ENVIRONMENT_COMMAND_BUDGET_GRACE_MS + budgetMs > + (limits.minCommandAdmissionMs ?? CODE_ENVIRONMENT_COMMAND_ADMISSION_DEFAULT_MS) + + CODE_ENVIRONMENT_COMMAND_BUDGET_GRACE_MS ) { return; } + const field = + limits.maxRunTimeoutMs === budgetMs ? 'maxRunTimeoutMs' : 'maxRequestTimeoutMs'; context.addIssue({ code: z.ZodIssueCode.custom, - path: [ - limits.minCommandAdmissionMs == null ? 'maxRequestTimeoutMs' : 'minCommandAdmissionMs', - ], + path: [limits.minCommandAdmissionMs == null ? field : 'minCommandAdmissionMs'], message: 'Command admission and settlement reserves must leave time for execution', }); })