From c0affede00a9607c9c3fff75778c97a2077fd772 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 19 Sep 2026 09:03:06 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=9F=EF=B8=8F=20fix:=20Preserve=20BYOM?= =?UTF-8?q?=20Tool=20Calls=20Through=20Capacity=20Windows=20(#16060)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ⏳ fix: Keep BYOM tools queued across capacity windows * ⏳ fix: Configure the admission budget and refresh queued credentials Addresses the Codex review on #16060: - Add `limits.maxQueueWaitMs` to the code environment config schema so a deployment can shorten or disable the five-minute admission budget from `librechat.yaml`; omission keeps the merged behavior, `0` surfaces the first capacity expiry to the model. Resolved through `resolveAttachedWorkspaceQueueWaitMs` and passed by ToolService, matching how `maxCommandTimeoutMs` already flows. - Accept an auth-header supplier in `executeWorkspaceTool` and mint it per admission attempt, so a call queued past the Code API token TTL presents fresh credentials instead of failing permanently with 401. The attached Bash tool and the workspace file operations now pass suppliers. - Recheck the deadline after a clamped wait and throw the saved rejection rather than opening an admission window that could still be admitted once the budget is spent. * fix: Complete workspace admission wiring and caller regressions * fix: Share protected edit retry budgets and schema limits --------- Co-authored-by: Lia --- api/server/services/Files/Code/process.js | 60 ++-- .../services/Files/Code/process.spec.js | 86 +++-- api/server/services/ToolService.js | 4 + .../services/__tests__/ToolService.spec.js | 3 +- packages/api/src/agents/handlers.spec.ts | 165 ++++++---- packages/api/src/agents/handlers.ts | 43 ++- packages/api/src/code/command.spec.ts | 12 + packages/api/src/code/command.ts | 23 +- packages/api/src/code/workspace.spec.ts | 301 ++++++++++++++++++ packages/api/src/code/workspace.ts | 158 ++++++--- packages/data-provider/src/config.spec.ts | 23 ++ packages/data-provider/src/config.ts | 16 + 12 files changed, 739 insertions(+), 155 deletions(-) diff --git a/api/server/services/Files/Code/process.js b/api/server/services/Files/Code/process.js index ed800aaf37..34b4e3f953 100644 --- a/api/server/services/Files/Code/process.js +++ b/api/server/services/Files/Code/process.js @@ -1130,14 +1130,16 @@ async function readWorkspaceFile({ bridgeWorkerId, req, signal, + maxQueueWaitMs, }) { - const authHeaders = await getCodeApiAuthHeaders(req, bridgeWorkerId); return executeWorkspaceTool({ baseURL: codeApiBaseUrl, - authHeaders: { - ...authHeaders, + maxQueueWaitMs, + /** Minted per admission attempt: a queued call outlives one token TTL. */ + authHeaders: async () => ({ + ...(await getCodeApiAuthHeaders(req, bridgeWorkerId)), ...codeExecutionHeaders({ executionProfile, bridgeWorkerId }), - }, + }), request: { protocolVersion: 1, operation: 'read_file', @@ -1174,14 +1176,16 @@ async function searchWorkspace({ bridgeWorkerId, req, signal, + maxQueueWaitMs, }) { - const authHeaders = await getCodeApiAuthHeaders(req, bridgeWorkerId); return executeWorkspaceTool({ baseURL: codeApiBaseUrl, - authHeaders: { - ...authHeaders, + maxQueueWaitMs, + /** Minted per admission attempt: a queued call outlives one token TTL. */ + authHeaders: async () => ({ + ...(await getCodeApiAuthHeaders(req, bridgeWorkerId)), ...codeExecutionHeaders({ executionProfile, bridgeWorkerId }), - }, + }), request: { protocolVersion: 1, operation: 'search_text', @@ -1218,14 +1222,16 @@ async function listWorkspaceFiles({ bridgeWorkerId, req, signal, + maxQueueWaitMs, }) { - const authHeaders = await getCodeApiAuthHeaders(req, bridgeWorkerId); return executeWorkspaceTool({ baseURL: codeApiBaseUrl, - authHeaders: { - ...authHeaders, + maxQueueWaitMs, + /** Minted per admission attempt: a queued call outlives one token TTL. */ + authHeaders: async () => ({ + ...(await getCodeApiAuthHeaders(req, bridgeWorkerId)), ...codeExecutionHeaders({ executionProfile, bridgeWorkerId }), - }, + }), request: { protocolVersion: 1, operation: 'list_files', @@ -1249,14 +1255,16 @@ async function writeWorkspaceFile({ bridgeWorkerId, req, signal, + maxQueueWaitMs, }) { - const authHeaders = await getCodeApiAuthHeaders(req, bridgeWorkerId); return executeWorkspaceTool({ baseURL: codeApiBaseUrl, - authHeaders: { - ...authHeaders, + maxQueueWaitMs, + /** Minted per admission attempt: a queued call outlives one token TTL. */ + authHeaders: async () => ({ + ...(await getCodeApiAuthHeaders(req, bridgeWorkerId)), ...codeExecutionHeaders({ executionProfile, bridgeWorkerId }), - }, + }), request: { protocolVersion: 1, operation: 'write_file', @@ -1280,14 +1288,16 @@ async function editWorkspaceFile({ bridgeWorkerId, req, signal, + maxQueueWaitMs, }) { - const authHeaders = await getCodeApiAuthHeaders(req, bridgeWorkerId); return executeWorkspaceTool({ baseURL: codeApiBaseUrl, - authHeaders: { - ...authHeaders, + maxQueueWaitMs, + /** Minted per admission attempt: a queued call outlives one token TTL. */ + authHeaders: async () => ({ + ...(await getCodeApiAuthHeaders(req, bridgeWorkerId)), ...codeExecutionHeaders({ executionProfile, bridgeWorkerId }), - }, + }), request: { protocolVersion: 1, operation: 'edit_file', @@ -1310,14 +1320,16 @@ async function previewWorkspaceEdit({ bridgeWorkerId, req, signal, + maxQueueWaitMs, }) { - const authHeaders = await getCodeApiAuthHeaders(req, bridgeWorkerId); return executeWorkspaceTool({ baseURL: codeApiBaseUrl, - authHeaders: { - ...authHeaders, + maxQueueWaitMs, + /** Minted per admission attempt: a queued call outlives one token TTL. */ + authHeaders: async () => ({ + ...(await getCodeApiAuthHeaders(req, bridgeWorkerId)), ...codeExecutionHeaders({ executionProfile, bridgeWorkerId }), - }, + }), request: { protocolVersion: 1, operation: 'preview_edit', diff --git a/api/server/services/Files/Code/process.spec.js b/api/server/services/Files/Code/process.spec.js index 0a87df47bc..99e3c7464b 100644 --- a/api/server/services/Files/Code/process.spec.js +++ b/api/server/services/Files/Code/process.spec.js @@ -2057,17 +2057,27 @@ describe('Code Process', () => { bridgeWorkerId: 'worker-user-1', req: mockReq, signal: controller.signal, + maxQueueWaitMs: 0, }), ).resolves.toBe(result); - expect(getCodeApiAuthHeaders).toHaveBeenCalledWith(mockReq, 'worker-user-1'); + expect(getCodeApiAuthHeaders).not.toHaveBeenCalled(); + const { authHeaders } = mockExecuteWorkspaceTool.mock.calls[0][0]; + await expect(authHeaders()).resolves.toEqual({ + Authorization: 'Bearer workspace-token', + 'X-CodeAPI-Expected-Profile': 'stateful', + 'X-LibreChat-Code-Worker-ID': 'worker-user-1', + }); + getCodeApiAuthHeaders.mockResolvedValueOnce({ Authorization: 'Bearer refreshed-token' }); + await expect(authHeaders()).resolves.toMatchObject({ + Authorization: 'Bearer refreshed-token', + 'X-LibreChat-Code-Worker-ID': 'worker-user-1', + }); + expect(getCodeApiAuthHeaders).toHaveBeenNthCalledWith(2, mockReq, 'worker-user-1'); expect(mockExecuteWorkspaceTool).toHaveBeenCalledWith({ baseURL: 'https://attached-code.example.com/v1', - authHeaders: { - Authorization: 'Bearer workspace-token', - 'X-CodeAPI-Expected-Profile': 'stateful', - 'X-LibreChat-Code-Worker-ID': 'worker-user-1', - }, + authHeaders: expect.any(Function), + maxQueueWaitMs: 0, request: { protocolVersion: 1, operation: 'read_file', @@ -2105,16 +2115,27 @@ describe('Code Process', () => { bridgeWorkerId: 'worker-user-1', req: mockReq, signal: controller.signal, + maxQueueWaitMs: 0, }), ).resolves.toBe(result); + expect(getCodeApiAuthHeaders).not.toHaveBeenCalled(); + const { authHeaders } = mockExecuteWorkspaceTool.mock.calls[0][0]; + await expect(authHeaders()).resolves.toEqual({ + Authorization: 'Bearer workspace-token', + 'X-CodeAPI-Expected-Profile': 'stateful', + 'X-LibreChat-Code-Worker-ID': 'worker-user-1', + }); + getCodeApiAuthHeaders.mockResolvedValueOnce({ Authorization: 'Bearer refreshed-token' }); + await expect(authHeaders()).resolves.toMatchObject({ + Authorization: 'Bearer refreshed-token', + 'X-LibreChat-Code-Worker-ID': 'worker-user-1', + }); + expect(getCodeApiAuthHeaders).toHaveBeenNthCalledWith(2, mockReq, 'worker-user-1'); expect(mockExecuteWorkspaceTool).toHaveBeenCalledWith({ baseURL: 'https://attached-code.example.com/v1', - authHeaders: { - Authorization: 'Bearer workspace-token', - 'X-CodeAPI-Expected-Profile': 'stateful', - 'X-LibreChat-Code-Worker-ID': 'worker-user-1', - }, + authHeaders: expect.any(Function), + maxQueueWaitMs: 0, request: { protocolVersion: 1, operation: 'search_text', @@ -2152,16 +2173,27 @@ describe('Code Process', () => { bridgeWorkerId: 'worker-user-1', req: mockReq, signal: controller.signal, + maxQueueWaitMs: 0, }), ).resolves.toBe(result); + expect(getCodeApiAuthHeaders).not.toHaveBeenCalled(); + const { authHeaders } = mockExecuteWorkspaceTool.mock.calls[0][0]; + await expect(authHeaders()).resolves.toEqual({ + Authorization: 'Bearer workspace-token', + 'X-CodeAPI-Expected-Profile': 'stateful', + 'X-LibreChat-Code-Worker-ID': 'worker-user-1', + }); + getCodeApiAuthHeaders.mockResolvedValueOnce({ Authorization: 'Bearer refreshed-token' }); + await expect(authHeaders()).resolves.toMatchObject({ + Authorization: 'Bearer refreshed-token', + 'X-LibreChat-Code-Worker-ID': 'worker-user-1', + }); + expect(getCodeApiAuthHeaders).toHaveBeenNthCalledWith(2, mockReq, 'worker-user-1'); expect(mockExecuteWorkspaceTool).toHaveBeenCalledWith({ baseURL: 'https://attached-code.example.com/v1', - authHeaders: { - Authorization: 'Bearer workspace-token', - 'X-CodeAPI-Expected-Profile': 'stateful', - 'X-LibreChat-Code-Worker-ID': 'worker-user-1', - }, + authHeaders: expect.any(Function), + maxQueueWaitMs: 0, request: { protocolVersion: 1, operation: 'list_files', @@ -2200,16 +2232,27 @@ describe('Code Process', () => { bridgeWorkerId: 'worker-user-1', req: mockReq, signal: controller.signal, + maxQueueWaitMs: 0, }), ).resolves.toBe(result); + expect(getCodeApiAuthHeaders).not.toHaveBeenCalled(); + const { authHeaders } = mockExecuteWorkspaceTool.mock.calls[0][0]; + await expect(authHeaders()).resolves.toEqual({ + Authorization: 'Bearer workspace-token', + 'X-CodeAPI-Expected-Profile': 'stateful', + 'X-LibreChat-Code-Worker-ID': 'worker-user-1', + }); + getCodeApiAuthHeaders.mockResolvedValueOnce({ Authorization: 'Bearer refreshed-token' }); + await expect(authHeaders()).resolves.toMatchObject({ + Authorization: 'Bearer refreshed-token', + 'X-LibreChat-Code-Worker-ID': 'worker-user-1', + }); + expect(getCodeApiAuthHeaders).toHaveBeenNthCalledWith(2, mockReq, 'worker-user-1'); expect(mockExecuteWorkspaceTool).toHaveBeenCalledWith({ baseURL: 'https://attached-code.example.com/v1', - authHeaders: { - Authorization: 'Bearer workspace-token', - 'X-CodeAPI-Expected-Profile': 'stateful', - 'X-LibreChat-Code-Worker-ID': 'worker-user-1', - }, + authHeaders: expect.any(Function), + maxQueueWaitMs: 0, request: { protocolVersion: 1, operation: 'write_file', @@ -2859,6 +2902,7 @@ describe('Code Process', () => { tool_resources: { execute_code: { file_ids: [dbFile.file_id], files: [] } }, agentId: 'agent-id', signal: controller.signal, + maxQueueWaitMs: 0, }), ).rejects.toMatchObject({ name: 'AbortError' }); expect(handleFileUpload).toHaveBeenCalledTimes(1); diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.js index 7e4ceca545..faf785e2b1 100644 --- a/api/server/services/ToolService.js +++ b/api/server/services/ToolService.js @@ -51,6 +51,7 @@ const { createRepositoryInstructionSource, createRepositoryInstructionLoader, resolveAttachedWorkspaceCommandTimeoutMax, + resolveAttachedWorkspaceQueueWaitMs, createContextProgrammaticBashTool, resolveCodeExecutionContext, resolveCodeExecutionWorkspaceContext, @@ -2300,6 +2301,9 @@ async function loadToolsForExecution({ maxTimeoutMs: resolveAttachedWorkspaceCommandTimeoutMax( codeExecutionContext.codeEnvironmentConfigSchema, ), + maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs( + codeExecutionContext.codeEnvironmentConfigSchema, + ), }) : createBashExecutionTool({ authHeaders, diff --git a/api/server/services/__tests__/ToolService.spec.js b/api/server/services/__tests__/ToolService.spec.js index 05641b3a67..558be1d5e0 100644 --- a/api/server/services/__tests__/ToolService.spec.js +++ b/api/server/services/__tests__/ToolService.spec.js @@ -2898,7 +2898,7 @@ describe('ToolService - Action Capability Gating', () => { environmentType: 'attached', environmentId: 'personal-machine', bridgeWorkerId: 'worker-abc', - codeEnvironmentConfigSchema: { limits: { maxCommandTimeoutMs: 120000 } }, + codeEnvironmentConfigSchema: { limits: { maxCommandTimeoutMs: 120000, maxQueueWaitMs: 0 } }, }); const toolRegistry = new Map([ [AgentConstants.BASH_TOOL, { name: AgentConstants.BASH_TOOL }], @@ -2925,6 +2925,7 @@ describe('ToolService - Action Capability Gating', () => { workspaceId: 'project-a', gitIdentity: { name: 'LibreChat Agent', email: 'agent@example.com' }, maxTimeoutMs: 120000, + maxQueueWaitMs: 0, }); expect(mockResolveCodeExecutionWorkspaceContext).toHaveBeenCalledWith( expect.objectContaining({ requestedSelections: req.body.codeWorkspaces }), diff --git a/packages/api/src/agents/handlers.spec.ts b/packages/api/src/agents/handlers.spec.ts index daa6eebabf..d67cdb0402 100644 --- a/packages/api/src/agents/handlers.spec.ts +++ b/packages/api/src/agents/handlers.spec.ts @@ -4539,6 +4539,7 @@ describe('createToolExecuteHandler', () => { executionProfile: 'stateful', statefulSessions: true, environmentType: 'attached', + codeEnvironmentConfigSchema: { limits: { maxQueueWaitMs: 0 } }, bridgeWorkerId: 'user-worker', }, }, @@ -4568,6 +4569,7 @@ describe('createToolExecuteHandler', () => { content: 'export const ok = 1;', overwrite: false, workspace_id: 'project-a', + maxQueueWaitMs: 0, codeApiBaseUrl: 'https://code.example.com', executionProfile: 'stateful', bridgeWorkerId: 'user-worker', @@ -4654,6 +4656,7 @@ describe('createToolExecuteHandler', () => { executionProfile: 'stateful', statefulSessions: true, environmentType: 'attached', + codeEnvironmentConfigSchema: { limits: { maxQueueWaitMs: 0 } }, bridgeWorkerId: 'user-worker', }, }, @@ -4688,6 +4691,7 @@ describe('createToolExecuteHandler', () => { { oldText: 'false', newText: 'true' }, ], workspace_id: 'project-a', + maxQueueWaitMs: 0, codeApiBaseUrl: 'https://code.example.com', executionProfile: 'stateful', bridgeWorkerId: 'user-worker', @@ -4760,78 +4764,105 @@ describe('createToolExecuteHandler', () => { expect(editWorkspaceFile).not.toHaveBeenCalled(); }); - it('commits inspected attached edits with the preview revision fence', async () => { - const previewWorkspaceEdit = jest.fn(async () => ({ - protocolVersion: 1 as const, - operation: 'preview_edit' as const, - workspaceId: 'primary', - path: 'src/app.ts', - content: 'const state = "ready";', - hasUtf8Bom: false, - baseSha256: 'b'.repeat(64), - replacements: 1, - bytesWritten: 22, - })); - const editWorkspaceFile = jest.fn(async () => ({ - protocolVersion: 1 as const, - operation: 'edit_file' as const, - workspaceId: 'primary', - path: 'src/app.ts', - replacements: 1, - bytesWritten: 22, - })); - const protectedReq = { - user: { id: 'user-1' }, - config: { - filters: { - files: { - pii: { - fields: ['content'], - starterPatterns: [], - customPatterns: [ - { - id: 'blocked-placeholder', - label: 'blocked placeholder', - regex: 'NEVER-MATCH-THIS', - }, - ], + it.each([ + { budget: 1200, elapsed: 0, remaining: 1200 }, + { budget: 1200, elapsed: 400, remaining: 800 }, + { budget: 1200, elapsed: 1200, remaining: null }, + { budget: 1200, elapsed: 1500, remaining: null }, + { budget: 0, elapsed: 400, remaining: 0 }, + ])( + 'shares a protected edit retry horizon ($budget ms, preview $elapsed ms)', + async ({ budget, elapsed, remaining }) => { + let nowMs = Date.now(); + jest.spyOn(Date, 'now').mockImplementation(() => nowMs); + const previewWorkspaceEdit = jest.fn(async () => { + nowMs += elapsed; + return { + protocolVersion: 1 as const, + operation: 'preview_edit' as const, + workspaceId: 'primary', + path: 'src/app.ts', + content: 'const state = "ready";', + hasUtf8Bom: false, + baseSha256: 'b'.repeat(64), + replacements: 1, + bytesWritten: 22, + }; + }); + const editWorkspaceFile = jest.fn(async () => ({ + protocolVersion: 1 as const, + operation: 'edit_file' as const, + workspaceId: 'primary', + path: 'src/app.ts', + replacements: 1, + bytesWritten: 22, + })); + const protectedReq = { + user: { id: 'user-1' }, + config: { + filters: { + files: { + pii: { + fields: ['content'], + starterPatterns: [], + customPatterns: [ + { + id: 'blocked-placeholder', + label: 'blocked placeholder', + regex: 'NEVER-MATCH-THIS', + }, + ], + }, }, }, }, - }, - } as never; - const handler = makeSandboxAuthoringHandler( - { previewWorkspaceEdit, editWorkspaceFile }, - { - req: protectedReq, - codeExecutionContext: { - baseUrl: 'https://code.example.com', - codeSessionKey: 'attached-session', - executionProfile: 'stateful', - statefulSessions: true, - environmentType: 'attached', - bridgeWorkerId: 'user-worker', + } as never; + const handler = makeSandboxAuthoringHandler( + { previewWorkspaceEdit, editWorkspaceFile }, + { + req: protectedReq, + codeExecutionContext: { + baseUrl: 'https://code.example.com', + codeSessionKey: 'attached-session', + executionProfile: 'stateful', + statefulSessions: true, + environmentType: 'attached', + codeEnvironmentConfigSchema: { limits: { maxQueueWaitMs: budget } }, + bridgeWorkerId: 'user-worker', + }, }, - }, - ); + ); - const [result] = await invokeHandler(handler, [ - { - id: 'call_edit_workspace_inspected', - name: 'edit_file', - args: { - path: 'workspace/src/app.ts', - old_text: 'draft', - new_text: 'ready', + const [result] = await invokeHandler(handler, [ + { + id: 'call_edit_workspace_inspected', + name: 'edit_file', + args: { + path: 'workspace/src/app.ts', + old_text: 'draft', + new_text: 'ready', + }, }, - }, - ]); + ]); - expect(result.status).toBe('success'); - expect(editWorkspaceFile).toHaveBeenCalledWith( - expect.objectContaining({ expected_base_sha256: 'b'.repeat(64) }), - ); - }); + expect(previewWorkspaceEdit).toHaveBeenCalledWith( + expect.objectContaining({ maxQueueWaitMs: budget }), + ); + if (remaining == null) { + expect(result.status).toBe('error'); + expect(result.errorMessage).toContain('The file was not modified'); + expect(editWorkspaceFile).not.toHaveBeenCalled(); + return; + } + expect(result.status).toBe('success'); + expect(editWorkspaceFile).toHaveBeenCalledWith( + expect.objectContaining({ + expected_base_sha256: 'b'.repeat(64), + maxQueueWaitMs: remaining, + }), + ); + }, + ); it('contains a file-artifact policy rejection to its call without rejecting the batch', async () => { const detectorLabel = 'generated-file bearer token'; @@ -5516,6 +5547,7 @@ describe('createToolExecuteHandler', () => { codeSessionKey: 'execute_code:stateful:attached', executionProfile: 'stateful', environmentType: 'attached', + codeEnvironmentConfigSchema: { limits: { maxQueueWaitMs: 0 } }, bridgeWorkerId: 'personal-worker-1', statefulSessions: true, }, @@ -5534,6 +5566,7 @@ describe('createToolExecuteHandler', () => { expect(readWorkspaceFile).toHaveBeenCalledWith({ file_path: 'src/app.ts', workspace_id: 'project-a', + maxQueueWaitMs: 0, start_line: 1, max_lines: 200, codeApiBaseUrl: 'https://code.example.com/v1', @@ -5840,6 +5873,7 @@ describe('createToolExecuteHandler', () => { expect(searchWorkspace).toHaveBeenCalledWith({ query: 'needle', workspace_id: 'project-a', + maxQueueWaitMs: 300000, path: 'src', max_results: 20, codeApiBaseUrl: 'https://code.example.com/v1', @@ -5992,6 +6026,7 @@ describe('createToolExecuteHandler', () => { expect(listWorkspaceFiles).toHaveBeenCalledWith({ workspace_id: 'project-a', + maxQueueWaitMs: 300000, path: 'src', after_path: 'src/app.ts', max_results: 20, diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts index 4c4a836781..82816ac709 100644 --- a/packages/api/src/agents/handlers.ts +++ b/packages/api/src/agents/handlers.ts @@ -116,6 +116,7 @@ import { } from './intent'; import { buildSkillPrimeMessage, isSkillFilePath, SKILL_FILE_PREFIX } from './skills'; import { resolveCallerCapabilityProjectionSnapshot } from './callerCapabilities'; +import { resolveAttachedWorkspaceQueueWaitMs } from '~/code/command'; import { mergeCodeFilesIntoContext } from './codeFilesSession'; import { toolValidationFeedback } from './validationFeedback'; import { createSkillContentDigest } from './compatibility'; @@ -540,6 +541,7 @@ export interface ToolExecuteOptions { bridgeWorkerId?: string; req?: ServerRequest; signal?: AbortSignal; + maxQueueWaitMs?: number; }) => Promise; /** Searches literal text within an attached worker's logical workspace. */ searchWorkspace?: (params: { @@ -552,6 +554,7 @@ export interface ToolExecuteOptions { bridgeWorkerId?: string; req?: ServerRequest; signal?: AbortSignal; + maxQueueWaitMs?: number; }) => Promise; /** Lists relative file paths within an attached worker's logical workspace. */ listWorkspaceFiles?: (params: { @@ -564,6 +567,7 @@ export interface ToolExecuteOptions { bridgeWorkerId?: string; req?: ServerRequest; signal?: AbortSignal; + maxQueueWaitMs?: number; }) => Promise; /** Writes a UTF-8 file within an attached worker's logical workspace. */ writeWorkspaceFile?: (params: { @@ -576,6 +580,7 @@ export interface ToolExecuteOptions { bridgeWorkerId?: string; req?: ServerRequest; signal?: AbortSignal; + maxQueueWaitMs?: number; }) => Promise; /** Previews exact replacements without mutating an attached worker workspace. */ previewWorkspaceEdit?: (params: { @@ -587,6 +592,7 @@ export interface ToolExecuteOptions { bridgeWorkerId?: string; req?: ServerRequest; signal?: AbortSignal; + maxQueueWaitMs?: number; }) => Promise; /** Applies exact replacements atomically within an attached worker workspace. */ editWorkspaceFile?: (params: { @@ -599,6 +605,7 @@ export interface ToolExecuteOptions { bridgeWorkerId?: string; req?: ServerRequest; signal?: AbortSignal; + maxQueueWaitMs?: number; }) => Promise; /** * Reads a code-execution sandbox file by shelling `cat` through the @@ -2421,6 +2428,9 @@ async function handleWorkspaceFileRead( start_line: startLine, max_lines: maxLines, codeApiBaseUrl: codeExecutionContext.baseUrl, + maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs( + codeExecutionContext.codeEnvironmentConfigSchema, + ), executionProfile: codeExecutionContext.executionProfile, ...(codeExecutionContext.bridgeWorkerId ? { bridgeWorkerId: codeExecutionContext.bridgeWorkerId } @@ -2523,6 +2533,9 @@ async function handleWorkspaceSearchCall( ...(typeof args.path === 'string' && args.path.length > 0 ? { path: args.path } : {}), max_results: Number(maxResults), codeApiBaseUrl: codeExecutionContext.baseUrl, + maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs( + codeExecutionContext.codeEnvironmentConfigSchema, + ), executionProfile: codeExecutionContext.executionProfile, ...(codeExecutionContext.bridgeWorkerId ? { bridgeWorkerId: codeExecutionContext.bridgeWorkerId } @@ -2610,6 +2623,9 @@ async function handleWorkspaceListCall( : {}), max_results: Number(maxResults), codeApiBaseUrl: codeExecutionContext.baseUrl, + maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs( + codeExecutionContext.codeEnvironmentConfigSchema, + ), executionProfile: codeExecutionContext.executionProfile, ...(codeExecutionContext.bridgeWorkerId ? { bridgeWorkerId: codeExecutionContext.bridgeWorkerId } @@ -3764,10 +3780,14 @@ function attachedWorkspaceMutationParams( bridgeWorkerId?: string; req?: ServerRequest; signal?: AbortSignal; + maxQueueWaitMs: number; } { return { workspace_id: workspaceId, codeApiBaseUrl: codeExecutionContext.baseUrl, + maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs( + codeExecutionContext.codeEnvironmentConfigSchema, + ), executionProfile: codeExecutionContext.executionProfile, ...(codeExecutionContext.bridgeWorkerId ? { bridgeWorkerId: codeExecutionContext.bridgeWorkerId } @@ -3884,6 +3904,13 @@ async function handleAttachedWorkspaceEditFileCall({ oldText: edit.old_text, newText: edit.new_text, })); + const workspaceParams = attachedWorkspaceMutationParams( + codeExecutionContext, + workspaceId, + req, + signal, + ); + const queueDeadlineAt = Date.now() + workspaceParams.maxQueueWaitMs; let expectedBaseSha256: string | undefined; if (hasActiveFileFieldPolicy(req?.config?.filters, ['content', 'extracted_text'])) { if (!options.previewWorkspaceEdit) { @@ -3900,7 +3927,7 @@ async function handleAttachedWorkspaceEditFileCall({ preview = await options.previewWorkspaceEdit({ file_path: path.filePath, edits: workspaceEdits, - ...attachedWorkspaceMutationParams(codeExecutionContext, workspaceId, req, signal), + ...workspaceParams, }); } catch (error) { if (signal?.aborted === true && isAbortError(error)) throw error; @@ -3913,12 +3940,24 @@ async function handleAttachedWorkspaceEditFileCall({ const filteredContent = filteredFileResult(tc, req, path.filePath, preview.content); if (filteredContent != null) return filteredContent; expectedBaseSha256 = preview.baseSha256; + /** Zero disables retries, not the two operations required for a protected + * edit. Positive horizons must not restart after a successful preview. */ + if (workspaceParams.maxQueueWaitMs > 0) { + const remainingMs = queueDeadlineAt - Date.now(); + if (remainingMs <= 0) { + return errorResult( + tc, + 'The workspace retry budget expired after preview. The file was not modified.', + ); + } + workspaceParams.maxQueueWaitMs = remainingMs; + } } const result = await options.editWorkspaceFile({ file_path: path.filePath, edits: workspaceEdits, ...(expectedBaseSha256 ? { expected_base_sha256: expectedBaseSha256 } : {}), - ...attachedWorkspaceMutationParams(codeExecutionContext, workspaceId, req, signal), + ...workspaceParams, }); return successResult( tc, diff --git a/packages/api/src/code/command.spec.ts b/packages/api/src/code/command.spec.ts index f936ddbf59..769878928b 100644 --- a/packages/api/src/code/command.spec.ts +++ b/packages/api/src/code/command.spec.ts @@ -8,6 +8,7 @@ import { createAttachedWorkspaceBashTool, createGitIdentityProgrammaticBashTool, resolveAttachedWorkspaceCommandTimeoutMax, + resolveAttachedWorkspaceQueueWaitMs, } from './command'; describe('programmatic Bash Git identity', () => { @@ -265,6 +266,17 @@ describe('createAttachedWorkspaceBashTool', () => { expect(resolveAttachedWorkspaceCommandTimeoutMax()).toBe(30_000); }); + test('resolves the administrator-configured admission budget', () => { + expect(resolveAttachedWorkspaceQueueWaitMs()).toBe(5 * 60_000); + expect(resolveAttachedWorkspaceQueueWaitMs({ limits: { maxQueueWaitMs: 30_000 } })).toBe( + 30_000, + ); + expect(resolveAttachedWorkspaceQueueWaitMs({ limits: { maxQueueWaitMs: 0 } })).toBe(0); + expect(resolveAttachedWorkspaceQueueWaitMs({ limits: { maxQueueWaitMs: 10 * 60_000 } })).toBe( + 5 * 60_000, + ); + }); + test('lowers the omitted timeout when the deployment ceiling is below 30 seconds', async () => { const fetchImpl: CodeBridgeFetch = jest.fn(async () => commandResponse()); const bashTool = createAttachedWorkspaceBashTool({ diff --git a/packages/api/src/code/command.ts b/packages/api/src/code/command.ts index e32da5a295..6b1f90d661 100644 --- a/packages/api/src/code/command.ts +++ b/packages/api/src/code/command.ts @@ -19,6 +19,7 @@ import { executeWorkspaceTool, WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + WORKSPACE_QUEUE_MAX_WAIT_MS, } from './workspace'; const DEFAULT_OUTPUT_BYTES = 256 * 1024; @@ -93,6 +94,21 @@ export function resolveAttachedWorkspaceCommandTimeoutMax( : normalizeAttachedWorkspaceCommandTimeoutMax(configured); } +/** + * Client retry horizon for one capacity-blocked invocation. `0` surfaces the + * first capacity expiry without retrying; an in-flight server admission window + * and execution retain their own budgets. + */ +export function resolveAttachedWorkspaceQueueWaitMs( + configSchema?: CodeEnvironmentUserConfigSchema, +): number { + const configured = configSchema?.limits?.maxQueueWaitMs; + if (configured == null || !Number.isSafeInteger(configured) || configured < 0) { + return WORKSPACE_QUEUE_MAX_WAIT_MS; + } + return Math.min(WORKSPACE_QUEUE_MAX_WAIT_MS, configured); +} + export function buildAttachedWorkspaceBashSchema( maxTimeoutMs: number = WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, environment?: CodeWorkspaceDescriptor['environment'], @@ -235,6 +251,7 @@ export function createAttachedWorkspaceBashTool({ environment, gitIdentity, maxTimeoutMs = WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, + maxQueueWaitMs, fetchImpl, }: { baseUrl: string; @@ -244,6 +261,8 @@ export function createAttachedWorkspaceBashTool({ gitIdentity?: AgentGitIdentity | null; /** Deployment ceiling already intersected with the protocol hard cap. */ maxTimeoutMs?: number; + /** Deployment admission budget; omitted keeps the built-in default. */ + maxQueueWaitMs?: number; fetchImpl?: CodeBridgeFetch; }): DynamicStructuredTool { const effectiveMaxTimeoutMs = normalizeAttachedWorkspaceCommandTimeoutMax(maxTimeoutMs); @@ -301,7 +320,8 @@ export function createAttachedWorkspaceBashTool({ try { const result = await executeWorkspaceTool({ baseURL: baseUrl, - authHeaders: await authHeaders(), + /** Passed as a supplier: a queued call outlives its minted token. */ + authHeaders, request: { protocolVersion: 1, operation: 'execute_command', @@ -316,6 +336,7 @@ export function createAttachedWorkspaceBashTool({ }, signal, fetchImpl, + ...(maxQueueWaitMs == null ? {} : { maxQueueWaitMs }), }); if (result.operation !== 'execute_command') { throw new Error('Attached workspace returned an unexpected command result.'); diff --git a/packages/api/src/code/workspace.spec.ts b/packages/api/src/code/workspace.spec.ts index c19c2efd1f..8aa3b21288 100644 --- a/packages/api/src/code/workspace.spec.ts +++ b/packages/api/src/code/workspace.spec.ts @@ -15,6 +15,7 @@ describe('workspace admission feedback', () => { baseURL: 'https://code.example/v1', authHeaders: {}, fetchImpl, + maxQueueWaitMs: 0, request: { protocolVersion: 1, operation: 'execute_command', @@ -26,6 +27,261 @@ describe('workspace admission feedback', () => { expect(fetchImpl).toHaveBeenCalledTimes(1); }); + test('keeps one tool call queued when capacity expires before admission', async () => { + const unavailable = () => + new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), { + status: 503, + headers: { 'Retry-After': '0' }, + }); + const fetchImpl = jest + .fn() + .mockResolvedValueOnce(unavailable()) + .mockResolvedValueOnce(unavailable()) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'src/app.ts', + replacements: 1, + bytesWritten: 24, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ), + ); + + await expect( + executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + request: { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'src/app.ts', + edits: [{ oldText: 'const old = true;', newText: 'const ready = true;' }], + }, + }), + ).resolves.toMatchObject({ operation: 'edit_file', replacements: 1 }); + + expect(fetchImpl).toHaveBeenCalledTimes(3); + expect(fetchImpl.mock.calls.map((call) => call[1]?.body)).toEqual([ + fetchImpl.mock.calls[0][1]?.body, + fetchImpl.mock.calls[0][1]?.body, + fetchImpl.mock.calls[0][1]?.body, + ]); + }); + + test('mints fresh credentials for every admission attempt', async () => { + let minted = 0; + const authHeaders = jest.fn(async () => ({ Authorization: `Bearer token-${++minted}` })); + const fetchImpl = jest + .fn() + .mockResolvedValueOnce( + new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), { + status: 503, + headers: { 'Retry-After': '0' }, + }), + ) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'src/app.ts', + replacements: 1, + bytesWritten: 24, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ), + ); + + await expect( + executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders, + fetchImpl, + request: { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'src/app.ts', + edits: [{ oldText: 'const old = true;', newText: 'const ready = true;' }], + }, + }), + ).resolves.toMatchObject({ operation: 'edit_file' }); + + // A token minted before the first capacity window expires within the + // admission budget, so reusing it would fail the retry with a 401. + expect(authHeaders).toHaveBeenCalledTimes(2); + expect( + fetchImpl.mock.calls.map( + (call) => (call[1]?.headers as Record).Authorization, + ), + ).toEqual(['Bearer token-1', 'Bearer token-2']); + }); + + test('never opens another admission window once the queue budget is spent', async () => { + jest.useFakeTimers(); + try { + const fetchImpl = jest.fn().mockResolvedValue( + new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), { + status: 503, + headers: { 'Retry-After': '1' }, + }), + ); + + const result = executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + maxQueueWaitMs: 150, + request: { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'src/app.ts', + edits: [{ oldText: 'const old = true;', newText: 'const ready = true;' }], + }, + }).catch((error: Error) => error); + + await jest.advanceTimersByTimeAsync(150); + expect(await result).toMatchObject({ + message: expect.stringContaining('The operation was not started'), + }); + + // The clamped wait lands on the deadline: a further dispatch could still be + // admitted and run a long command after the budget expired. + expect(fetchImpl).toHaveBeenCalledTimes(1); + } finally { + jest.useRealTimers(); + } + }); + + test.each([true, false])( + 'does not dispatch when cancelled before or during credential refresh (%s)', + async (beforeRefresh) => { + const controller = new AbortController(); + const reason = new DOMException('Stopped', 'AbortError'); + const authHeaders = jest.fn(async () => { + controller.abort(reason); + return {}; + }); + const fetchImpl = jest.fn(); + if (beforeRefresh) controller.abort(reason); + await expect( + executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders, + fetchImpl, + signal: controller.signal, + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'src/app.ts', + }, + }), + ).rejects.toBe(reason); + expect(fetchImpl).not.toHaveBeenCalled(); + expect(authHeaders).toHaveBeenCalledTimes(beforeRefresh ? 0 : 1); + }, + ); + + test('does not dispatch a retry if credential refresh spends the remaining queue budget', async () => { + const now = jest.spyOn(Date, 'now').mockReturnValue(1000); + const authHeaders = jest + .fn() + .mockResolvedValueOnce({}) + .mockImplementationOnce(async () => { + now.mockReturnValue(2000); + return {}; + }); + const fetchImpl = jest.fn().mockResolvedValueOnce( + new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), { + status: 503, + headers: { 'Retry-After': '0' }, + }), + ); + try { + await expect( + executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders, + fetchImpl, + maxQueueWaitMs: 1000, + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'src/app.ts', + }, + }), + ).rejects.toThrow('The operation was not started'); + expect(authHeaders).toHaveBeenCalledTimes(2); + expect(fetchImpl).toHaveBeenCalledTimes(1); + } finally { + now.mockRestore(); + } + }); + + test('stops waiting for capacity when the chat is cancelled', async () => { + const controller = new AbortController(); + const reason = new DOMException('Stopped', 'AbortError'); + const fetchImpl = jest.fn().mockResolvedValue( + new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), { + status: 503, + headers: { 'Retry-After': '1' }, + }), + ); + const request = executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + signal: controller.signal, + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'src/app.ts', + }, + }); + while (fetchImpl.mock.calls.length === 0) await Promise.resolve(); + controller.abort(reason); + + await expect(request).rejects.toBe(reason); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + test('never retries an ambiguous capacity-looking failure', async () => { + const fetchImpl = jest.fn().mockResolvedValue( + new Response('Gateway unavailable', { + status: 503, + headers: { 'Retry-After': '0' }, + }), + ); + + await expect( + executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + request: { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'src/app.ts', + edits: [{ oldText: 'const old = true;', newText: 'const ready = true;' }], + }, + }), + ).rejects.toMatchObject({ upstreamStatus: 503 }); + + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + test.each([ [504, '{"code":"ASSIGNMENT_EXPIRED"}', false], [503, 'Gateway unavailable', false], @@ -42,6 +298,51 @@ describe('workspace admission feedback', () => { }); describe('executeWorkspaceTool', () => { + test.each([0, 100])( + 'preserves admitted command execution beyond a %i ms retry horizon', + async (maxQueueWaitMs) => { + const now = jest.spyOn(Date, 'now').mockReturnValue(1000); + const timeout = jest.spyOn(AbortSignal, 'timeout'); + const fetchImpl = jest.fn(async (...[_url, init]: Parameters) => { + // The endpoint does not report admission separately. A successful command + // may finish after the client retry horizon without being cancelled. + now.mockReturnValue(2000); + expect(init?.signal?.aborted).toBe(false); + return new Response( + JSON.stringify({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + stdout: 'ready', + stderr: '', + exitCode: 0, + timedOut: false, + truncated: false, + }), + { status: 200 }, + ); + }); + await expect( + executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + maxQueueWaitMs, + fetchImpl, + request: { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: 'echo ready', + timeoutMs: 300000, + }, + }), + ).resolves.toMatchObject({ stdout: 'ready', exitCode: 0 }); + expect(timeout).toHaveBeenCalledTimes(1); + expect(timeout).toHaveBeenCalledWith(340000); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }, + ); + test.each<[WorkspaceToolRequest, number]>([ [ { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, diff --git a/packages/api/src/code/workspace.ts b/packages/api/src/code/workspace.ts index d043fde185..af5f55336a 100644 --- a/packages/api/src/code/workspace.ts +++ b/packages/api/src/code/workspace.ts @@ -1,3 +1,4 @@ +import { CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS } from 'librechat-data-provider'; import type { CodeBridgeFetch } from './bridge'; const WORKSPACE_TOOL_TIMEOUT_MS = 30_000; @@ -20,6 +21,9 @@ const MAX_COMMAND_SIGNAL_LENGTH = 32; const WORKSPACE_COMMAND_TRANSPORT_GRACE_MS = 5_000; /** Matches Code API's bounded admission wait and command settlement allowance. */ const WORKSPACE_QUEUE_TIMEOUT_MS = 30_000; +/** Compatibility export; the deployment schema owns the default and hard cap. */ +export const WORKSPACE_QUEUE_MAX_WAIT_MS: number = CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS; +const WORKSPACE_QUEUE_RETRY_DELAY_MS = 1_000; const WORKSPACE_COMMAND_SETTLEMENT_GRACE_MS = 5_000; const MAX_RESPONSE_BYTES = 4 * 1024 * 1024; const MAX_ERROR_BODY_BYTES = 4096; @@ -282,6 +286,30 @@ function isWorkspaceAdmissionTimeout(status?: number, body?: string, truncated = } } +function waitForWorkspaceAdmission(delayMs: number, signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + return new Promise((resolve, reject) => { + const cleanup = () => signal?.removeEventListener('abort', abort); + const finish = () => { + cleanup(); + resolve(); + }; + const abort = () => { + clearTimeout(timer); + cleanup(); + reject(signal?.reason); + }; + const timer = setTimeout(finish, delayMs); + signal?.addEventListener('abort', abort, { once: true }); + if (signal?.aborted === true) abort(); + }); +} + +function workspaceAdmissionRetryDelay(value: string | null): number { + if (value == null || !/^\d+$/.test(value)) return WORKSPACE_QUEUE_RETRY_DELAY_MS; + return Math.max(100, Math.min(Number(value) * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS)); +} + /** Keep a received HTTP status even if reading its diagnostic body fails or stalls. */ async function readErrorBody( response: Response, @@ -703,62 +731,110 @@ function getWorkspaceToolTimeoutMs(request: WorkspaceToolRequest): number { return WORKSPACE_QUEUE_TIMEOUT_MS + executionBudgetMs + WORKSPACE_COMMAND_TRANSPORT_GRACE_MS; } +/** + * Credentials for one admission attempt. A supplier is minted per attempt, so a + * call that stays queued past the Code API token TTL presents a fresh token + * instead of failing permanently with 401 while capacity is still pending. + */ +export type WorkspaceToolAuthHeaders = + | Record + | (() => Promise> | Record); + export async function executeWorkspaceTool({ baseURL, authHeaders, request, signal, fetchImpl = fetch, + maxQueueWaitMs = WORKSPACE_QUEUE_MAX_WAIT_MS, }: { baseURL: string; - authHeaders: Record; + authHeaders: WorkspaceToolAuthHeaders; request: WorkspaceToolRequest; signal?: AbortSignal; fetchImpl?: CodeBridgeFetch; + maxQueueWaitMs?: number; }): Promise { - if (!isValidRequest(request)) { + if ( + !isValidRequest(request) || + !Number.isSafeInteger(maxQueueWaitMs) || + maxQueueWaitMs < 0 || + maxQueueWaitMs > WORKSPACE_QUEUE_MAX_WAIT_MS + ) { throw new WorkspaceToolHttpError('invalid'); } - try { - const timeoutSignal = AbortSignal.timeout(getWorkspaceToolTimeoutMs(request)); - const requestSignal = - signal != null && typeof AbortSignal.any === 'function' - ? AbortSignal.any([signal, timeoutSignal]) - : timeoutSignal; - const response = await fetchImpl( - `${baseURL.trim().replace(/\/+$/, '')}/workspace-tools/execute`, - { - method: 'POST', - headers: { - ...authHeaders, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(request), - redirect: 'error', - signal: requestSignal, - }, - ); - if (!response.ok) { - const { body, truncated } = await readErrorBody(response, requestSignal); + const queueDeadlineAt = Date.now() + maxQueueWaitMs; + const body = JSON.stringify(request); + let lastAdmissionRejection: WorkspaceToolHttpError | undefined; + while (true) { + try { signal?.throwIfAborted(); - throw new WorkspaceToolHttpError('rejected', response.status, body, truncated); + /** The endpoint replies only after execution settles. Capping this by the + * retry horizon would also abort already-admitted commands, with an unknown + * mutation outcome. Server admission and execution retain separate budgets. */ + const timeoutSignal = AbortSignal.timeout(getWorkspaceToolTimeoutMs(request)); + const requestSignal = + signal != null && typeof AbortSignal.any === 'function' + ? AbortSignal.any([signal, timeoutSignal]) + : timeoutSignal; + const attemptHeaders = typeof authHeaders === 'function' ? await authHeaders() : authHeaders; + requestSignal.throwIfAborted(); + if (lastAdmissionRejection && Date.now() >= queueDeadlineAt) { + throw lastAdmissionRejection; + } + const response = await fetchImpl( + `${baseURL.trim().replace(/\/+$/, '')}/workspace-tools/execute`, + { + method: 'POST', + headers: { + ...attemptHeaders, + 'Content-Type': 'application/json', + }, + body, + redirect: 'error', + signal: requestSignal, + }, + ); + if (!response.ok) { + const { body, truncated } = await readErrorBody(response, requestSignal); + signal?.throwIfAborted(); + const rejection = new WorkspaceToolHttpError('rejected', response.status, body, truncated); + if ( + !isWorkspaceAdmissionTimeout(response.status, body, truncated) || + Date.now() >= queueDeadlineAt + ) { + throw rejection; + } + const delayMs = Math.min( + workspaceAdmissionRetryDelay(response.headers.get('Retry-After')), + queueDeadlineAt - Date.now(), + ); + lastAdmissionRejection = rejection; + await waitForWorkspaceAdmission(delayMs, signal); + /** A clamped delay can land exactly on the deadline: the budget is spent, + * so never open another admission window that could still be admitted. */ + if (Date.now() >= queueDeadlineAt) { + throw rejection; + } + continue; + } + const result = await readBoundedJson(response, requestSignal); + if (!isValidResult(request, result)) { + throw new WorkspaceToolHttpError('invalid'); + } + return result; + } catch (error) { + if (error instanceof WorkspaceToolHttpError) throw error; + if ( + signal?.aborted === true && + (error === signal.reason || (isRecord(error) && error.name === 'AbortError')) + ) { + throw error; + } + if (error instanceof Error && error.name === 'TimeoutError') { + throw new WorkspaceToolHttpError('timeout'); + } + throw new WorkspaceToolHttpError('failed'); } - const result = await readBoundedJson(response, requestSignal); - if (!isValidResult(request, result)) { - throw new WorkspaceToolHttpError('invalid'); - } - return result; - } catch (error) { - if (error instanceof WorkspaceToolHttpError) throw error; - if ( - signal?.aborted === true && - (error === signal.reason || (isRecord(error) && error.name === 'AbortError')) - ) { - throw error; - } - if (error instanceof Error && error.name === 'TimeoutError') { - throw new WorkspaceToolHttpError('timeout'); - } - throw new WorkspaceToolHttpError('failed'); } } diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts index 9617283920..b582ba9a6d 100644 --- a/packages/data-provider/src/config.spec.ts +++ b/packages/data-provider/src/config.spec.ts @@ -6,6 +6,7 @@ import { DEFAULT_MAX_RETAINED_TOOL_COUNT_CHARS, bedrockModels, configSchema, + codeEnvironmentUserConfigSchema, excludedKeys, resolveEndpointType, webSearchSchema, @@ -398,6 +399,28 @@ describe('Agent Management authentication config', () => { }); describe('attached code environment user config schema', () => { + it.each([0, 1200, 300000])( + 'preserves an admission budget of %i milliseconds', + (maxQueueWaitMs) => { + expect(codeEnvironmentUserConfigSchema.parse({ limits: { maxQueueWaitMs } })).toEqual({ + limits: { maxQueueWaitMs }, + }); + }, + ); + + it.each([-1, 0.5, 300001, NaN, Infinity])( + 'rejects an invalid admission budget of %s', + (maxQueueWaitMs) => { + expect( + codeEnvironmentUserConfigSchema.safeParse({ limits: { maxQueueWaitMs } }).success, + ).toBe(false); + }, + ); + + it('keeps an omitted admission budget backward compatible', () => { + expect(codeEnvironmentUserConfigSchema.parse({ limits: {} })).toEqual({ limits: {} }); + }); + it('accepts typed permission controls exposed by the administrator', () => { const result = configSchema.safeParse({ version: '1.0', diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index f8a3a730ab..0b0ec4f21e 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -1160,6 +1160,12 @@ const codeEnvironmentPermissionFieldSchema = z export const CODE_ENVIRONMENT_COMMAND_TIMEOUT_DEFAULT_MS = 30_000; /** Protocol-level ceiling; deployments may only lower this value. */ export const CODE_ENVIRONMENT_COMMAND_TIMEOUT_HARD_MAX_MS = 5 * 60_000; +/** + * Client retry horizon across Code API admission windows. Also the hard cap: + * deployments may only lower it. `0` disables client retries, not the server's + * initial admission wait or an already-admitted operation's execution budget. + */ +export const CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS = 5 * 60_000; /** * Typed user-tunable surface for one attached code environment. Omitted fields @@ -1185,6 +1191,16 @@ export const codeEnvironmentUserConfigSchema = z .min(1) .max(CODE_ENVIRONMENT_COMMAND_TIMEOUT_HARD_MAX_MS) .optional(), + /** Client retry horizon for capacity expirations, shared by preview/edit. + * Omission keeps five minutes; `0` makes each required operation try once. + * An in-flight request retains Code API's admission/execution budgets and + * can finish after this horizon. This is not a server admission timeout. */ + maxQueueWaitMs: z + .number() + .int() + .min(0) + .max(CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS) + .optional(), }) .strict() .optional(),