mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-16 13:00:55 +00:00
🥧 fix: Reserve Execution Time in Workspace Admission (#16421)
* 🥧 fix: Reserve Execution Time in Workspace Admission * 🥧 fix: Pass Workspace Admission Static Checks --------- Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
parent
8ec632ff0e
commit
bf91d2b59c
12 changed files with 656 additions and 47 deletions
|
|
@ -1088,10 +1088,14 @@ async function readWorkspaceFile({
|
|||
req,
|
||||
signal,
|
||||
maxQueueWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
deadlineAtMs,
|
||||
}) {
|
||||
return executeWorkspaceTool({
|
||||
baseURL: codeApiBaseUrl,
|
||||
maxQueueWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
deadlineAtMs,
|
||||
/** Minted per admission attempt: a queued call outlives one token TTL. */
|
||||
authHeaders: async () => ({
|
||||
...(await getCodeApiAuthHeaders(req, bridgeWorkerId)),
|
||||
|
|
@ -1137,10 +1141,14 @@ async function searchWorkspace({
|
|||
req,
|
||||
signal,
|
||||
maxQueueWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
deadlineAtMs,
|
||||
}) {
|
||||
return executeWorkspaceTool({
|
||||
baseURL: codeApiBaseUrl,
|
||||
maxQueueWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
deadlineAtMs,
|
||||
/** Minted per admission attempt: a queued call outlives one token TTL. */
|
||||
authHeaders: async () => ({
|
||||
...(await getCodeApiAuthHeaders(req, bridgeWorkerId)),
|
||||
|
|
@ -1186,10 +1194,14 @@ async function listWorkspaceFiles({
|
|||
req,
|
||||
signal,
|
||||
maxQueueWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
deadlineAtMs,
|
||||
}) {
|
||||
return executeWorkspaceTool({
|
||||
baseURL: codeApiBaseUrl,
|
||||
maxQueueWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
deadlineAtMs,
|
||||
/** Minted per admission attempt: a queued call outlives one token TTL. */
|
||||
authHeaders: async () => ({
|
||||
...(await getCodeApiAuthHeaders(req, bridgeWorkerId)),
|
||||
|
|
@ -1221,10 +1233,14 @@ async function writeWorkspaceFile({
|
|||
req,
|
||||
signal,
|
||||
maxQueueWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
deadlineAtMs,
|
||||
}) {
|
||||
return executeWorkspaceTool({
|
||||
baseURL: codeApiBaseUrl,
|
||||
maxQueueWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
deadlineAtMs,
|
||||
/** Minted per admission attempt: a queued call outlives one token TTL. */
|
||||
authHeaders: async () => ({
|
||||
...(await getCodeApiAuthHeaders(req, bridgeWorkerId)),
|
||||
|
|
@ -1256,10 +1272,14 @@ async function editWorkspaceFile({
|
|||
req,
|
||||
signal,
|
||||
maxQueueWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
deadlineAtMs,
|
||||
}) {
|
||||
return executeWorkspaceTool({
|
||||
baseURL: codeApiBaseUrl,
|
||||
maxQueueWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
deadlineAtMs,
|
||||
/** Minted per admission attempt: a queued call outlives one token TTL. */
|
||||
authHeaders: async () => ({
|
||||
...(await getCodeApiAuthHeaders(req, bridgeWorkerId)),
|
||||
|
|
@ -1290,10 +1310,14 @@ async function previewWorkspaceEdit({
|
|||
req,
|
||||
signal,
|
||||
maxQueueWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
deadlineAtMs,
|
||||
}) {
|
||||
return executeWorkspaceTool({
|
||||
baseURL: codeApiBaseUrl,
|
||||
maxQueueWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
deadlineAtMs,
|
||||
/** Minted per admission attempt: a queued call outlives one token TTL. */
|
||||
authHeaders: async () => ({
|
||||
...(await getCodeApiAuthHeaders(req, bridgeWorkerId)),
|
||||
|
|
|
|||
|
|
@ -2061,6 +2061,8 @@ describe('Code Process', () => {
|
|||
req: mockReq,
|
||||
signal: controller.signal,
|
||||
maxQueueWaitMs: 0,
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
deadlineAtMs: 160_000,
|
||||
}),
|
||||
).resolves.toBe(result);
|
||||
|
||||
|
|
@ -2081,6 +2083,8 @@ describe('Code Process', () => {
|
|||
baseURL: 'https://attached-code.example.com/v1',
|
||||
authHeaders: expect.any(Function),
|
||||
maxQueueWaitMs: 0,
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
deadlineAtMs: 160_000,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'read_file',
|
||||
|
|
@ -2140,6 +2144,8 @@ describe('Code Process', () => {
|
|||
baseURL: 'https://attached-code.example.com/v1',
|
||||
authHeaders: expect.any(Function),
|
||||
maxQueueWaitMs: 0,
|
||||
maxRequestTimeoutMs: undefined,
|
||||
deadlineAtMs: undefined,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'search_text',
|
||||
|
|
@ -2198,6 +2204,8 @@ describe('Code Process', () => {
|
|||
baseURL: 'https://attached-code.example.com/v1',
|
||||
authHeaders: expect.any(Function),
|
||||
maxQueueWaitMs: 0,
|
||||
maxRequestTimeoutMs: undefined,
|
||||
deadlineAtMs: undefined,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'list_files',
|
||||
|
|
@ -2258,6 +2266,8 @@ describe('Code Process', () => {
|
|||
baseURL: 'https://attached-code.example.com/v1',
|
||||
authHeaders: expect.any(Function),
|
||||
maxQueueWaitMs: 0,
|
||||
maxRequestTimeoutMs: undefined,
|
||||
deadlineAtMs: undefined,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'write_file',
|
||||
|
|
@ -2296,11 +2306,15 @@ describe('Code Process', () => {
|
|||
bridgeWorkerId: 'worker-user-1',
|
||||
req: mockReq,
|
||||
expected_base_sha256: 'a'.repeat(64),
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
deadlineAtMs: 160_000,
|
||||
}),
|
||||
).resolves.toBe(result);
|
||||
|
||||
expect(mockExecuteWorkspaceTool).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
deadlineAtMs: 160_000,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'edit_file',
|
||||
|
|
@ -2338,11 +2352,15 @@ describe('Code Process', () => {
|
|||
codeApiBaseUrl: 'https://attached-code.example.com/v1',
|
||||
executionProfile: 'stateful',
|
||||
req: mockReq,
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
deadlineAtMs: 160_000,
|
||||
}),
|
||||
).resolves.toBe(result);
|
||||
|
||||
expect(mockExecuteWorkspaceTool).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
deadlineAtMs: 160_000,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'preview_edit',
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ const {
|
|||
createRepositoryInstructionLoader,
|
||||
resolveAttachedWorkspaceCommandTimeoutMax,
|
||||
resolveAttachedWorkspaceQueueWaitMs,
|
||||
resolveAttachedWorkspaceRequestTimeoutMs,
|
||||
createContextProgrammaticBashTool,
|
||||
resolveCodeExecutionContext,
|
||||
resolveCodeExecutionWorkspaceContext,
|
||||
|
|
@ -2308,6 +2309,9 @@ async function loadToolsForExecution({
|
|||
maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs(
|
||||
codeExecutionContext.codeEnvironmentConfigSchema,
|
||||
),
|
||||
maxRequestTimeoutMs: resolveAttachedWorkspaceRequestTimeoutMs(
|
||||
codeExecutionContext.codeEnvironmentConfigSchema,
|
||||
),
|
||||
})
|
||||
: createBashExecutionTool({
|
||||
authHeaders,
|
||||
|
|
|
|||
|
|
@ -720,6 +720,17 @@ endpoints:
|
|||
# # Maximum Bash execution budget the model may request per call.
|
||||
# # Omit to preserve the historical 30-second limit. Hard cap: 300000.
|
||||
# maxCommandTimeoutMs: 120000
|
||||
# # Optional total HTTP budget per workspace tool call, including
|
||||
# # admission, retries, worst-case execution, settlement, and delivery.
|
||||
# # Do NOT enable until every LibreChat API replica recognizes this
|
||||
# # strict config field, Code API honors X-LibreChat-Workspace-Queue-Wait-Ms,
|
||||
# # and a non-mutating timed canary verifies the live CDN, load balancer,
|
||||
# # ingress, and caller can all outlast this value. The Nginx example
|
||||
# # (120s) cannot establish the live limit. Omission keeps 30s admission.
|
||||
# # With 125s total, a read can queue <=90s, a default command <=85s,
|
||||
# # a 90s command <=25s. Commands >115s cannot fit; the 120s command
|
||||
# # ceiling above requires a larger VERIFIED total budget to be usable.
|
||||
# # maxRequestTimeoutMs: 125000
|
||||
# permissions:
|
||||
# fileWrite:
|
||||
# allowed: [allow, ask, deny]
|
||||
|
|
|
|||
|
|
@ -5610,7 +5610,9 @@ describe('createToolExecuteHandler', () => {
|
|||
workspaceInstanceId: 'a'.repeat(64),
|
||||
operations: TEST_ATTACHED_WORKSPACE_OPERATIONS,
|
||||
},
|
||||
codeEnvironmentConfigSchema: { limits: { maxQueueWaitMs: 0 } },
|
||||
codeEnvironmentConfigSchema: {
|
||||
limits: { maxQueueWaitMs: 0, maxRequestTimeoutMs: 125_000 },
|
||||
},
|
||||
bridgeWorkerId: 'user-worker',
|
||||
},
|
||||
},
|
||||
|
|
@ -5642,6 +5644,8 @@ describe('createToolExecuteHandler', () => {
|
|||
workspace_id: 'project-a',
|
||||
workspace_instance_id: 'a'.repeat(64),
|
||||
maxQueueWaitMs: 0,
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
deadlineAtMs: expect.any(Number),
|
||||
codeApiBaseUrl: 'https://code.example.com',
|
||||
executionProfile: 'stateful',
|
||||
bridgeWorkerId: 'user-worker',
|
||||
|
|
@ -5845,7 +5849,8 @@ describe('createToolExecuteHandler', () => {
|
|||
])(
|
||||
'shares a protected edit retry horizon ($budget ms, preview $elapsed ms)',
|
||||
async ({ budget, elapsed, remaining }) => {
|
||||
let nowMs = Date.now();
|
||||
const startedAt = Date.now();
|
||||
let nowMs = startedAt;
|
||||
jest.spyOn(Date, 'now').mockImplementation(() => nowMs);
|
||||
const previewWorkspaceEdit = jest.fn(async () => {
|
||||
nowMs += elapsed;
|
||||
|
|
@ -5899,7 +5904,9 @@ describe('createToolExecuteHandler', () => {
|
|||
executionProfile: 'stateful',
|
||||
statefulSessions: true,
|
||||
environmentType: 'attached',
|
||||
codeEnvironmentConfigSchema: { limits: { maxQueueWaitMs: budget } },
|
||||
codeEnvironmentConfigSchema: {
|
||||
limits: { maxQueueWaitMs: budget, maxRequestTimeoutMs: 125_000 },
|
||||
},
|
||||
bridgeWorkerId: 'user-worker',
|
||||
},
|
||||
},
|
||||
|
|
@ -5918,7 +5925,11 @@ describe('createToolExecuteHandler', () => {
|
|||
]);
|
||||
|
||||
expect(previewWorkspaceEdit).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ maxQueueWaitMs: budget }),
|
||||
expect.objectContaining({
|
||||
maxQueueWaitMs: budget,
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
deadlineAtMs: startedAt + 125_000,
|
||||
}),
|
||||
);
|
||||
if (remaining == null) {
|
||||
expect(result.status).toBe('error');
|
||||
|
|
@ -5931,6 +5942,8 @@ describe('createToolExecuteHandler', () => {
|
|||
expect.objectContaining({
|
||||
expected_base_sha256: 'b'.repeat(64),
|
||||
maxQueueWaitMs: remaining,
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
deadlineAtMs: startedAt + 125_000,
|
||||
}),
|
||||
);
|
||||
},
|
||||
|
|
@ -6626,7 +6639,9 @@ describe('createToolExecuteHandler', () => {
|
|||
workspaceInstanceId: 'b'.repeat(64),
|
||||
operations: TEST_ATTACHED_WORKSPACE_OPERATIONS,
|
||||
},
|
||||
codeEnvironmentConfigSchema: { limits: { maxQueueWaitMs: 0 } },
|
||||
codeEnvironmentConfigSchema: {
|
||||
limits: { maxQueueWaitMs: 0, maxRequestTimeoutMs: 125_000 },
|
||||
},
|
||||
bridgeWorkerId: 'personal-worker-1',
|
||||
statefulSessions: true,
|
||||
},
|
||||
|
|
@ -6647,6 +6662,7 @@ describe('createToolExecuteHandler', () => {
|
|||
workspace_id: 'project-a',
|
||||
workspace_instance_id: 'b'.repeat(64),
|
||||
maxQueueWaitMs: 0,
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
start_line: 1,
|
||||
max_lines: 200,
|
||||
codeApiBaseUrl: 'https://code.example.com/v1',
|
||||
|
|
@ -6930,6 +6946,7 @@ describe('createToolExecuteHandler', () => {
|
|||
executionProfile: 'stateful',
|
||||
environmentType: 'attached',
|
||||
bridgeWorkerId: 'personal-worker-1',
|
||||
codeEnvironmentConfigSchema: { limits: { maxRequestTimeoutMs: 125_000 } },
|
||||
statefulSessions: true,
|
||||
},
|
||||
searchWorkspace,
|
||||
|
|
@ -6954,6 +6971,7 @@ describe('createToolExecuteHandler', () => {
|
|||
query: 'needle',
|
||||
workspace_id: 'project-a',
|
||||
maxQueueWaitMs: 300000,
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
path: 'src',
|
||||
max_results: 20,
|
||||
codeApiBaseUrl: 'https://code.example.com/v1',
|
||||
|
|
@ -7084,6 +7102,7 @@ describe('createToolExecuteHandler', () => {
|
|||
executionProfile: 'stateful',
|
||||
environmentType: 'attached',
|
||||
bridgeWorkerId: 'personal-worker-1',
|
||||
codeEnvironmentConfigSchema: { limits: { maxRequestTimeoutMs: 125_000 } },
|
||||
statefulSessions: true,
|
||||
},
|
||||
listWorkspaceFiles,
|
||||
|
|
@ -7107,6 +7126,7 @@ describe('createToolExecuteHandler', () => {
|
|||
expect(listWorkspaceFiles).toHaveBeenCalledWith({
|
||||
workspace_id: 'project-a',
|
||||
maxQueueWaitMs: 300000,
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
path: 'src',
|
||||
after_path: 'src/app.ts',
|
||||
max_results: 20,
|
||||
|
|
|
|||
|
|
@ -119,6 +119,10 @@ import {
|
|||
WORKSPACE_EDIT_MAX_COUNT,
|
||||
WORKSPACE_WRITE_MAX_BYTES,
|
||||
} from '~/code/workspace';
|
||||
import {
|
||||
resolveAttachedWorkspaceQueueWaitMs,
|
||||
resolveAttachedWorkspaceRequestTimeoutMs,
|
||||
} from '~/code/command';
|
||||
import {
|
||||
hasIntentArg,
|
||||
stripIntentArg,
|
||||
|
|
@ -127,7 +131,6 @@ import {
|
|||
} from './intent';
|
||||
import { buildSkillPrimeMessage, isSkillFilePath, SKILL_FILE_PREFIX } from './skills';
|
||||
import { resolveCallerCapabilityProjectionSnapshot } from './callerCapabilities';
|
||||
import { resolveAttachedWorkspaceQueueWaitMs } from '~/code/command';
|
||||
import { BACKGROUND_TOOL_INVOCATION_CONFIG_KEY } from './invocation';
|
||||
import { mergeCodeFilesIntoContext } from './codeFilesSession';
|
||||
import { toolValidationFeedback } from './validationFeedback';
|
||||
|
|
@ -568,6 +571,8 @@ export interface ToolExecuteOptions {
|
|||
req?: ServerRequest;
|
||||
signal?: AbortSignal;
|
||||
maxQueueWaitMs?: number;
|
||||
maxRequestTimeoutMs?: number;
|
||||
deadlineAtMs?: number;
|
||||
}) => Promise<WorkspaceReadResult>;
|
||||
/** Searches literal text within an attached worker's logical workspace. */
|
||||
searchWorkspace?: (params: {
|
||||
|
|
@ -582,6 +587,8 @@ export interface ToolExecuteOptions {
|
|||
req?: ServerRequest;
|
||||
signal?: AbortSignal;
|
||||
maxQueueWaitMs?: number;
|
||||
maxRequestTimeoutMs?: number;
|
||||
deadlineAtMs?: number;
|
||||
}) => Promise<WorkspaceSearchResult>;
|
||||
/** Lists relative file paths within an attached worker's logical workspace. */
|
||||
listWorkspaceFiles?: (params: {
|
||||
|
|
@ -596,6 +603,8 @@ export interface ToolExecuteOptions {
|
|||
req?: ServerRequest;
|
||||
signal?: AbortSignal;
|
||||
maxQueueWaitMs?: number;
|
||||
maxRequestTimeoutMs?: number;
|
||||
deadlineAtMs?: number;
|
||||
}) => Promise<WorkspaceListResult>;
|
||||
/** Writes a UTF-8 file within an attached worker's logical workspace. */
|
||||
writeWorkspaceFile?: (params: {
|
||||
|
|
@ -610,6 +619,8 @@ export interface ToolExecuteOptions {
|
|||
req?: ServerRequest;
|
||||
signal?: AbortSignal;
|
||||
maxQueueWaitMs?: number;
|
||||
maxRequestTimeoutMs?: number;
|
||||
deadlineAtMs?: number;
|
||||
}) => Promise<WorkspaceWriteResult>;
|
||||
/** Previews exact replacements without mutating an attached worker workspace. */
|
||||
previewWorkspaceEdit?: (params: {
|
||||
|
|
@ -623,6 +634,8 @@ export interface ToolExecuteOptions {
|
|||
req?: ServerRequest;
|
||||
signal?: AbortSignal;
|
||||
maxQueueWaitMs?: number;
|
||||
maxRequestTimeoutMs?: number;
|
||||
deadlineAtMs?: number;
|
||||
}) => Promise<WorkspacePreviewEditResult>;
|
||||
/** Applies exact replacements atomically within an attached worker workspace. */
|
||||
editWorkspaceFile?: (params: {
|
||||
|
|
@ -637,6 +650,8 @@ export interface ToolExecuteOptions {
|
|||
req?: ServerRequest;
|
||||
signal?: AbortSignal;
|
||||
maxQueueWaitMs?: number;
|
||||
maxRequestTimeoutMs?: number;
|
||||
deadlineAtMs?: number;
|
||||
}) => Promise<WorkspaceEditResult>;
|
||||
/**
|
||||
* Reads a code-execution sandbox file by shelling `cat` through the
|
||||
|
|
@ -2462,9 +2477,7 @@ async function handleWorkspaceFileRead(
|
|||
start_line: startLine,
|
||||
max_lines: maxLines,
|
||||
codeApiBaseUrl: codeExecutionContext.baseUrl,
|
||||
maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs(
|
||||
codeExecutionContext.codeEnvironmentConfigSchema,
|
||||
),
|
||||
...attachedWorkspaceRequestLimits(codeExecutionContext),
|
||||
executionProfile: codeExecutionContext.executionProfile,
|
||||
...(codeExecutionContext.bridgeWorkerId
|
||||
? { bridgeWorkerId: codeExecutionContext.bridgeWorkerId }
|
||||
|
|
@ -2570,9 +2583,7 @@ async function handleWorkspaceSearchCall(
|
|||
...(typeof args.path === 'string' && args.path.length > 0 ? { path: args.path } : {}),
|
||||
max_results: Number(maxResults),
|
||||
codeApiBaseUrl: codeExecutionContext.baseUrl,
|
||||
maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs(
|
||||
codeExecutionContext.codeEnvironmentConfigSchema,
|
||||
),
|
||||
...attachedWorkspaceRequestLimits(codeExecutionContext),
|
||||
executionProfile: codeExecutionContext.executionProfile,
|
||||
...(codeExecutionContext.bridgeWorkerId
|
||||
? { bridgeWorkerId: codeExecutionContext.bridgeWorkerId }
|
||||
|
|
@ -2663,9 +2674,7 @@ async function handleWorkspaceListCall(
|
|||
: {}),
|
||||
max_results: Number(maxResults),
|
||||
codeApiBaseUrl: codeExecutionContext.baseUrl,
|
||||
maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs(
|
||||
codeExecutionContext.codeEnvironmentConfigSchema,
|
||||
),
|
||||
...attachedWorkspaceRequestLimits(codeExecutionContext),
|
||||
executionProfile: codeExecutionContext.executionProfile,
|
||||
...(codeExecutionContext.bridgeWorkerId
|
||||
? { bridgeWorkerId: codeExecutionContext.bridgeWorkerId }
|
||||
|
|
@ -3910,6 +3919,18 @@ function attachedWorkspaceAuthoringPath(
|
|||
return pathError ? errorResult(tc, pathError) : { filePath: relativePath };
|
||||
}
|
||||
|
||||
function attachedWorkspaceRequestLimits(codeExecutionContext: CodeExecutionContext): {
|
||||
maxQueueWaitMs: number;
|
||||
maxRequestTimeoutMs?: number;
|
||||
} {
|
||||
const config = codeExecutionContext.codeEnvironmentConfigSchema;
|
||||
const maxRequestTimeoutMs = resolveAttachedWorkspaceRequestTimeoutMs(config);
|
||||
return {
|
||||
maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs(config),
|
||||
...(maxRequestTimeoutMs == null ? {} : { maxRequestTimeoutMs }),
|
||||
};
|
||||
}
|
||||
|
||||
function attachedWorkspaceMutationParams(
|
||||
codeExecutionContext: CodeExecutionContext,
|
||||
workspaceId: string,
|
||||
|
|
@ -3924,16 +3945,20 @@ function attachedWorkspaceMutationParams(
|
|||
req?: ServerRequest;
|
||||
signal?: AbortSignal;
|
||||
maxQueueWaitMs: number;
|
||||
maxRequestTimeoutMs?: number;
|
||||
deadlineAtMs?: number;
|
||||
} {
|
||||
const limits = attachedWorkspaceRequestLimits(codeExecutionContext);
|
||||
return {
|
||||
workspace_id: workspaceId,
|
||||
...(codeExecutionContext.codeWorkspace?.workspaceInstanceId
|
||||
? { workspace_instance_id: codeExecutionContext.codeWorkspace.workspaceInstanceId }
|
||||
: {}),
|
||||
codeApiBaseUrl: codeExecutionContext.baseUrl,
|
||||
maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs(
|
||||
codeExecutionContext.codeEnvironmentConfigSchema,
|
||||
),
|
||||
...limits,
|
||||
...(limits.maxRequestTimeoutMs == null
|
||||
? {}
|
||||
: { deadlineAtMs: Date.now() + limits.maxRequestTimeoutMs }),
|
||||
executionProfile: codeExecutionContext.executionProfile,
|
||||
...(codeExecutionContext.bridgeWorkerId
|
||||
? { bridgeWorkerId: codeExecutionContext.bridgeWorkerId }
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import {
|
|||
resolveAttachedWorkspaceCommandTimeoutMax,
|
||||
resolveAttachedWorkspaceProgrammaticTimeout,
|
||||
resolveAttachedWorkspaceQueueWaitMs,
|
||||
resolveAttachedWorkspaceRequestTimeoutMs,
|
||||
} from './command';
|
||||
import { BACKGROUND_TOOL_INVOCATION_CONFIG_KEY } from '~/agents/invocation';
|
||||
|
||||
|
|
@ -376,7 +377,7 @@ describe('createAttachedWorkspaceBashTool', () => {
|
|||
expect(foregroundRequest).toMatchObject({ timeoutMs: 30_000 });
|
||||
});
|
||||
|
||||
test('resolves the administrator-configured admission budget', () => {
|
||||
test('resolves the administrator-configured retry horizon', () => {
|
||||
expect(resolveAttachedWorkspaceQueueWaitMs()).toBe(5 * 60_000);
|
||||
expect(resolveAttachedWorkspaceQueueWaitMs({ limits: { maxQueueWaitMs: 30_000 } })).toBe(
|
||||
30_000,
|
||||
|
|
@ -387,6 +388,37 @@ describe('createAttachedWorkspaceBashTool', () => {
|
|||
);
|
||||
});
|
||||
|
||||
test('opts Bash into a verified HTTP limit without changing execution or disabling the initial attempt', async () => {
|
||||
jest.spyOn(Date, 'now').mockReturnValue(1_000);
|
||||
expect(resolveAttachedWorkspaceRequestTimeoutMs()).toBeUndefined();
|
||||
expect(
|
||||
resolveAttachedWorkspaceRequestTimeoutMs({ limits: { maxRequestTimeoutMs: 125_000 } }),
|
||||
).toBe(125_000);
|
||||
expect(
|
||||
resolveAttachedWorkspaceRequestTimeoutMs({ limits: { maxRequestTimeoutMs: 610_001 } }),
|
||||
).toBeUndefined();
|
||||
const fetchImpl: CodeBridgeFetch = jest.fn(async () => commandResponse());
|
||||
const bashTool = createAttachedWorkspaceBashTool({
|
||||
baseUrl: 'https://code.example.com/v1',
|
||||
authHeaders: () => ({}),
|
||||
workspaceId: 'project-a',
|
||||
maxTimeoutMs: 90_000,
|
||||
maxQueueWaitMs: 0,
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
fetchImpl,
|
||||
});
|
||||
|
||||
await bashTool.func({ command: 'sleep 90', timeoutMs: 90_000 }, undefined, {});
|
||||
await bashTool.func({ command: 'pwd' }, undefined, {});
|
||||
|
||||
const [, long] = (fetchImpl as jest.Mock).mock.calls[0];
|
||||
const [, short] = (fetchImpl as jest.Mock).mock.calls[1];
|
||||
expect(JSON.parse(long.body).timeoutMs).toBe(90_000);
|
||||
expect(JSON.parse(short.body).timeoutMs).toBe(30_000);
|
||||
expect(long.headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe('25000');
|
||||
expect(short.headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe('85000');
|
||||
});
|
||||
|
||||
test('lowers the omitted timeout when the deployment ceiling is below 30 seconds', async () => {
|
||||
const fetchImpl: CodeBridgeFetch = jest.fn(async () => commandResponse());
|
||||
const bashTool = createAttachedWorkspaceBashTool({
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { logger } from '@librechat/data-schemas';
|
||||
import { tool } from '@librechat/agents/langchain/tools';
|
||||
import { CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS } from 'librechat-data-provider';
|
||||
import {
|
||||
BashExecutionToolDefinition,
|
||||
BashToolOutputReferencesGuide,
|
||||
|
|
@ -140,6 +141,21 @@ export function resolveAttachedWorkspaceQueueWaitMs(
|
|||
return Math.min(WORKSPACE_QUEUE_MAX_WAIT_MS, configured);
|
||||
}
|
||||
|
||||
export function resolveAttachedWorkspaceRequestTimeoutMs(
|
||||
configSchema?: CodeEnvironmentUserConfigSchema,
|
||||
): number | undefined {
|
||||
const configured = configSchema?.limits?.maxRequestTimeoutMs;
|
||||
if (
|
||||
configured == null ||
|
||||
!Number.isSafeInteger(configured) ||
|
||||
configured < 1 ||
|
||||
configured > CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
return configured;
|
||||
}
|
||||
|
||||
export function buildAttachedWorkspaceBashSchema(
|
||||
maxTimeoutMs: number = WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS,
|
||||
environment?: CodeWorkspaceDescriptor['environment'],
|
||||
|
|
@ -286,6 +302,7 @@ export function createAttachedWorkspaceBashTool({
|
|||
gitIdentity,
|
||||
maxTimeoutMs = WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS,
|
||||
maxQueueWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
fetchImpl,
|
||||
}: {
|
||||
baseUrl: string;
|
||||
|
|
@ -296,8 +313,10 @@ export function createAttachedWorkspaceBashTool({
|
|||
gitIdentity?: AgentGitIdentity | null;
|
||||
/** Effective admin/upstream ceiling already intersected with the protocol hard cap. */
|
||||
maxTimeoutMs?: number;
|
||||
/** Deployment admission budget; omitted keeps the built-in default. */
|
||||
/** Retry horizon across typed queue expirations, not an admission budget. */
|
||||
maxQueueWaitMs?: number;
|
||||
/** Verified total HTTP budget; omission keeps the legacy per-attempt timeout. */
|
||||
maxRequestTimeoutMs?: number;
|
||||
fetchImpl?: CodeBridgeFetch;
|
||||
}): DynamicStructuredTool {
|
||||
const effectiveMaxTimeoutMs = normalizeAttachedWorkspaceCommandTimeoutMax(maxTimeoutMs);
|
||||
|
|
@ -376,6 +395,7 @@ export function createAttachedWorkspaceBashTool({
|
|||
signal,
|
||||
fetchImpl,
|
||||
...(maxQueueWaitMs == null ? {} : { maxQueueWaitMs }),
|
||||
...(maxRequestTimeoutMs == null ? {} : { maxRequestTimeoutMs }),
|
||||
});
|
||||
if (result.operation !== 'execute_command') {
|
||||
throw new Error('Attached workspace returned an unexpected command result.');
|
||||
|
|
|
|||
|
|
@ -242,6 +242,37 @@ describe('workspace admission feedback', () => {
|
|||
},
|
||||
);
|
||||
|
||||
test('cancels a stalled credential refresh without opening a workspace admission window', async () => {
|
||||
const controller = new AbortController();
|
||||
const reason = new DOMException('Chat stopped', 'AbortError');
|
||||
const fetchImpl = jest.fn();
|
||||
let entered!: () => void;
|
||||
const started = new Promise<void>((resolve) => {
|
||||
entered = resolve;
|
||||
});
|
||||
const operation = executeWorkspaceTool({
|
||||
baseURL: 'https://code.example.com/v1',
|
||||
authHeaders: async () => {
|
||||
entered();
|
||||
return new Promise<Record<string, string>>(() => {});
|
||||
},
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'edit_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'src/app.ts',
|
||||
edits: [{ oldText: 'a', newText: 'b' }],
|
||||
},
|
||||
signal: controller.signal,
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
fetchImpl,
|
||||
});
|
||||
await started;
|
||||
controller.abort(reason);
|
||||
await expect(operation).rejects.toBe(reason);
|
||||
expect(fetchImpl).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('does not dispatch a retry if credential refresh spends the remaining queue budget', async () => {
|
||||
const now = jest.spyOn(Date, 'now').mockReturnValue(1000);
|
||||
const authHeaders = jest
|
||||
|
|
@ -438,6 +469,287 @@ describe('executeWorkspaceTool', () => {
|
|||
).rejects.toMatchObject({ upstreamStatus: 504 });
|
||||
expect(timeout).toHaveBeenCalledWith(budget);
|
||||
expect(fetchImpl.mock.calls[0][1].body).toBe(JSON.stringify(request));
|
||||
const admissionMs = Number(
|
||||
fetchImpl.mock.calls[0][1].headers['X-LibreChat-Workspace-Queue-Wait-Ms'],
|
||||
);
|
||||
expect(admissionMs).toBeGreaterThan(0);
|
||||
expect(admissionMs).toBeLessThanOrEqual(30_000);
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test.each<[WorkspaceToolRequest, number, number]>([
|
||||
[
|
||||
{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' },
|
||||
125_000,
|
||||
90_000,
|
||||
],
|
||||
[
|
||||
{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'pwd' },
|
||||
125_000,
|
||||
85_000,
|
||||
],
|
||||
[
|
||||
{
|
||||
protocolVersion: 1,
|
||||
operation: 'execute_command',
|
||||
workspaceId: 'primary',
|
||||
command: 'pwd',
|
||||
timeoutMs: 90_000,
|
||||
},
|
||||
125_000,
|
||||
25_000,
|
||||
],
|
||||
[
|
||||
{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' },
|
||||
60_000,
|
||||
25_000,
|
||||
],
|
||||
[
|
||||
{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'pwd' },
|
||||
60_000,
|
||||
20_000,
|
||||
],
|
||||
[
|
||||
{
|
||||
protocolVersion: 1,
|
||||
operation: 'execute_command',
|
||||
workspaceId: 'primary',
|
||||
command: 'pwd',
|
||||
timeoutMs: 300_000,
|
||||
},
|
||||
610_000,
|
||||
300_000,
|
||||
],
|
||||
])(
|
||||
'advertises only the remaining queue time for %j under a %i ms limit',
|
||||
async (request, maxRequestTimeoutMs, allowanceMs) => {
|
||||
jest.spyOn(Date, 'now').mockReturnValue(1_000);
|
||||
const timeout = jest.spyOn(AbortSignal, 'timeout');
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockResolvedValue(
|
||||
new Response(JSON.stringify({ code: 'ASSIGNMENT_EXPIRED' }), { status: 504 }),
|
||||
);
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example.com/v1',
|
||||
authHeaders: {},
|
||||
request,
|
||||
maxRequestTimeoutMs,
|
||||
fetchImpl,
|
||||
}),
|
||||
).rejects.toMatchObject({ upstreamStatus: 504 });
|
||||
expect(timeout).toHaveBeenCalledWith(maxRequestTimeoutMs);
|
||||
expect(fetchImpl.mock.calls[0][1].headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe(
|
||||
String(allowanceMs),
|
||||
);
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
},
|
||||
);
|
||||
|
||||
test('accepts a result after 31 simulated seconds of admission with an explicit longer budget', async () => {
|
||||
const now = jest.spyOn(Date, 'now').mockReturnValue(1_000);
|
||||
const fetchImpl: CodeBridgeFetch = jest.fn(async (_url, init) => {
|
||||
expect((init?.headers as Record<string, string>)['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe(
|
||||
'90000',
|
||||
);
|
||||
now.mockReturnValue(32_000);
|
||||
expect(init?.signal?.aborted).toBe(false);
|
||||
return Response.json({
|
||||
protocolVersion: 1,
|
||||
operation: 'read_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'README.md',
|
||||
content: 'ready',
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
truncated: false,
|
||||
});
|
||||
});
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example.com/v1',
|
||||
authHeaders: {},
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'read_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'README.md',
|
||||
},
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
fetchImpl,
|
||||
}),
|
||||
).resolves.toMatchObject({ content: 'ready' });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('uses the earlier caller deadline rather than assuming the abort signal reveals it', async () => {
|
||||
jest.spyOn(Date, 'now').mockReturnValue(1_000);
|
||||
const timeout = jest.spyOn(AbortSignal, 'timeout');
|
||||
const fetchImpl = jest.fn().mockResolvedValue(new Response('{}', { status: 504 }));
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example.com/v1',
|
||||
authHeaders: {},
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'read_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'README.md',
|
||||
},
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
deadlineAtMs: 61_000,
|
||||
fetchImpl,
|
||||
}),
|
||||
).rejects.toMatchObject({ upstreamStatus: 504 });
|
||||
expect(timeout).toHaveBeenCalledWith(60_000);
|
||||
expect(fetchImpl.mock.calls[0][1].headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe('25000');
|
||||
});
|
||||
|
||||
test('subtracts credential acquisition from the allowance before dispatching', async () => {
|
||||
const now = jest.spyOn(Date, 'now').mockReturnValue(1_000);
|
||||
const fetchImpl = jest.fn().mockResolvedValue(new Response('{}', { status: 504 }));
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example.com/v1',
|
||||
authHeaders: async () => {
|
||||
now.mockReturnValue(20_000);
|
||||
return {};
|
||||
},
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'read_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'README.md',
|
||||
},
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
fetchImpl,
|
||||
}),
|
||||
).rejects.toMatchObject({ upstreamStatus: 504 });
|
||||
expect(fetchImpl.mock.calls[0][1].headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe('71000');
|
||||
});
|
||||
|
||||
test.each<[WorkspaceToolRequest, number]>([
|
||||
[
|
||||
{
|
||||
protocolVersion: 1,
|
||||
operation: 'execute_command',
|
||||
workspaceId: 'primary',
|
||||
command: 'pwd',
|
||||
timeoutMs: 300_000,
|
||||
},
|
||||
125_000,
|
||||
],
|
||||
[
|
||||
{
|
||||
protocolVersion: 1,
|
||||
operation: 'execute_command',
|
||||
workspaceId: 'primary',
|
||||
command: 'pwd',
|
||||
timeoutMs: 90_000,
|
||||
},
|
||||
60_000,
|
||||
],
|
||||
])(
|
||||
'does not dispatch when %j cannot fit within the %i ms ingress limit',
|
||||
async (request, maxRequestTimeoutMs) => {
|
||||
const fetchImpl = jest.fn();
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example.com/v1',
|
||||
authHeaders: {},
|
||||
request,
|
||||
maxRequestTimeoutMs,
|
||||
fetchImpl,
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
reason: 'insufficient_time',
|
||||
message: expect.stringContaining('not started'),
|
||||
});
|
||||
expect(fetchImpl).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
test('keeps the retry horizon separate from one longer admission attempt', async () => {
|
||||
jest.spyOn(Date, 'now').mockReturnValue(1_000);
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockResolvedValue(
|
||||
new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), { status: 503 }),
|
||||
);
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example.com/v1',
|
||||
authHeaders: {},
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'read_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'README.md',
|
||||
},
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
maxQueueWaitMs: 0,
|
||||
fetchImpl,
|
||||
}),
|
||||
).rejects.toThrow('The operation was not started');
|
||||
expect(fetchImpl.mock.calls[0][1].headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe('90000');
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('keeps Retry-After capped at 30 seconds even when one admission may wait longer', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), {
|
||||
status: 503,
|
||||
headers: { 'Retry-After': '300' },
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(new Response('{}', { status: 504 }));
|
||||
const result = executeWorkspaceTool({
|
||||
baseURL: 'https://code.example.com/v1',
|
||||
authHeaders: {},
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'read_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'README.md',
|
||||
},
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
fetchImpl,
|
||||
}).catch((error: WorkspaceToolHttpError) => error);
|
||||
await jest.advanceTimersByTimeAsync(29_999);
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
await expect(result).resolves.toMatchObject({ upstreamStatus: 504 });
|
||||
expect(fetchImpl.mock.calls[1][1].headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe(
|
||||
'60000',
|
||||
);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test('never retries a post-dispatch timeout with an unknown mutation outcome', async () => {
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockRejectedValue(new DOMException('response deadline', 'TimeoutError'));
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example.com/v1',
|
||||
authHeaders: {},
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'execute_command',
|
||||
workspaceId: 'primary',
|
||||
command: 'touch marker',
|
||||
},
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
fetchImpl,
|
||||
}),
|
||||
).rejects.toMatchObject({ reason: 'timeout' });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,8 @@
|
|||
import { CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS } from 'librechat-data-provider';
|
||||
import {
|
||||
CODE_ENVIRONMENT_ADMISSION_MAX_MS,
|
||||
CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS,
|
||||
CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS,
|
||||
} from 'librechat-data-provider';
|
||||
import type { CodeBridgeFetch } from './bridge';
|
||||
|
||||
const WORKSPACE_TOOL_TIMEOUT_MS = 30_000;
|
||||
|
|
@ -19,8 +23,9 @@ const DEFAULT_COMMAND_OUTPUT_BYTES = 256 * 1024;
|
|||
const MAX_COMMAND_OUTPUT_BYTES = 1024 * 1024;
|
||||
const MAX_COMMAND_SIGNAL_LENGTH = 32;
|
||||
const WORKSPACE_COMMAND_TRANSPORT_GRACE_MS = 5_000;
|
||||
/** Matches Code API's bounded admission wait and command settlement allowance. */
|
||||
/** Legacy admission window and Retry-After cap, independent of the retry horizon. */
|
||||
const WORKSPACE_QUEUE_TIMEOUT_MS = 30_000;
|
||||
const WORKSPACE_QUEUE_WAIT_HEADER = 'X-LibreChat-Workspace-Queue-Wait-Ms';
|
||||
/** Compatibility export; the deployment schema owns the default and hard cap. */
|
||||
export const WORKSPACE_QUEUE_MAX_WAIT_MS: number = CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS;
|
||||
const WORKSPACE_QUEUE_RETRY_DELAY_MS = 1_000;
|
||||
|
|
@ -263,16 +268,25 @@ export type WorkspaceToolResult =
|
|||
|
||||
export class WorkspaceToolHttpError extends Error {
|
||||
constructor(
|
||||
public readonly reason: 'rejected' | 'invalid' | 'timeout' | 'failed',
|
||||
public readonly reason: 'rejected' | 'invalid' | 'timeout' | 'failed' | 'insufficient_time',
|
||||
public readonly upstreamStatus?: number,
|
||||
public readonly upstreamBody?: string,
|
||||
public readonly upstreamBodyTruncated = false,
|
||||
) {
|
||||
super(
|
||||
(reason === 'rejected' &&
|
||||
let message = `Workspace tool request ${reason}`;
|
||||
if (
|
||||
reason === 'rejected' &&
|
||||
isWorkspaceAdmissionTimeout(upstreamStatus, upstreamBody, upstreamBodyTruncated)
|
||||
? 'Workspace capacity was unavailable before the queue deadline. The operation was not started. Wait for active work to finish or select an independent workspace on a machine with available capacity.'
|
||||
: `Workspace tool request ${reason}`) +
|
||||
) {
|
||||
message =
|
||||
'Workspace capacity was unavailable before the queue deadline. The operation was not started. Wait for active work to finish or select an independent workspace on a machine with available capacity.';
|
||||
}
|
||||
if (reason === 'insufficient_time') {
|
||||
message =
|
||||
'Workspace execution cannot fit within the remaining HTTP budget. The operation was not started.';
|
||||
}
|
||||
super(
|
||||
message +
|
||||
(upstreamStatus == null ? '' : ` (upstreamStatus: ${upstreamStatus})`) +
|
||||
(upstreamBody ? `; upstreamBody: ${JSON.stringify(upstreamBody)}` : '') +
|
||||
(upstreamBodyTruncated ? ' [body truncated or incomplete]' : ''),
|
||||
|
|
@ -731,13 +745,19 @@ function isValidResult(
|
|||
);
|
||||
}
|
||||
|
||||
function getWorkspaceToolTimeoutMs(request: WorkspaceToolRequest): number {
|
||||
const executionBudgetMs =
|
||||
request.operation === 'execute_command'
|
||||
? (request.timeoutMs ?? WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS) +
|
||||
function getWorkspaceExecutionBudgetMs(request: WorkspaceToolRequest): number {
|
||||
return request.operation === 'execute_command'
|
||||
? (request.timeoutMs ?? WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS) +
|
||||
WORKSPACE_COMMAND_SETTLEMENT_GRACE_MS
|
||||
: WORKSPACE_TOOL_TIMEOUT_MS;
|
||||
return WORKSPACE_QUEUE_TIMEOUT_MS + executionBudgetMs + WORKSPACE_COMMAND_TRANSPORT_GRACE_MS;
|
||||
: WORKSPACE_TOOL_TIMEOUT_MS;
|
||||
}
|
||||
|
||||
function getWorkspaceToolTimeoutMs(request: WorkspaceToolRequest): number {
|
||||
return (
|
||||
WORKSPACE_QUEUE_TIMEOUT_MS +
|
||||
getWorkspaceExecutionBudgetMs(request) +
|
||||
WORKSPACE_COMMAND_TRANSPORT_GRACE_MS
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -749,6 +769,31 @@ export type WorkspaceToolAuthHeaders =
|
|||
| Record<string, string>
|
||||
| (() => Promise<Record<string, string>> | Record<string, string>);
|
||||
|
||||
function getWorkspaceAuthHeaders(
|
||||
supplier: () => Promise<Record<string, string>> | Record<string, string>,
|
||||
signal: AbortSignal,
|
||||
): Promise<Record<string, string>> {
|
||||
signal.throwIfAborted();
|
||||
return new Promise((resolve, reject) => {
|
||||
const abort = () => reject(signal.reason);
|
||||
signal.addEventListener('abort', abort, { once: true });
|
||||
if (signal.aborted) {
|
||||
abort();
|
||||
return;
|
||||
}
|
||||
try {
|
||||
void Promise.resolve(supplier())
|
||||
.then(resolve, reject)
|
||||
.finally(() => {
|
||||
signal.removeEventListener('abort', abort);
|
||||
});
|
||||
} catch (error) {
|
||||
signal.removeEventListener('abort', abort);
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export async function executeWorkspaceTool({
|
||||
baseURL,
|
||||
authHeaders,
|
||||
|
|
@ -756,6 +801,8 @@ export async function executeWorkspaceTool({
|
|||
signal,
|
||||
fetchImpl = fetch,
|
||||
maxQueueWaitMs = WORKSPACE_QUEUE_MAX_WAIT_MS,
|
||||
maxRequestTimeoutMs,
|
||||
deadlineAtMs,
|
||||
}: {
|
||||
baseURL: string;
|
||||
authHeaders: WorkspaceToolAuthHeaders;
|
||||
|
|
@ -763,34 +810,81 @@ export async function executeWorkspaceTool({
|
|||
signal?: AbortSignal;
|
||||
fetchImpl?: CodeBridgeFetch;
|
||||
maxQueueWaitMs?: number;
|
||||
/** End-to-end HTTP budget for this call. Omission keeps legacy per-attempt timeouts. */
|
||||
maxRequestTimeoutMs?: number;
|
||||
/** Optional earlier caller deadline; a signal alone has no remaining-time value. */
|
||||
deadlineAtMs?: number;
|
||||
}): Promise<WorkspaceToolResult> {
|
||||
if (
|
||||
!isValidRequest(request) ||
|
||||
!Number.isSafeInteger(maxQueueWaitMs) ||
|
||||
maxQueueWaitMs < 0 ||
|
||||
maxQueueWaitMs > WORKSPACE_QUEUE_MAX_WAIT_MS
|
||||
maxQueueWaitMs > WORKSPACE_QUEUE_MAX_WAIT_MS ||
|
||||
(maxRequestTimeoutMs !== undefined &&
|
||||
(!Number.isSafeInteger(maxRequestTimeoutMs) ||
|
||||
maxRequestTimeoutMs < 1 ||
|
||||
maxRequestTimeoutMs > CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS)) ||
|
||||
(deadlineAtMs !== undefined && (!Number.isSafeInteger(deadlineAtMs) || deadlineAtMs < 1))
|
||||
) {
|
||||
throw new WorkspaceToolHttpError('invalid');
|
||||
}
|
||||
const executionBudgetMs = getWorkspaceExecutionBudgetMs(request);
|
||||
const completionReserveMs = executionBudgetMs + WORKSPACE_COMMAND_TRANSPORT_GRACE_MS;
|
||||
const perAttemptTimeoutMs = maxRequestTimeoutMs ?? getWorkspaceToolTimeoutMs(request);
|
||||
const callerDeadlineAt = Math.min(
|
||||
deadlineAtMs ?? Infinity,
|
||||
maxRequestTimeoutMs == null ? Infinity : Date.now() + maxRequestTimeoutMs,
|
||||
);
|
||||
const queueDeadlineAt = Date.now() + maxQueueWaitMs;
|
||||
const retryDeadlineAt = Math.min(queueDeadlineAt, callerDeadlineAt - completionReserveMs);
|
||||
const body = JSON.stringify(request);
|
||||
let lastAdmissionRejection: WorkspaceToolHttpError | undefined;
|
||||
while (true) {
|
||||
try {
|
||||
signal?.throwIfAborted();
|
||||
/** The endpoint replies only after execution settles. Capping this by the
|
||||
* retry horizon would also abort already-admitted commands, with an unknown
|
||||
* mutation outcome. Server admission and execution retain separate budgets. */
|
||||
const timeoutSignal = AbortSignal.timeout(getWorkspaceToolTimeoutMs(request));
|
||||
/** Admission and retries cannot consume the reserved execution or delivery time. */
|
||||
const attemptTimeoutMs = Math.floor(
|
||||
Math.min(perAttemptTimeoutMs, callerDeadlineAt - Date.now()),
|
||||
);
|
||||
if (attemptTimeoutMs <= completionReserveMs) {
|
||||
throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time');
|
||||
}
|
||||
const attemptStartedAt = Date.now();
|
||||
const timeoutSignal = AbortSignal.timeout(attemptTimeoutMs);
|
||||
const requestSignal =
|
||||
signal != null && typeof AbortSignal.any === 'function'
|
||||
? AbortSignal.any([signal, timeoutSignal])
|
||||
: timeoutSignal;
|
||||
const attemptHeaders = typeof authHeaders === 'function' ? await authHeaders() : authHeaders;
|
||||
requestSignal.throwIfAborted();
|
||||
if (lastAdmissionRejection && Date.now() >= queueDeadlineAt) {
|
||||
let attemptHeaders: Record<string, string>;
|
||||
try {
|
||||
attemptHeaders =
|
||||
typeof authHeaders === 'function'
|
||||
? await getWorkspaceAuthHeaders(authHeaders, requestSignal)
|
||||
: authHeaders;
|
||||
} catch (error) {
|
||||
if (timeoutSignal.aborted && signal?.aborted !== true) {
|
||||
throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
signal?.throwIfAborted();
|
||||
if (timeoutSignal.aborted) {
|
||||
throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time');
|
||||
}
|
||||
if (lastAdmissionRejection && Date.now() >= retryDeadlineAt) {
|
||||
throw lastAdmissionRejection;
|
||||
}
|
||||
const remainingMs = Math.min(
|
||||
attemptTimeoutMs - (Date.now() - attemptStartedAt),
|
||||
callerDeadlineAt - Date.now(),
|
||||
);
|
||||
const queueAllowanceMs = Math.min(
|
||||
CODE_ENVIRONMENT_ADMISSION_MAX_MS,
|
||||
Math.floor(remainingMs - completionReserveMs),
|
||||
);
|
||||
if (queueAllowanceMs < 1) {
|
||||
throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time');
|
||||
}
|
||||
const response = await fetchImpl(
|
||||
`${baseURL.trim().replace(/\/+$/, '')}/workspace-tools/execute`,
|
||||
{
|
||||
|
|
@ -798,6 +892,7 @@ export async function executeWorkspaceTool({
|
|||
headers: {
|
||||
...attemptHeaders,
|
||||
'Content-Type': 'application/json',
|
||||
[WORKSPACE_QUEUE_WAIT_HEADER]: String(queueAllowanceMs),
|
||||
},
|
||||
body,
|
||||
redirect: 'error',
|
||||
|
|
@ -816,13 +911,14 @@ export async function executeWorkspaceTool({
|
|||
}
|
||||
const delayMs = Math.min(
|
||||
workspaceAdmissionRetryDelay(response.headers.get('Retry-After')),
|
||||
queueDeadlineAt - Date.now(),
|
||||
retryDeadlineAt - Date.now(),
|
||||
);
|
||||
lastAdmissionRejection = rejection;
|
||||
if (delayMs <= 0) throw rejection;
|
||||
await waitForWorkspaceAdmission(delayMs, signal);
|
||||
/** A clamped delay can land exactly on the deadline: the budget is spent,
|
||||
* so never open another admission window that could still be admitted. */
|
||||
if (Date.now() >= queueDeadlineAt) {
|
||||
/** A clamped delay can land exactly on either deadline. Never open
|
||||
* another admission window without the full execution reserve. */
|
||||
if (Date.now() >= retryDeadlineAt) {
|
||||
throw rejection;
|
||||
}
|
||||
continue;
|
||||
|
|
@ -840,7 +936,7 @@ export async function executeWorkspaceTool({
|
|||
) {
|
||||
throw error;
|
||||
}
|
||||
if (error instanceof Error && error.name === 'TimeoutError') {
|
||||
if (isRecord(error) && error.name === 'TimeoutError') {
|
||||
throw new WorkspaceToolHttpError('timeout');
|
||||
}
|
||||
throw new WorkspaceToolHttpError('failed');
|
||||
|
|
|
|||
|
|
@ -561,6 +561,24 @@ describe('attached code environment user config schema', () => {
|
|||
},
|
||||
);
|
||||
|
||||
it.each([60_000, 125_000, 610_000])(
|
||||
'accepts a bounded %i ms workspace HTTP limit',
|
||||
(maxRequestTimeoutMs) => {
|
||||
expect(codeEnvironmentUserConfigSchema.parse({ limits: { maxRequestTimeoutMs } })).toEqual({
|
||||
limits: { maxRequestTimeoutMs },
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each([0, -1, 0.5, 610_001, NaN, Infinity])(
|
||||
'rejects an invalid workspace HTTP limit of %s',
|
||||
(maxRequestTimeoutMs) => {
|
||||
expect(
|
||||
codeEnvironmentUserConfigSchema.safeParse({ limits: { maxRequestTimeoutMs } }).success,
|
||||
).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it('keeps an omitted admission budget backward compatible', () => {
|
||||
expect(codeEnvironmentUserConfigSchema.parse({ limits: {} })).toEqual({ limits: {} });
|
||||
});
|
||||
|
|
@ -584,7 +602,7 @@ describe('attached code environment user config schema', () => {
|
|||
fileWrite: { allowed: ['allow', 'ask', 'deny'], default: 'ask' },
|
||||
commandExecution: { allowed: ['ask', 'deny'], default: 'ask' },
|
||||
},
|
||||
limits: { maxCommandTimeoutMs: 120000 },
|
||||
limits: { maxCommandTimeoutMs: 120000, maxRequestTimeoutMs: 125_000 },
|
||||
},
|
||||
},
|
||||
],
|
||||
|
|
@ -597,6 +615,19 @@ describe('attached code environment user config schema', () => {
|
|||
throw new Error(result.error.toString());
|
||||
}
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).toMatchObject({
|
||||
endpoints: {
|
||||
agents: {
|
||||
statefulCodeSessions: {
|
||||
environments: [
|
||||
{
|
||||
configSchema: { limits: { maxRequestTimeoutMs: 125_000 } },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects an attached command timeout above the protocol hard cap', () => {
|
||||
|
|
|
|||
|
|
@ -1205,6 +1205,11 @@ export const CODE_ENVIRONMENT_COMMAND_TIMEOUT_HARD_MAX_MS = 5 * 60_000;
|
|||
* initial admission wait or an already-admitted operation's execution budget.
|
||||
*/
|
||||
export const CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS = 5 * 60_000;
|
||||
/** Code API's per-request admission ceiling, independent of the retry horizon. */
|
||||
export const CODE_ENVIRONMENT_ADMISSION_MAX_MS = 5 * 60_000;
|
||||
/** Maximum opt-in HTTP budget: five minutes of admission and execution plus ten seconds for settlement and delivery. */
|
||||
export const CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS =
|
||||
CODE_ENVIRONMENT_ADMISSION_MAX_MS + CODE_ENVIRONMENT_COMMAND_TIMEOUT_HARD_MAX_MS + 10_000;
|
||||
|
||||
/**
|
||||
* Typed user-tunable surface for one attached code environment. Omitted fields
|
||||
|
|
@ -1240,6 +1245,17 @@ export const codeEnvironmentUserConfigSchema = z
|
|||
.min(0)
|
||||
.max(CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS)
|
||||
.optional(),
|
||||
/** Total HTTP budget for one workspace tool call, including retries,
|
||||
* execution, settlement, and delivery. Only set this after verifying
|
||||
* the shortest timeout on the actual Code API path and updating Code API
|
||||
* to honor per-request queue allowances. Omission keeps the 30-second
|
||||
* per-attempt admission budget. */
|
||||
maxRequestTimeoutMs: z
|
||||
.number()
|
||||
.int()
|
||||
.min(1)
|
||||
.max(CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS)
|
||||
.optional(),
|
||||
})
|
||||
.strict()
|
||||
.optional(),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue