mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-17 05:20:55 +00:00
🪀 fix: Separate Workspace Admission and Transport Budgets (#16728)
* fix: Separate workspace admission and transport budgets * fix: Validate workspace run reserves and retry test types --------- Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
parent
6298bfc920
commit
5674faaa01
12 changed files with 712 additions and 160 deletions
|
|
@ -1092,6 +1092,8 @@ async function readWorkspaceFile({
|
|||
signal,
|
||||
maxQueueWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
maxRunTimeoutMs,
|
||||
admission,
|
||||
deadlineAtMs,
|
||||
}) {
|
||||
return executeWorkspaceTool({
|
||||
|
|
@ -1100,6 +1102,8 @@ async function readWorkspaceFile({
|
|||
maxQueueWaitMs,
|
||||
codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
...(maxRunTimeoutMs == null ? {} : { maxRunTimeoutMs }),
|
||||
...(admission == null ? {} : { admission }),
|
||||
deadlineAtMs,
|
||||
/** Minted per admission attempt: a queued call outlives one token TTL. */
|
||||
authHeaders: async () => ({
|
||||
|
|
@ -1149,6 +1153,8 @@ async function searchWorkspace({
|
|||
signal,
|
||||
maxQueueWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
maxRunTimeoutMs,
|
||||
admission,
|
||||
deadlineAtMs,
|
||||
}) {
|
||||
return executeWorkspaceTool({
|
||||
|
|
@ -1157,6 +1163,8 @@ async function searchWorkspace({
|
|||
maxQueueWaitMs,
|
||||
codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
...(maxRunTimeoutMs == null ? {} : { maxRunTimeoutMs }),
|
||||
...(admission == null ? {} : { admission }),
|
||||
deadlineAtMs,
|
||||
/** Minted per admission attempt: a queued call outlives one token TTL. */
|
||||
authHeaders: async () => ({
|
||||
|
|
@ -1206,6 +1214,8 @@ async function listWorkspaceFiles({
|
|||
signal,
|
||||
maxQueueWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
maxRunTimeoutMs,
|
||||
admission,
|
||||
deadlineAtMs,
|
||||
}) {
|
||||
return executeWorkspaceTool({
|
||||
|
|
@ -1214,6 +1224,8 @@ async function listWorkspaceFiles({
|
|||
maxQueueWaitMs,
|
||||
codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
...(maxRunTimeoutMs == null ? {} : { maxRunTimeoutMs }),
|
||||
...(admission == null ? {} : { admission }),
|
||||
deadlineAtMs,
|
||||
/** Minted per admission attempt: a queued call outlives one token TTL. */
|
||||
authHeaders: async () => ({
|
||||
|
|
@ -1248,6 +1260,8 @@ async function writeWorkspaceFile({
|
|||
signal,
|
||||
maxQueueWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
maxRunTimeoutMs,
|
||||
admission,
|
||||
deadlineAtMs,
|
||||
}) {
|
||||
return executeWorkspaceTool({
|
||||
|
|
@ -1256,6 +1270,8 @@ async function writeWorkspaceFile({
|
|||
maxQueueWaitMs,
|
||||
codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
...(maxRunTimeoutMs == null ? {} : { maxRunTimeoutMs }),
|
||||
...(admission == null ? {} : { admission }),
|
||||
deadlineAtMs,
|
||||
/** Minted per admission attempt: a queued call outlives one token TTL. */
|
||||
authHeaders: async () => ({
|
||||
|
|
@ -1291,6 +1307,8 @@ async function editWorkspaceFile({
|
|||
signal,
|
||||
maxQueueWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
maxRunTimeoutMs,
|
||||
admission,
|
||||
deadlineAtMs,
|
||||
}) {
|
||||
return executeWorkspaceTool({
|
||||
|
|
@ -1299,6 +1317,8 @@ async function editWorkspaceFile({
|
|||
maxQueueWaitMs,
|
||||
codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
...(maxRunTimeoutMs == null ? {} : { maxRunTimeoutMs }),
|
||||
...(admission == null ? {} : { admission }),
|
||||
deadlineAtMs,
|
||||
/** Minted per admission attempt: a queued call outlives one token TTL. */
|
||||
authHeaders: async () => ({
|
||||
|
|
@ -1334,6 +1354,8 @@ async function previewWorkspaceEdit({
|
|||
signal,
|
||||
maxQueueWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
maxRunTimeoutMs,
|
||||
admission,
|
||||
deadlineAtMs,
|
||||
}) {
|
||||
return executeWorkspaceTool({
|
||||
|
|
@ -1342,6 +1364,8 @@ async function previewWorkspaceEdit({
|
|||
maxQueueWaitMs,
|
||||
codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
...(maxRunTimeoutMs == null ? {} : { maxRunTimeoutMs }),
|
||||
...(admission == null ? {} : { admission }),
|
||||
deadlineAtMs,
|
||||
/** Minted per admission attempt: a queued call outlives one token TTL. */
|
||||
authHeaders: async () => ({
|
||||
|
|
|
|||
|
|
@ -2085,6 +2085,14 @@ describe('Code Process', () => {
|
|||
signal: controller.signal,
|
||||
maxQueueWaitMs: 0,
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
maxRunTimeoutMs: 180_000,
|
||||
admission: {
|
||||
queueWaitMs: 60_000,
|
||||
initialDelayMs: 1_000,
|
||||
maxDelayMs: 30_000,
|
||||
multiplier: 2,
|
||||
jitterRatio: 0.2,
|
||||
},
|
||||
deadlineAtMs: 160_000,
|
||||
}),
|
||||
).resolves.toBe(result);
|
||||
|
|
@ -2108,6 +2116,14 @@ describe('Code Process', () => {
|
|||
maxQueueWaitMs: 0,
|
||||
codeApiMaxRetryWaitMs: undefined,
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
maxRunTimeoutMs: 180_000,
|
||||
admission: {
|
||||
queueWaitMs: 60_000,
|
||||
initialDelayMs: 1_000,
|
||||
maxDelayMs: 30_000,
|
||||
multiplier: 2,
|
||||
jitterRatio: 0.2,
|
||||
},
|
||||
deadlineAtMs: 160_000,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ const {
|
|||
createRepositoryInstructionLoader,
|
||||
resolveAttachedWorkspaceCommandTimeoutMax,
|
||||
resolveAttachedWorkspaceQueueWaitMs,
|
||||
resolveAttachedWorkspaceAdmissionOptions,
|
||||
resolveAttachedWorkspaceRequestTimeoutMs,
|
||||
createContextProgrammaticBashTool,
|
||||
resolveCodeExecutionContext,
|
||||
|
|
@ -2383,6 +2384,9 @@ async function loadToolsForExecution({
|
|||
maxRequestTimeoutMs: resolveAttachedWorkspaceRequestTimeoutMs(
|
||||
codeExecutionContext.codeEnvironmentConfigSchema,
|
||||
),
|
||||
...resolveAttachedWorkspaceAdmissionOptions(
|
||||
codeExecutionContext.codeEnvironmentConfigSchema,
|
||||
),
|
||||
minCommandAdmissionMs:
|
||||
codeExecutionContext.codeEnvironmentConfigSchema?.limits?.minCommandAdmissionMs,
|
||||
})
|
||||
|
|
|
|||
|
|
@ -3062,8 +3062,16 @@ describe('ToolService - Action Capability Gating', () => {
|
|||
defaultCommandTimeoutMs: 60_000,
|
||||
maxQueueWaitMs: 0,
|
||||
maxRequestTimeoutMs: 90_000,
|
||||
maxRunTimeoutMs: 180_000,
|
||||
minCommandAdmissionMs: 15_000,
|
||||
},
|
||||
admission: {
|
||||
queueWaitMs: 60_000,
|
||||
initialDelayMs: 1_000,
|
||||
maxDelayMs: 30_000,
|
||||
multiplier: 2,
|
||||
jitterRatio: 0.2,
|
||||
},
|
||||
},
|
||||
});
|
||||
const toolRegistry = new Map([
|
||||
|
|
@ -3095,6 +3103,14 @@ describe('ToolService - Action Capability Gating', () => {
|
|||
maxQueueWaitMs: 0,
|
||||
codeApiMaxRetryWaitMs: 0,
|
||||
maxRequestTimeoutMs: 90_000,
|
||||
maxRunTimeoutMs: 180_000,
|
||||
admission: {
|
||||
queueWaitMs: 60_000,
|
||||
initialDelayMs: 1_000,
|
||||
maxDelayMs: 30_000,
|
||||
multiplier: 2,
|
||||
jitterRatio: 0.2,
|
||||
},
|
||||
minCommandAdmissionMs: 15_000,
|
||||
});
|
||||
expect(mockResolveCodeExecutionWorkspaceContext).toHaveBeenCalledWith(
|
||||
|
|
|
|||
|
|
@ -7394,7 +7394,18 @@ describe('createToolExecuteHandler', () => {
|
|||
statefulSessions: true,
|
||||
environmentType: 'attached',
|
||||
codeEnvironmentConfigSchema: {
|
||||
limits: { maxQueueWaitMs: budget, maxRequestTimeoutMs: 125_000 },
|
||||
limits: {
|
||||
maxQueueWaitMs: budget,
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
maxRunTimeoutMs: 180_000,
|
||||
},
|
||||
admission: {
|
||||
queueWaitMs: 60_000,
|
||||
initialDelayMs: 1_000,
|
||||
maxDelayMs: 30_000,
|
||||
multiplier: 2,
|
||||
jitterRatio: 0.2,
|
||||
},
|
||||
},
|
||||
bridgeWorkerId: 'user-worker',
|
||||
},
|
||||
|
|
@ -7417,7 +7428,13 @@ describe('createToolExecuteHandler', () => {
|
|||
expect.objectContaining({
|
||||
maxQueueWaitMs: budget,
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
deadlineAtMs: startedAt + 125_000,
|
||||
maxRunTimeoutMs: 180_000,
|
||||
admission: expect.objectContaining({
|
||||
queueWaitMs: 60_000,
|
||||
multiplier: 2,
|
||||
jitterRatio: 0.2,
|
||||
}),
|
||||
deadlineAtMs: startedAt + 180_000,
|
||||
}),
|
||||
);
|
||||
expect(result.status).toBe('success');
|
||||
|
|
@ -7426,7 +7443,13 @@ describe('createToolExecuteHandler', () => {
|
|||
expected_base_sha256: 'b'.repeat(64),
|
||||
maxQueueWaitMs: remaining,
|
||||
maxRequestTimeoutMs: 125_000,
|
||||
deadlineAtMs: startedAt + 125_000,
|
||||
maxRunTimeoutMs: 180_000,
|
||||
admission: expect.objectContaining({
|
||||
queueWaitMs: 60_000,
|
||||
multiplier: 2,
|
||||
jitterRatio: 0.2,
|
||||
}),
|
||||
deadlineAtMs: startedAt + 180_000,
|
||||
}),
|
||||
);
|
||||
},
|
||||
|
|
@ -9052,8 +9075,9 @@ describe('createToolExecuteHandler', () => {
|
|||
async (name) => {
|
||||
const message = await reject(name, legacyBody);
|
||||
|
||||
expect(message).toContain('Invalid workspace path');
|
||||
expect(message.endsWith('"}"')).toBe(true);
|
||||
expect(message).toContain('INVALID_PATH');
|
||||
expect(message).not.toContain('Invalid workspace path');
|
||||
expect(message).not.toContain('upstreamBody');
|
||||
},
|
||||
);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ import type { CodeEnvRef, CodeWorkspaceOperation, PtcToolCallEvent } from 'libre
|
|||
import type { StructuredToolInterface } from '@librechat/agents/langchain/tools';
|
||||
import type { CodeEnvFile, CodeSessionContext } from '@librechat/agents';
|
||||
import type {
|
||||
WorkspaceAdmissionOptions,
|
||||
WorkspaceEditResult,
|
||||
WorkspaceTextEdit,
|
||||
WorkspacePreviewEditResult,
|
||||
|
|
@ -106,6 +107,7 @@ import {
|
|||
import {
|
||||
resolveAttachedWorkspaceReadFileLines,
|
||||
WorkspaceToolHttpError,
|
||||
resolveAttachedWorkspaceAdmissionOptions,
|
||||
WORKSPACE_EDIT_MAX_COUNT,
|
||||
WORKSPACE_WRITE_MAX_BYTES,
|
||||
} from '~/code/workspace';
|
||||
|
|
@ -605,6 +607,8 @@ export interface ToolExecuteOptions {
|
|||
signal?: AbortSignal;
|
||||
maxQueueWaitMs?: number;
|
||||
maxRequestTimeoutMs?: number;
|
||||
maxRunTimeoutMs?: number;
|
||||
admission?: WorkspaceAdmissionOptions['admission'];
|
||||
deadlineAtMs?: number;
|
||||
}) => Promise<WorkspaceReadResult>;
|
||||
/** Searches literal text within an attached worker's logical workspace. */
|
||||
|
|
@ -622,6 +626,8 @@ export interface ToolExecuteOptions {
|
|||
signal?: AbortSignal;
|
||||
maxQueueWaitMs?: number;
|
||||
maxRequestTimeoutMs?: number;
|
||||
maxRunTimeoutMs?: number;
|
||||
admission?: WorkspaceAdmissionOptions['admission'];
|
||||
deadlineAtMs?: number;
|
||||
}) => Promise<WorkspaceSearchResult>;
|
||||
/** Lists relative file paths within an attached worker's logical workspace. */
|
||||
|
|
@ -639,6 +645,8 @@ export interface ToolExecuteOptions {
|
|||
signal?: AbortSignal;
|
||||
maxQueueWaitMs?: number;
|
||||
maxRequestTimeoutMs?: number;
|
||||
maxRunTimeoutMs?: number;
|
||||
admission?: WorkspaceAdmissionOptions['admission'];
|
||||
deadlineAtMs?: number;
|
||||
}) => Promise<WorkspaceListResult>;
|
||||
/** Writes a UTF-8 file within an attached worker's logical workspace. */
|
||||
|
|
@ -656,6 +664,8 @@ export interface ToolExecuteOptions {
|
|||
signal?: AbortSignal;
|
||||
maxQueueWaitMs?: number;
|
||||
maxRequestTimeoutMs?: number;
|
||||
maxRunTimeoutMs?: number;
|
||||
admission?: WorkspaceAdmissionOptions['admission'];
|
||||
deadlineAtMs?: number;
|
||||
}) => Promise<WorkspaceWriteResult>;
|
||||
/** Previews exact replacements without mutating an attached worker workspace. */
|
||||
|
|
@ -674,6 +684,8 @@ export interface ToolExecuteOptions {
|
|||
signal?: AbortSignal;
|
||||
maxQueueWaitMs?: number;
|
||||
maxRequestTimeoutMs?: number;
|
||||
maxRunTimeoutMs?: number;
|
||||
admission?: WorkspaceAdmissionOptions['admission'];
|
||||
deadlineAtMs?: number;
|
||||
}) => Promise<WorkspacePreviewEditResult>;
|
||||
/** Applies exact replacements atomically within an attached worker workspace. */
|
||||
|
|
@ -693,6 +705,8 @@ export interface ToolExecuteOptions {
|
|||
signal?: AbortSignal;
|
||||
maxQueueWaitMs?: number;
|
||||
maxRequestTimeoutMs?: number;
|
||||
maxRunTimeoutMs?: number;
|
||||
admission?: WorkspaceAdmissionOptions['admission'];
|
||||
deadlineAtMs?: number;
|
||||
}) => Promise<WorkspaceEditResult>;
|
||||
/** Bounded reads return complete text; omitted maxBytes retains the legacy stdout path. */
|
||||
|
|
@ -3842,10 +3856,11 @@ function attachedWorkspaceAuthoringPath(
|
|||
function attachedWorkspaceRequestLimits(codeExecutionContext: CodeExecutionContext): {
|
||||
maxQueueWaitMs: number;
|
||||
maxRequestTimeoutMs?: number;
|
||||
} {
|
||||
} & WorkspaceAdmissionOptions {
|
||||
const config = codeExecutionContext.codeEnvironmentConfigSchema;
|
||||
const maxRequestTimeoutMs = resolveAttachedWorkspaceRequestTimeoutMs(config);
|
||||
return {
|
||||
...resolveAttachedWorkspaceAdmissionOptions(config),
|
||||
maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs(config),
|
||||
...(maxRequestTimeoutMs == null ? {} : { maxRequestTimeoutMs }),
|
||||
};
|
||||
|
|
@ -3868,8 +3883,9 @@ function attachedWorkspaceMutationParams(
|
|||
maxQueueWaitMs: number;
|
||||
maxRequestTimeoutMs?: number;
|
||||
deadlineAtMs?: number;
|
||||
} {
|
||||
} & WorkspaceAdmissionOptions {
|
||||
const limits = attachedWorkspaceRequestLimits(codeExecutionContext);
|
||||
const runTimeoutMs = limits.maxRunTimeoutMs ?? limits.maxRequestTimeoutMs;
|
||||
return {
|
||||
workspace_id: workspaceId,
|
||||
...(codeExecutionContext.codeWorkspace?.workspaceInstanceId
|
||||
|
|
@ -3878,9 +3894,7 @@ function attachedWorkspaceMutationParams(
|
|||
...(codeExecutionContext.codeWorkspace?.linkedWorktrees ? { linked_worktrees: true } : {}),
|
||||
codeApiBaseUrl: codeExecutionContext.baseUrl,
|
||||
...limits,
|
||||
...(limits.maxRequestTimeoutMs == null
|
||||
? {}
|
||||
: { deadlineAtMs: Date.now() + limits.maxRequestTimeoutMs }),
|
||||
...(runTimeoutMs == null ? {} : { deadlineAtMs: Date.now() + runTimeoutMs }),
|
||||
executionProfile: codeExecutionContext.executionProfile,
|
||||
...(codeExecutionContext.bridgeWorkerId
|
||||
? { bridgeWorkerId: codeExecutionContext.bridgeWorkerId }
|
||||
|
|
|
|||
|
|
@ -1308,3 +1308,55 @@ describe('createAttachedWorkspaceBashTool', () => {
|
|||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('attached command admission configuration', () => {
|
||||
test('bounds the advertised command ceiling by the shorter overall run budget', () => {
|
||||
expect(
|
||||
resolveAttachedWorkspaceCommandTimeoutMax({
|
||||
limits: {
|
||||
maxCommandTimeoutMs: 100_000,
|
||||
maxRequestTimeoutMs: 100_000,
|
||||
maxRunTimeoutMs: 65_000,
|
||||
},
|
||||
}),
|
||||
).toBe(45_000);
|
||||
});
|
||||
|
||||
test('passes separate run and retry policies through the actual tool invocation', async () => {
|
||||
const admission = {
|
||||
queueWaitMs: 60_000,
|
||||
initialDelayMs: 1_000,
|
||||
maxDelayMs: 30_000,
|
||||
multiplier: 2,
|
||||
jitterRatio: 0.2,
|
||||
};
|
||||
const fetchImpl = jest.fn<ReturnType<CodeBridgeFetch>, Parameters<CodeBridgeFetch>>(
|
||||
async () =>
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
protocolVersion: 1,
|
||||
operation: 'execute_command',
|
||||
workspaceId: 'primary',
|
||||
exitCode: 0,
|
||||
stdout: 'ready',
|
||||
stderr: '',
|
||||
timedOut: false,
|
||||
truncated: false,
|
||||
}),
|
||||
),
|
||||
);
|
||||
const bash = createAttachedWorkspaceBashTool({
|
||||
baseUrl: 'https://code.example/v1',
|
||||
authHeaders: () => ({}),
|
||||
workspaceId: 'primary',
|
||||
fetchImpl,
|
||||
maxRequestTimeoutMs: 100_000,
|
||||
maxRunTimeoutMs: 180_000,
|
||||
admission,
|
||||
});
|
||||
await bash.invoke({ command: 'echo ready' });
|
||||
expect(
|
||||
new Headers(fetchImpl.mock.calls[0][1]?.headers).get('X-LibreChat-Workspace-Queue-Wait-Ms'),
|
||||
).toBe('60000');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import type {
|
|||
import type { createBashProgrammaticToolCallingSchema } from '@librechat/agents';
|
||||
import type { DynamicStructuredTool } from '@librechat/agents/langchain/tools';
|
||||
import type { LCTool } from '@librechat/agents';
|
||||
import type { WorkspaceExecuteCommandResult } from './workspace';
|
||||
import type { WorkspaceAdmissionOptions, WorkspaceExecuteCommandResult } from './workspace';
|
||||
import type { CodeExecutionContext } from '~/agents/execution';
|
||||
import type { CodeBridgeFetch } from './bridge';
|
||||
import {
|
||||
|
|
@ -173,7 +173,10 @@ export function resolveAttachedWorkspaceCommandTimeoutMax(
|
|||
}
|
||||
return fitCommandTimeoutMaxToBudget(
|
||||
Math.min(requested, upstream),
|
||||
resolveAttachedWorkspaceRequestTimeoutMs(configSchema),
|
||||
Math.min(
|
||||
resolveAttachedWorkspaceRequestTimeoutMs(configSchema) ?? Infinity,
|
||||
configSchema?.limits?.maxRunTimeoutMs ?? Infinity,
|
||||
),
|
||||
configSchema?.limits?.minCommandAdmissionMs,
|
||||
);
|
||||
}
|
||||
|
|
@ -183,7 +186,7 @@ function fitCommandTimeoutMaxToBudget(
|
|||
maxRequestTimeoutMs?: number,
|
||||
minCommandAdmissionMs?: number,
|
||||
): number {
|
||||
if (maxRequestTimeoutMs == null) return maxTimeoutMs;
|
||||
if (maxRequestTimeoutMs == null || !Number.isFinite(maxRequestTimeoutMs)) return maxTimeoutMs;
|
||||
return Math.min(
|
||||
maxTimeoutMs,
|
||||
fitWorkspaceCommandTimeoutToBudget(maxRequestTimeoutMs, minCommandAdmissionMs),
|
||||
|
|
@ -450,6 +453,8 @@ export function createAttachedWorkspaceBashTool({
|
|||
maxQueueWaitMs,
|
||||
codeApiMaxRetryWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
maxRunTimeoutMs,
|
||||
admission,
|
||||
minCommandAdmissionMs,
|
||||
linkedWorktrees = false,
|
||||
nativeSandbox = false,
|
||||
|
|
@ -477,10 +482,10 @@ export function createAttachedWorkspaceBashTool({
|
|||
/** Minimum time for command admission inside an opted-in HTTP budget. */
|
||||
minCommandAdmissionMs?: number;
|
||||
fetchImpl?: CodeBridgeFetch;
|
||||
}): DynamicStructuredTool {
|
||||
} & WorkspaceAdmissionOptions): DynamicStructuredTool {
|
||||
const effectiveMaxTimeoutMs = fitCommandTimeoutMaxToBudget(
|
||||
normalizeAttachedWorkspaceCommandTimeoutMax(maxTimeoutMs),
|
||||
maxRequestTimeoutMs,
|
||||
Math.min(maxRequestTimeoutMs ?? Infinity, maxRunTimeoutMs ?? Infinity),
|
||||
minCommandAdmissionMs,
|
||||
);
|
||||
const effectiveDefaultTimeoutMs = resolveAttachedWorkspaceCommandTimeoutDefault(
|
||||
|
|
@ -570,6 +575,8 @@ export function createAttachedWorkspaceBashTool({
|
|||
...(maxQueueWaitMs == null ? {} : { maxQueueWaitMs }),
|
||||
...(codeApiMaxRetryWaitMs == null ? {} : { codeApiMaxRetryWaitMs }),
|
||||
...(maxRequestTimeoutMs == null ? {} : { maxRequestTimeoutMs }),
|
||||
...(maxRunTimeoutMs == null ? {} : { maxRunTimeoutMs }),
|
||||
...(admission == null ? {} : { admission }),
|
||||
});
|
||||
if (result.operation !== 'execute_command') {
|
||||
throw new Error('Attached workspace returned an unexpected command result.');
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import { logger } from '@librechat/data-schemas';
|
||||
import { codeEnvironmentAdmissionSchema } from 'librechat-data-provider';
|
||||
import type { WorkspaceToolRequest } from './workspace';
|
||||
import type { CodeBridgeFetch } from './bridge';
|
||||
import {
|
||||
|
|
@ -1137,7 +1139,9 @@ describe('executeWorkspaceTool', () => {
|
|||
}),
|
||||
).rejects.toMatchObject({ upstreamStatus: 504 });
|
||||
expect(timeout).toHaveBeenCalledWith(60_000);
|
||||
expect(fetchImpl.mock.calls[0][1].headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe('25000');
|
||||
expect(
|
||||
new Headers(fetchImpl.mock.calls[0][1]?.headers).get('X-LibreChat-Workspace-Queue-Wait-Ms'),
|
||||
).toBe('25000');
|
||||
});
|
||||
|
||||
test('subtracts credential acquisition from the allowance before dispatching', async () => {
|
||||
|
|
@ -1230,7 +1234,7 @@ describe('executeWorkspaceTool', () => {
|
|||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('keeps Retry-After capped at 30 seconds even when one admission may wait longer', async () => {
|
||||
test('does not shorten a Retry-After hint to fit a shrinking total budget', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const fetchImpl = jest
|
||||
|
|
@ -1254,13 +1258,8 @@ describe('executeWorkspaceTool', () => {
|
|||
maxRequestTimeoutMs: 125_000,
|
||||
fetchImpl,
|
||||
}).catch((error: WorkspaceToolHttpError) => error);
|
||||
await jest.advanceTimersByTimeAsync(29_999);
|
||||
await expect(result).resolves.toMatchObject({ upstreamStatus: 503 });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
await expect(result).resolves.toMatchObject({ upstreamStatus: 504 });
|
||||
expect(fetchImpl.mock.calls[1][1].headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe(
|
||||
'60000',
|
||||
);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
|
|
@ -2425,3 +2424,221 @@ describe('linked worktree lanes', () => {
|
|||
).rejects.toMatchObject({ reason: 'invalid' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('workspace admission policy and independent budgets', () => {
|
||||
const command: WorkspaceToolRequest = {
|
||||
protocolVersion: 1,
|
||||
operation: 'execute_command',
|
||||
workspaceId: 'primary',
|
||||
command: 'echo ready',
|
||||
timeoutMs: 30_000,
|
||||
};
|
||||
const success = () =>
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
protocolVersion: 1,
|
||||
operation: 'execute_command',
|
||||
workspaceId: 'primary',
|
||||
exitCode: 0,
|
||||
stdout: 'ready',
|
||||
stderr: '',
|
||||
timedOut: false,
|
||||
truncated: false,
|
||||
}),
|
||||
);
|
||||
const queueTimeout = () =>
|
||||
new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), {
|
||||
status: 503,
|
||||
headers: { 'Retry-After': '0' },
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
jest.useFakeTimers();
|
||||
jest.setSystemTime(new Date('2026-10-04T00:00:00.000Z'));
|
||||
});
|
||||
afterEach(() => jest.useRealTimers());
|
||||
|
||||
test('keeps the full 30-second execution budget after 60 seconds waiting', async () => {
|
||||
let attempts = 0;
|
||||
const fetchImpl = jest.fn<ReturnType<CodeBridgeFetch>, Parameters<CodeBridgeFetch>>(
|
||||
async (_url, init) => {
|
||||
if (init == null) throw new Error('Missing request init');
|
||||
attempts++;
|
||||
if (attempts === 1) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 30_000));
|
||||
return queueTimeout();
|
||||
}
|
||||
expect(Number(new Headers(init.headers).get('X-LibreChat-Workspace-Queue-Wait-Ms'))).toBe(
|
||||
60_000,
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, 59_900));
|
||||
expect(init.signal?.aborted).toBe(false);
|
||||
return success();
|
||||
},
|
||||
);
|
||||
const result = executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
request: command,
|
||||
maxRequestTimeoutMs: 100_000,
|
||||
maxRunTimeoutMs: 180_000,
|
||||
admission: codeEnvironmentAdmissionSchema.parse({ queueWaitMs: 60_000 }),
|
||||
});
|
||||
await jest.advanceTimersByTimeAsync(90_000);
|
||||
await expect(result).resolves.toMatchObject({ stdout: 'ready' });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
||||
expect(fetchImpl.mock.calls[1][1]?.body).toBe(fetchImpl.mock.calls[0][1]?.body);
|
||||
});
|
||||
|
||||
test('bounds queue allowance by the transport reserve, not the larger run deadline', async () => {
|
||||
const fetchImpl = jest.fn<ReturnType<CodeBridgeFetch>, Parameters<CodeBridgeFetch>>(async () =>
|
||||
success(),
|
||||
);
|
||||
await executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
request: command,
|
||||
maxRequestTimeoutMs: 65_000,
|
||||
maxRunTimeoutMs: 180_000,
|
||||
admission: codeEnvironmentAdmissionSchema.parse({ queueWaitMs: 180_000 }),
|
||||
});
|
||||
expect(
|
||||
new Headers(fetchImpl.mock.calls[0][1]?.headers).get('X-LibreChat-Workspace-Queue-Wait-Ms'),
|
||||
).toBe('25000');
|
||||
});
|
||||
|
||||
test('honors an earlier caller deadline even when overall and transport budgets are larger', async () => {
|
||||
const fetchImpl = jest.fn<ReturnType<CodeBridgeFetch>, Parameters<CodeBridgeFetch>>(async () =>
|
||||
success(),
|
||||
);
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
request: command,
|
||||
maxRequestTimeoutMs: 100_000,
|
||||
maxRunTimeoutMs: 180_000,
|
||||
deadlineAtMs: Date.now() + 40_000,
|
||||
}),
|
||||
).rejects.toMatchObject({ reason: 'insufficient_time' });
|
||||
expect(fetchImpl).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('adds exponential jitter only to whitelisted rejections and caps local backoff', async () => {
|
||||
jest.spyOn(Math, 'random').mockReturnValue(1 - Number.EPSILON);
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(new Response('{"code":"WORKSPACE_QUEUE_TIMEOUT"}', { status: 503 }))
|
||||
.mockResolvedValueOnce(new Response('{"code":"WORKSPACE_QUEUE_TIMEOUT"}', { status: 503 }))
|
||||
.mockResolvedValueOnce(new Response('{"code":"WORKSPACE_QUEUE_TIMEOUT"}', { status: 503 }))
|
||||
.mockResolvedValueOnce(success());
|
||||
const result = executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
request: command,
|
||||
admission: codeEnvironmentAdmissionSchema.parse({
|
||||
initialDelayMs: 1_000,
|
||||
maxDelayMs: 3_000,
|
||||
multiplier: 2,
|
||||
jitterRatio: 0.5,
|
||||
}),
|
||||
});
|
||||
await jest.advanceTimersByTimeAsync(1_499);
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
||||
await jest.advanceTimersByTimeAsync(3_000);
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(3);
|
||||
await jest.advanceTimersByTimeAsync(3_000);
|
||||
await expect(result).resolves.toMatchObject({ stdout: 'ready' });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
|
||||
test.each(['60', 'date', 'body'])(
|
||||
'does not retry before a 60-second %s hint with jitter',
|
||||
async (hint) => {
|
||||
jest.spyOn(Math, 'random').mockReturnValue(1 - Number.EPSILON);
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
error: 'rate_limited',
|
||||
...(hint === 'body' ? { retry_after_seconds: 60 } : {}),
|
||||
}),
|
||||
{
|
||||
status: 429,
|
||||
headers:
|
||||
hint === 'body'
|
||||
? {}
|
||||
: {
|
||||
'Retry-After':
|
||||
hint === 'date' ? new Date(Date.now() + 60_000).toUTCString() : hint,
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
.mockResolvedValueOnce(success());
|
||||
const result = executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
request: command,
|
||||
maxRequestTimeoutMs: 100_000,
|
||||
maxRunTimeoutMs: 180_000,
|
||||
codeApiMaxRetryWaitMs: 180_000,
|
||||
admission: codeEnvironmentAdmissionSchema.parse({ jitterRatio: 0.5 }),
|
||||
});
|
||||
await jest.advanceTimersByTimeAsync(89_999);
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
await expect(result).resolves.toMatchObject({ stdout: 'ready' });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
||||
},
|
||||
);
|
||||
|
||||
test('keeps upstream bodies diagnostic-only for every rejection kind', () => {
|
||||
for (const status of [429, 503, 502, 409]) {
|
||||
const body = '{"code":"WORKSPACE_QUEUE_TIMEOUT","error":"private upstream diagnostic"}';
|
||||
const error = new WorkspaceToolHttpError('rejected', status, body);
|
||||
expect(error.message).not.toContain('private upstream diagnostic');
|
||||
expect(error.upstreamBody).toBe(body);
|
||||
}
|
||||
});
|
||||
|
||||
test('emits one aggregate outcome without calling HTTP latency queue wait', async () => {
|
||||
const log = jest.spyOn(logger, 'debug');
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(queueTimeout())
|
||||
.mockResolvedValueOnce(success());
|
||||
const result = executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
request: command,
|
||||
});
|
||||
await jest.advanceTimersByTimeAsync(100);
|
||||
await result;
|
||||
expect(log).toHaveBeenCalledTimes(1);
|
||||
expect(log).toHaveBeenCalledWith(
|
||||
'[WorkspaceAdmission] outcome',
|
||||
expect.objectContaining({
|
||||
attempts: 2,
|
||||
queueRejections: 1,
|
||||
retryWaitMs: 100,
|
||||
outcome: 'completed',
|
||||
}),
|
||||
);
|
||||
expect(log).toHaveBeenCalledWith(
|
||||
'[WorkspaceAdmission] outcome',
|
||||
expect.not.objectContaining({
|
||||
queueWaitMs: expect.any(Number),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
import { trace } from '@opentelemetry/api';
|
||||
import { logger } from '@librechat/data-schemas';
|
||||
import {
|
||||
codeEnvironmentAdmissionSchema,
|
||||
CODE_ENVIRONMENT_ADMISSION_MAX_MS,
|
||||
CODE_ENVIRONMENT_COMMAND_TIMEOUT_DEFAULT_MS,
|
||||
CODE_ENVIRONMENT_READ_FILE_DEFAULT_LINES,
|
||||
|
|
@ -32,12 +35,11 @@ const DEFAULT_COMMAND_OUTPUT_BYTES = 256 * 1024;
|
|||
const MAX_COMMAND_OUTPUT_BYTES = 1024 * 1024;
|
||||
const MAX_COMMAND_SIGNAL_LENGTH = 32;
|
||||
const WORKSPACE_COMMAND_TRANSPORT_GRACE_MS = 5_000;
|
||||
/** Legacy admission window and Retry-After cap, independent of the retry horizon. */
|
||||
/** Legacy per-request admission window, independent of the retry horizon. */
|
||||
const WORKSPACE_QUEUE_TIMEOUT_MS = 30_000;
|
||||
const WORKSPACE_QUEUE_WAIT_HEADER = 'X-LibreChat-Workspace-Queue-Wait-Ms';
|
||||
/** Compatibility export; the deployment schema owns the default and hard cap. */
|
||||
export const WORKSPACE_QUEUE_MAX_WAIT_MS: number = CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS;
|
||||
const WORKSPACE_QUEUE_RETRY_DELAY_MS = 1_000;
|
||||
const WORKSPACE_COMMAND_SETTLEMENT_GRACE_MS = 5_000;
|
||||
|
||||
/**
|
||||
|
|
@ -319,6 +321,7 @@ export class WorkspaceToolHttpError extends Error {
|
|||
public readonly upstreamBody?: string,
|
||||
public readonly upstreamBodyTruncated = false,
|
||||
) {
|
||||
const parsed = upstreamBodyTruncated ? undefined : parseErrorBody(upstreamBody);
|
||||
let message = `Workspace tool request ${reason}`;
|
||||
const admissionRejection =
|
||||
reason === 'rejected'
|
||||
|
|
@ -339,11 +342,9 @@ export class WorkspaceToolHttpError extends Error {
|
|||
super(
|
||||
message +
|
||||
(upstreamStatus == null ? '' : ` (upstreamStatus: ${upstreamStatus})`) +
|
||||
(upstreamBody ? `; upstreamBody: ${JSON.stringify(upstreamBody)}` : '') +
|
||||
(upstreamBodyTruncated ? ' [body truncated or incomplete]' : ''),
|
||||
(parsed?.code == null ? '' : ` (upstreamCode: ${parsed.code})`),
|
||||
);
|
||||
this.name = 'WorkspaceToolHttpError';
|
||||
const parsed = upstreamBodyTruncated ? undefined : parseErrorBody(upstreamBody);
|
||||
this.upstreamCode = parsed?.code;
|
||||
this.editConflict =
|
||||
reason === 'rejected' && upstreamStatus === 409 && parsed?.code === 'EDIT_CONFLICT'
|
||||
|
|
@ -425,25 +426,56 @@ function waitForWorkspaceAdmission(delayMs: number, signal?: AbortSignal): Promi
|
|||
});
|
||||
}
|
||||
|
||||
function workspaceAdmissionRetryDelay(value: string | null, rateLimitBody?: string): number {
|
||||
const DEFAULT_ADMISSION_POLICY = codeEnvironmentAdmissionSchema.parse({});
|
||||
|
||||
export interface WorkspaceAdmissionOptions {
|
||||
maxRunTimeoutMs?: number;
|
||||
admission?: CodeEnvironmentUserConfigSchema['admission'];
|
||||
}
|
||||
|
||||
/** Keep omitted policy absent so existing adapters and total-budget semantics remain unchanged. */
|
||||
export function resolveAttachedWorkspaceAdmissionOptions(
|
||||
config?: CodeEnvironmentUserConfigSchema,
|
||||
): WorkspaceAdmissionOptions {
|
||||
return {
|
||||
...(config?.limits?.maxRunTimeoutMs == null
|
||||
? {}
|
||||
: { maxRunTimeoutMs: config.limits.maxRunTimeoutMs }),
|
||||
...(config?.admission == null ? {} : { admission: config.admission }),
|
||||
};
|
||||
}
|
||||
|
||||
function workspaceAdmissionRetryDelay(
|
||||
value: string | null,
|
||||
body: string | undefined,
|
||||
attempt: number,
|
||||
policy: NonNullable<WorkspaceAdmissionOptions['admission']>,
|
||||
): number {
|
||||
let hintMs: number | undefined;
|
||||
if (value != null && /^\d+$/.test(value)) {
|
||||
const seconds = Number(value);
|
||||
if (Number.isFinite(seconds)) {
|
||||
return Math.max(100, Math.min(seconds * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS));
|
||||
}
|
||||
hintMs = Number(value) * 1_000;
|
||||
} else if (value != null) {
|
||||
const date = Date.parse(value);
|
||||
if (Number.isFinite(date)) hintMs = Math.max(0, date - Date.now());
|
||||
}
|
||||
if (rateLimitBody) {
|
||||
if (hintMs != null && !Number.isFinite(hintMs)) hintMs = undefined;
|
||||
if (hintMs == null && body) {
|
||||
try {
|
||||
const parsed: { retry_after_seconds?: number } | null = JSON.parse(rateLimitBody);
|
||||
const parsed: { retry_after_seconds?: number } | null = JSON.parse(body);
|
||||
const seconds = parsed?.retry_after_seconds;
|
||||
if (typeof seconds === 'number' && Number.isFinite(seconds) && seconds >= 0) {
|
||||
return Math.max(100, Math.min(seconds * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS));
|
||||
hintMs = seconds * 1_000;
|
||||
}
|
||||
} catch {
|
||||
// Fall back when the delay hint is unusable.
|
||||
// The rejection whitelist already requires complete JSON.
|
||||
}
|
||||
}
|
||||
return WORKSPACE_QUEUE_RETRY_DELAY_MS;
|
||||
const delayMs =
|
||||
hintMs == null
|
||||
? Math.min(policy.maxDelayMs, policy.initialDelayMs * policy.multiplier ** (attempt - 1))
|
||||
: Math.max(100, hintMs);
|
||||
const jitteredMs = Math.ceil(delayMs + delayMs * policy.jitterRatio * Math.random());
|
||||
return hintMs == null ? Math.min(policy.maxDelayMs, jitteredMs) : jitteredMs;
|
||||
}
|
||||
|
||||
/** Keep a received HTTP status even if reading its diagnostic body fails or stalls. */
|
||||
|
|
@ -1109,6 +1141,8 @@ export async function executeWorkspaceTool({
|
|||
maxQueueWaitMs = WORKSPACE_QUEUE_MAX_WAIT_MS,
|
||||
codeApiMaxRetryWaitMs = CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS,
|
||||
maxRequestTimeoutMs,
|
||||
maxRunTimeoutMs,
|
||||
admission,
|
||||
deadlineAtMs,
|
||||
linkedWorktrees = false,
|
||||
}: {
|
||||
|
|
@ -1120,14 +1154,18 @@ export async function executeWorkspaceTool({
|
|||
maxQueueWaitMs?: number;
|
||||
/** Maximum time waiting for Code API rate-limit admission, independent of queue retries. */
|
||||
codeApiMaxRetryWaitMs?: number;
|
||||
/** End-to-end HTTP budget for this call. Omission keeps legacy per-attempt timeouts. */
|
||||
/** Per-attempt transport ceiling; also the total budget unless maxRunTimeoutMs is set. */
|
||||
maxRequestTimeoutMs?: number;
|
||||
/** Optional earlier caller deadline; a signal alone has no remaining-time value. */
|
||||
deadlineAtMs?: number;
|
||||
/** The worker runs each `.worktrees/<name>` in its own lane; route matching requests there. */
|
||||
linkedWorktrees?: boolean;
|
||||
}): Promise<WorkspaceToolResult> {
|
||||
} & WorkspaceAdmissionOptions): Promise<WorkspaceToolResult> {
|
||||
const parsedPolicy = codeEnvironmentAdmissionSchema.safeParse(
|
||||
admission ?? DEFAULT_ADMISSION_POLICY,
|
||||
);
|
||||
if (
|
||||
!parsedPolicy.success ||
|
||||
!isValidRequest(request) ||
|
||||
!Number.isSafeInteger(maxQueueWaitMs) ||
|
||||
maxQueueWaitMs < 0 ||
|
||||
|
|
@ -1139,141 +1177,210 @@ export async function executeWorkspaceTool({
|
|||
(!Number.isSafeInteger(maxRequestTimeoutMs) ||
|
||||
maxRequestTimeoutMs < 1 ||
|
||||
maxRequestTimeoutMs > CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS)) ||
|
||||
(maxRunTimeoutMs !== undefined &&
|
||||
(!Number.isSafeInteger(maxRunTimeoutMs) ||
|
||||
maxRunTimeoutMs < 1 ||
|
||||
maxRunTimeoutMs > CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS)) ||
|
||||
(deadlineAtMs !== undefined && (!Number.isSafeInteger(deadlineAtMs) || deadlineAtMs < 1))
|
||||
) {
|
||||
throw new WorkspaceToolHttpError('invalid');
|
||||
}
|
||||
const policy = parsedPolicy.data;
|
||||
const lane = linkedWorktrees === true ? toLinkedWorktreeRequest(request) : undefined;
|
||||
const wireRequest: WorkspaceToolRequest = lane?.request ?? request;
|
||||
const executionBudgetMs = getWorkspaceExecutionBudgetMs(wireRequest);
|
||||
const completionReserveMs = executionBudgetMs + WORKSPACE_COMMAND_TRANSPORT_GRACE_MS;
|
||||
const perAttemptTimeoutMs = maxRequestTimeoutMs ?? getWorkspaceToolTimeoutMs(wireRequest);
|
||||
const startedAt = Date.now();
|
||||
const runTimeoutMs = maxRunTimeoutMs ?? maxRequestTimeoutMs;
|
||||
const callerDeadlineAt = Math.min(
|
||||
deadlineAtMs ?? Infinity,
|
||||
maxRequestTimeoutMs == null ? Infinity : Date.now() + maxRequestTimeoutMs,
|
||||
runTimeoutMs == null ? Infinity : startedAt + runTimeoutMs,
|
||||
);
|
||||
const queueDeadlineAt = Date.now() + maxQueueWaitMs;
|
||||
const queueDeadlineAt = startedAt + maxQueueWaitMs;
|
||||
const callerRetryDeadlineAt = callerDeadlineAt - completionReserveMs;
|
||||
const body = JSON.stringify(wireRequest);
|
||||
let lastAdmissionRejection: WorkspaceToolHttpError | undefined;
|
||||
let lastRetryDeadlineAt = Infinity;
|
||||
let rateLimitWaitedMs = 0;
|
||||
while (true) {
|
||||
try {
|
||||
signal?.throwIfAborted();
|
||||
/** Admission and retries cannot consume the reserved execution or delivery time. */
|
||||
const attemptTimeoutMs = Math.floor(
|
||||
Math.min(perAttemptTimeoutMs, callerDeadlineAt - Date.now()),
|
||||
);
|
||||
if (attemptTimeoutMs <= completionReserveMs) {
|
||||
throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time');
|
||||
}
|
||||
const attemptStartedAt = Date.now();
|
||||
const timeoutSignal = AbortSignal.timeout(attemptTimeoutMs);
|
||||
const requestSignal =
|
||||
signal != null && typeof AbortSignal.any === 'function'
|
||||
? AbortSignal.any([signal, timeoutSignal])
|
||||
: timeoutSignal;
|
||||
let attemptHeaders: Record<string, string>;
|
||||
let retryWaitMs = 0;
|
||||
let attempts = 0;
|
||||
let queueRejections = 0;
|
||||
let rateLimitRejections = 0;
|
||||
let outcome = 'failed';
|
||||
const span = trace.getTracer('librechat.workspace').startSpan('workspace.tool', {
|
||||
attributes: {
|
||||
'workspace.operation': wireRequest.operation,
|
||||
'workspace.id': wireRequest.workspaceId,
|
||||
...(lane == null ? {} : { 'workspace.lane': lane.worktree }),
|
||||
'workspace.execution_budget_ms': executionBudgetMs,
|
||||
'workspace.transport_budget_ms': perAttemptTimeoutMs,
|
||||
},
|
||||
});
|
||||
try {
|
||||
while (true) {
|
||||
try {
|
||||
attemptHeaders =
|
||||
typeof authHeaders === 'function'
|
||||
? await getWorkspaceAuthHeaders(authHeaders, requestSignal)
|
||||
: authHeaders;
|
||||
} catch (error) {
|
||||
if (timeoutSignal.aborted && signal?.aborted !== true) {
|
||||
signal?.throwIfAborted();
|
||||
/** Admission and retries cannot consume the reserved execution or delivery time. */
|
||||
const attemptTimeoutMs = Math.floor(
|
||||
Math.min(perAttemptTimeoutMs, callerDeadlineAt - Date.now()),
|
||||
);
|
||||
if (attemptTimeoutMs <= completionReserveMs) {
|
||||
throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
signal?.throwIfAborted();
|
||||
if (timeoutSignal.aborted) {
|
||||
throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time');
|
||||
}
|
||||
if (lastAdmissionRejection && Date.now() >= lastRetryDeadlineAt) {
|
||||
throw lastAdmissionRejection;
|
||||
}
|
||||
const remainingMs = Math.min(
|
||||
attemptTimeoutMs - (Date.now() - attemptStartedAt),
|
||||
callerDeadlineAt - Date.now(),
|
||||
);
|
||||
const queueAllowanceMs = Math.min(
|
||||
CODE_ENVIRONMENT_ADMISSION_MAX_MS,
|
||||
Math.floor(remainingMs - completionReserveMs),
|
||||
);
|
||||
if (queueAllowanceMs < 1) {
|
||||
throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time');
|
||||
}
|
||||
const response = await fetchImpl(
|
||||
`${baseURL.trim().replace(/\/+$/, '')}/workspace-tools/execute`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
...attemptHeaders,
|
||||
'Content-Type': 'application/json',
|
||||
[WORKSPACE_QUEUE_WAIT_HEADER]: String(queueAllowanceMs),
|
||||
},
|
||||
body,
|
||||
redirect: 'error',
|
||||
signal: requestSignal,
|
||||
},
|
||||
);
|
||||
if (!response.ok) {
|
||||
const { body, truncated } = await readErrorBody(response, requestSignal);
|
||||
const attemptStartedAt = Date.now();
|
||||
const timeoutSignal = AbortSignal.timeout(attemptTimeoutMs);
|
||||
const requestSignal =
|
||||
signal != null && typeof AbortSignal.any === 'function'
|
||||
? AbortSignal.any([signal, timeoutSignal])
|
||||
: timeoutSignal;
|
||||
let attemptHeaders: Record<string, string>;
|
||||
try {
|
||||
attemptHeaders =
|
||||
typeof authHeaders === 'function'
|
||||
? await getWorkspaceAuthHeaders(authHeaders, requestSignal)
|
||||
: authHeaders;
|
||||
} catch (error) {
|
||||
if (timeoutSignal.aborted && signal?.aborted !== true) {
|
||||
throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
signal?.throwIfAborted();
|
||||
const rejection = new WorkspaceToolHttpError('rejected', response.status, body, truncated);
|
||||
const admission = getWorkspaceAdmissionRejection(response.status, body, truncated);
|
||||
if (admission == null) throw rejection;
|
||||
const retryDeadlineAt = Math.min(
|
||||
admission === 'queue_timeout' ? queueDeadlineAt : Infinity,
|
||||
callerRetryDeadlineAt,
|
||||
if (timeoutSignal.aborted) {
|
||||
throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time');
|
||||
}
|
||||
if (lastAdmissionRejection && Date.now() >= lastRetryDeadlineAt) {
|
||||
throw lastAdmissionRejection;
|
||||
}
|
||||
const remainingMs = Math.min(
|
||||
attemptTimeoutMs - (Date.now() - attemptStartedAt),
|
||||
callerDeadlineAt - Date.now(),
|
||||
);
|
||||
if (Date.now() >= retryDeadlineAt) throw rejection;
|
||||
const retryAfterMs = workspaceAdmissionRetryDelay(
|
||||
response.headers.get('Retry-After'),
|
||||
admission === 'rate_limited' ? body : undefined,
|
||||
const queueAllowanceMs = Math.min(
|
||||
policy.queueWaitMs ?? CODE_ENVIRONMENT_ADMISSION_MAX_MS,
|
||||
Math.floor(remainingMs - completionReserveMs),
|
||||
);
|
||||
if (queueAllowanceMs < 1) {
|
||||
throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time');
|
||||
}
|
||||
attempts++;
|
||||
const response = await fetchImpl(
|
||||
`${baseURL.trim().replace(/\/+$/, '')}/workspace-tools/execute`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
...attemptHeaders,
|
||||
'Content-Type': 'application/json',
|
||||
[WORKSPACE_QUEUE_WAIT_HEADER]: String(queueAllowanceMs),
|
||||
},
|
||||
body,
|
||||
redirect: 'error',
|
||||
signal: requestSignal,
|
||||
},
|
||||
);
|
||||
if (!response.ok) {
|
||||
const { body, truncated } = await readErrorBody(response, requestSignal);
|
||||
signal?.throwIfAborted();
|
||||
const rejection = new WorkspaceToolHttpError(
|
||||
'rejected',
|
||||
response.status,
|
||||
body,
|
||||
truncated,
|
||||
);
|
||||
const admission = getWorkspaceAdmissionRejection(response.status, body, truncated);
|
||||
if (admission == null) throw rejection;
|
||||
if (admission === 'queue_timeout') queueRejections++;
|
||||
else rateLimitRejections++;
|
||||
const rejectionCount =
|
||||
admission === 'queue_timeout' ? queueRejections : rateLimitRejections;
|
||||
if (rejectionCount === 1) {
|
||||
span.addEvent('workspace.admission.rejected', { kind: admission });
|
||||
}
|
||||
const retryDeadlineAt = Math.min(
|
||||
admission === 'queue_timeout' ? queueDeadlineAt : Infinity,
|
||||
callerRetryDeadlineAt,
|
||||
);
|
||||
if (Date.now() >= retryDeadlineAt) throw rejection;
|
||||
const retryAfterMs = workspaceAdmissionRetryDelay(
|
||||
response.headers.get('Retry-After'),
|
||||
admission === 'rate_limited' ? body : undefined,
|
||||
rejectionCount,
|
||||
policy,
|
||||
);
|
||||
if (
|
||||
admission === 'rate_limited' &&
|
||||
retryAfterMs > codeApiMaxRetryWaitMs - rateLimitWaitedMs
|
||||
) {
|
||||
throw rejection;
|
||||
}
|
||||
// Never truncate a server hint into an early retry.
|
||||
if (!Number.isFinite(retryAfterMs) || retryAfterMs >= retryDeadlineAt - Date.now()) {
|
||||
throw rejection;
|
||||
}
|
||||
const delayMs = retryAfterMs;
|
||||
lastAdmissionRejection = rejection;
|
||||
lastRetryDeadlineAt = retryDeadlineAt;
|
||||
if (delayMs <= 0) throw rejection;
|
||||
const waitStartedAt = Date.now();
|
||||
await waitForWorkspaceAdmission(delayMs, signal);
|
||||
retryWaitMs += Math.max(delayMs, Date.now() - waitStartedAt);
|
||||
if (admission === 'rate_limited') {
|
||||
/** A timer may run late. Charge the actual wait, but honor the retry that
|
||||
* was authorized before waiting; another 429 cannot exceed the balance. */
|
||||
rateLimitWaitedMs += Math.max(retryAfterMs, Date.now() - waitStartedAt);
|
||||
}
|
||||
/** Timer oversleep cannot consume the execution reserve. */
|
||||
if (Date.now() >= retryDeadlineAt) {
|
||||
throw rejection;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const result = await readBoundedJson(response, requestSignal);
|
||||
if (!isValidResult(wireRequest, result)) {
|
||||
throw new WorkspaceToolHttpError('invalid');
|
||||
}
|
||||
outcome = 'completed';
|
||||
return lane ? fromLinkedWorktreeResult(result, lane.worktree) : result;
|
||||
} catch (error) {
|
||||
outcome = error instanceof WorkspaceToolHttpError ? error.reason : 'failed';
|
||||
if (signal?.aborted === true) outcome = 'cancelled';
|
||||
if (error instanceof WorkspaceToolHttpError) throw error;
|
||||
if (
|
||||
admission === 'rate_limited' &&
|
||||
retryAfterMs > codeApiMaxRetryWaitMs - rateLimitWaitedMs
|
||||
signal?.aborted === true &&
|
||||
(error === signal.reason || (isRecord(error) && error.name === 'AbortError'))
|
||||
) {
|
||||
throw rejection;
|
||||
throw error;
|
||||
}
|
||||
const delayMs = Math.min(retryAfterMs, retryDeadlineAt - Date.now());
|
||||
lastAdmissionRejection = rejection;
|
||||
lastRetryDeadlineAt = retryDeadlineAt;
|
||||
if (delayMs <= 0) throw rejection;
|
||||
const waitStartedAt = Date.now();
|
||||
await waitForWorkspaceAdmission(delayMs, signal);
|
||||
if (admission === 'rate_limited') {
|
||||
/** A timer may run late. Charge the actual wait, but honor the retry that
|
||||
* was authorized before waiting; another 429 cannot exceed the balance. */
|
||||
rateLimitWaitedMs += Math.max(retryAfterMs, Date.now() - waitStartedAt);
|
||||
if (isRecord(error) && error.name === 'TimeoutError') {
|
||||
outcome = 'timeout';
|
||||
throw new WorkspaceToolHttpError('timeout');
|
||||
}
|
||||
/** A clamped delay can land exactly on this admission's retry deadline. Never open
|
||||
* another admission window without the full execution reserve. */
|
||||
if (Date.now() >= retryDeadlineAt) {
|
||||
throw rejection;
|
||||
}
|
||||
continue;
|
||||
throw new WorkspaceToolHttpError('failed');
|
||||
}
|
||||
const result = await readBoundedJson(response, requestSignal);
|
||||
if (!isValidResult(wireRequest, result)) {
|
||||
throw new WorkspaceToolHttpError('invalid');
|
||||
}
|
||||
return lane ? fromLinkedWorktreeResult(result, lane.worktree) : result;
|
||||
} catch (error) {
|
||||
if (error instanceof WorkspaceToolHttpError) throw error;
|
||||
if (
|
||||
signal?.aborted === true &&
|
||||
(error === signal.reason || (isRecord(error) && error.name === 'AbortError'))
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
if (isRecord(error) && error.name === 'TimeoutError') {
|
||||
throw new WorkspaceToolHttpError('timeout');
|
||||
}
|
||||
throw new WorkspaceToolHttpError('failed');
|
||||
}
|
||||
} finally {
|
||||
const diagnostics = {
|
||||
operation: wireRequest.operation,
|
||||
workspaceId: wireRequest.workspaceId,
|
||||
...(lane == null ? {} : { lane: lane.worktree }),
|
||||
outcome,
|
||||
attempts,
|
||||
queueRejections,
|
||||
rateLimitRejections,
|
||||
retryWaitMs,
|
||||
rateLimitWaitedMs,
|
||||
elapsedMs: Math.max(0, Date.now() - startedAt),
|
||||
};
|
||||
span.setAttributes({
|
||||
'workspace.outcome': outcome,
|
||||
'workspace.attempts': attempts,
|
||||
'workspace.queue_rejections': queueRejections,
|
||||
'workspace.rate_limit_rejections': rateLimitRejections,
|
||||
'workspace.client_retry_wait_ms': retryWaitMs,
|
||||
'workspace.rate_limit_wait_ms': rateLimitWaitedMs,
|
||||
});
|
||||
span.end();
|
||||
logger.debug('[WorkspaceAdmission] outcome', diagnostics);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2518,3 +2518,48 @@ describe('subagent activity policy', () => {
|
|||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('workspace admission configuration', () => {
|
||||
it('preserves absent defaults and resolves an explicit admission policy', () => {
|
||||
expect(codeEnvironmentUserConfigSchema.parse({})).toEqual({});
|
||||
expect(codeEnvironmentUserConfigSchema.parse({ admission: {} }).admission).toEqual({
|
||||
initialDelayMs: 1_000,
|
||||
maxDelayMs: 30_000,
|
||||
multiplier: 1,
|
||||
jitterRatio: 0,
|
||||
});
|
||||
expect(
|
||||
codeEnvironmentUserConfigSchema.parse({
|
||||
admission: { queueWaitMs: 180_000, multiplier: 2, jitterRatio: 0.2 },
|
||||
limits: { maxRequestTimeoutMs: 220_000, maxRunTimeoutMs: 400_000 },
|
||||
}),
|
||||
).toMatchObject({
|
||||
admission: { queueWaitMs: 180_000, multiplier: 2, jitterRatio: 0.2 },
|
||||
limits: { maxRequestTimeoutMs: 220_000, maxRunTimeoutMs: 400_000 },
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ queueWaitMs: 0 },
|
||||
{ queueWaitMs: 300_001 },
|
||||
{ initialDelayMs: 99 },
|
||||
{ initialDelayMs: 2_000, maxDelayMs: 1_000 },
|
||||
{ multiplier: 0 },
|
||||
{ jitterRatio: -0.1 },
|
||||
{ jitterRatio: 1.1 },
|
||||
{ maxDelayMs: Infinity },
|
||||
{ queueWaitMs: 1.5 },
|
||||
{ unexpected: true },
|
||||
])('rejects an invalid policy %j', (admission) => {
|
||||
expect(codeEnvironmentUserConfigSchema.safeParse({ admission }).success).toBe(false);
|
||||
});
|
||||
|
||||
it.each([0, 1, -1, 1.5, 20_000, 610_001, Infinity])(
|
||||
'rejects an invalid run deadline %s',
|
||||
(maxRunTimeoutMs) => {
|
||||
expect(
|
||||
codeEnvironmentUserConfigSchema.safeParse({ limits: { maxRunTimeoutMs } }).success,
|
||||
).toBe(false);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1281,6 +1281,22 @@ export const CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS =
|
|||
CODE_ENVIRONMENT_COMMAND_TIMEOUT_HARD_MAX_MS +
|
||||
CODE_ENVIRONMENT_COMMAND_BUDGET_GRACE_MS;
|
||||
|
||||
export const codeEnvironmentAdmissionSchema = z
|
||||
.object({
|
||||
/** Optional per-request queue ceiling, bounded by transport and execution reserves. */
|
||||
queueWaitMs: z.number().int().min(1).max(CODE_ENVIRONMENT_ADMISSION_MAX_MS).optional(),
|
||||
initialDelayMs: z.number().int().min(100).max(30_000).optional().default(1_000),
|
||||
maxDelayMs: z.number().int().min(100).max(300_000).optional().default(30_000),
|
||||
multiplier: z.number().min(1).max(10).optional().default(1),
|
||||
/** Additive jitter never advances a server's Retry-After hint. */
|
||||
jitterRatio: z.number().min(0).max(1).optional().default(0),
|
||||
})
|
||||
.strict()
|
||||
.refine((policy) => policy.maxDelayMs >= policy.initialDelayMs, {
|
||||
path: ['maxDelayMs'],
|
||||
message: 'Maximum retry delay must cover the initial delay',
|
||||
});
|
||||
|
||||
/**
|
||||
* Typed user-tunable surface for one attached code environment. Omitted fields
|
||||
* remain fixed at LibreChat's safe baseline. Isolation, networking, mounts,
|
||||
|
|
@ -1295,6 +1311,7 @@ export const codeEnvironmentUserConfigSchema = z
|
|||
})
|
||||
.strict()
|
||||
.optional(),
|
||||
admission: codeEnvironmentAdmissionSchema.optional(),
|
||||
limits: z
|
||||
.object({
|
||||
/** Foreground Bash timeout when the call omits timeoutMs. Omission keeps 30 seconds;
|
||||
|
|
@ -1331,19 +1348,25 @@ export const codeEnvironmentUserConfigSchema = z
|
|||
.min(0)
|
||||
.max(CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS)
|
||||
.optional(),
|
||||
/** Total HTTP budget for one workspace tool call, including retries,
|
||||
* execution, settlement, and delivery. Only set this after verifying
|
||||
* the shortest timeout on the actual Code API path and updating Code API
|
||||
* to honor per-request queue allowances. Omission keeps the 30-second
|
||||
* per-attempt admission budget. */
|
||||
/** Transport ceiling after verifying the shortest timeout on the Code API path.
|
||||
* Also the overall call budget unless maxRunTimeoutMs is set. Omission keeps
|
||||
* the legacy per-attempt admission and execution budgets. */
|
||||
maxRequestTimeoutMs: z
|
||||
.number()
|
||||
.int()
|
||||
.min(1)
|
||||
.max(CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS)
|
||||
.optional(),
|
||||
/** Separate overall call deadline across credentials, backoff, and HTTP attempts.
|
||||
* Omission preserves maxRequestTimeoutMs as the total budget. */
|
||||
maxRunTimeoutMs: z
|
||||
.number()
|
||||
.int()
|
||||
.min(1)
|
||||
.max(CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS)
|
||||
.optional(),
|
||||
/** Admission allowance before local dispatch overhead for a Bash command inside
|
||||
* maxRequestTimeoutMs. Omission reserves ten seconds; ignored without a total HTTP budget. */
|
||||
* the transport/run budgets. Omission reserves ten seconds. */
|
||||
minCommandAdmissionMs: z
|
||||
.number()
|
||||
.int()
|
||||
|
|
@ -1353,19 +1376,22 @@ export const codeEnvironmentUserConfigSchema = z
|
|||
})
|
||||
.strict()
|
||||
.superRefine((limits, context) => {
|
||||
const budgetMs = Math.min(
|
||||
limits.maxRequestTimeoutMs ?? Infinity,
|
||||
limits.maxRunTimeoutMs ?? Infinity,
|
||||
);
|
||||
if (
|
||||
limits.maxRequestTimeoutMs == null ||
|
||||
limits.maxRequestTimeoutMs >
|
||||
(limits.minCommandAdmissionMs ?? CODE_ENVIRONMENT_COMMAND_ADMISSION_DEFAULT_MS) +
|
||||
CODE_ENVIRONMENT_COMMAND_BUDGET_GRACE_MS
|
||||
budgetMs >
|
||||
(limits.minCommandAdmissionMs ?? CODE_ENVIRONMENT_COMMAND_ADMISSION_DEFAULT_MS) +
|
||||
CODE_ENVIRONMENT_COMMAND_BUDGET_GRACE_MS
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const field =
|
||||
limits.maxRunTimeoutMs === budgetMs ? 'maxRunTimeoutMs' : 'maxRequestTimeoutMs';
|
||||
context.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: [
|
||||
limits.minCommandAdmissionMs == null ? 'maxRequestTimeoutMs' : 'minCommandAdmissionMs',
|
||||
],
|
||||
path: [limits.minCommandAdmissionMs == null ? field : 'minCommandAdmissionMs'],
|
||||
message: 'Command admission and settlement reserves must leave time for execution',
|
||||
});
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue