mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-17 21:43:06 +00:00
🎟️ fix: Preserve BYOM Tool Calls Through Capacity Windows (#16060)
* ⏳ fix: Keep BYOM tools queued across capacity windows * ⏳ fix: Configure the admission budget and refresh queued credentials Addresses the Codex review on #16060: - Add `limits.maxQueueWaitMs` to the code environment config schema so a deployment can shorten or disable the five-minute admission budget from `librechat.yaml`; omission keeps the merged behavior, `0` surfaces the first capacity expiry to the model. Resolved through `resolveAttachedWorkspaceQueueWaitMs` and passed by ToolService, matching how `maxCommandTimeoutMs` already flows. - Accept an auth-header supplier in `executeWorkspaceTool` and mint it per admission attempt, so a call queued past the Code API token TTL presents fresh credentials instead of failing permanently with 401. The attached Bash tool and the workspace file operations now pass suppliers. - Recheck the deadline after a clamped wait and throw the saved rejection rather than opening an admission window that could still be admitted once the budget is spent. * fix: Complete workspace admission wiring and caller regressions * fix: Share protected edit retry budgets and schema limits --------- Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
parent
c821638e3d
commit
c0affede00
12 changed files with 739 additions and 155 deletions
|
|
@ -8,6 +8,7 @@ import {
|
|||
createAttachedWorkspaceBashTool,
|
||||
createGitIdentityProgrammaticBashTool,
|
||||
resolveAttachedWorkspaceCommandTimeoutMax,
|
||||
resolveAttachedWorkspaceQueueWaitMs,
|
||||
} from './command';
|
||||
|
||||
describe('programmatic Bash Git identity', () => {
|
||||
|
|
@ -265,6 +266,17 @@ describe('createAttachedWorkspaceBashTool', () => {
|
|||
expect(resolveAttachedWorkspaceCommandTimeoutMax()).toBe(30_000);
|
||||
});
|
||||
|
||||
test('resolves the administrator-configured admission budget', () => {
|
||||
expect(resolveAttachedWorkspaceQueueWaitMs()).toBe(5 * 60_000);
|
||||
expect(resolveAttachedWorkspaceQueueWaitMs({ limits: { maxQueueWaitMs: 30_000 } })).toBe(
|
||||
30_000,
|
||||
);
|
||||
expect(resolveAttachedWorkspaceQueueWaitMs({ limits: { maxQueueWaitMs: 0 } })).toBe(0);
|
||||
expect(resolveAttachedWorkspaceQueueWaitMs({ limits: { maxQueueWaitMs: 10 * 60_000 } })).toBe(
|
||||
5 * 60_000,
|
||||
);
|
||||
});
|
||||
|
||||
test('lowers the omitted timeout when the deployment ceiling is below 30 seconds', async () => {
|
||||
const fetchImpl: CodeBridgeFetch = jest.fn(async () => commandResponse());
|
||||
const bashTool = createAttachedWorkspaceBashTool({
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import {
|
|||
executeWorkspaceTool,
|
||||
WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS,
|
||||
WORKSPACE_COMMAND_MAX_TIMEOUT_MS,
|
||||
WORKSPACE_QUEUE_MAX_WAIT_MS,
|
||||
} from './workspace';
|
||||
|
||||
const DEFAULT_OUTPUT_BYTES = 256 * 1024;
|
||||
|
|
@ -93,6 +94,21 @@ export function resolveAttachedWorkspaceCommandTimeoutMax(
|
|||
: normalizeAttachedWorkspaceCommandTimeoutMax(configured);
|
||||
}
|
||||
|
||||
/**
|
||||
* Client retry horizon for one capacity-blocked invocation. `0` surfaces the
|
||||
* first capacity expiry without retrying; an in-flight server admission window
|
||||
* and execution retain their own budgets.
|
||||
*/
|
||||
export function resolveAttachedWorkspaceQueueWaitMs(
|
||||
configSchema?: CodeEnvironmentUserConfigSchema,
|
||||
): number {
|
||||
const configured = configSchema?.limits?.maxQueueWaitMs;
|
||||
if (configured == null || !Number.isSafeInteger(configured) || configured < 0) {
|
||||
return WORKSPACE_QUEUE_MAX_WAIT_MS;
|
||||
}
|
||||
return Math.min(WORKSPACE_QUEUE_MAX_WAIT_MS, configured);
|
||||
}
|
||||
|
||||
export function buildAttachedWorkspaceBashSchema(
|
||||
maxTimeoutMs: number = WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS,
|
||||
environment?: CodeWorkspaceDescriptor['environment'],
|
||||
|
|
@ -235,6 +251,7 @@ export function createAttachedWorkspaceBashTool({
|
|||
environment,
|
||||
gitIdentity,
|
||||
maxTimeoutMs = WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS,
|
||||
maxQueueWaitMs,
|
||||
fetchImpl,
|
||||
}: {
|
||||
baseUrl: string;
|
||||
|
|
@ -244,6 +261,8 @@ export function createAttachedWorkspaceBashTool({
|
|||
gitIdentity?: AgentGitIdentity | null;
|
||||
/** Deployment ceiling already intersected with the protocol hard cap. */
|
||||
maxTimeoutMs?: number;
|
||||
/** Deployment admission budget; omitted keeps the built-in default. */
|
||||
maxQueueWaitMs?: number;
|
||||
fetchImpl?: CodeBridgeFetch;
|
||||
}): DynamicStructuredTool {
|
||||
const effectiveMaxTimeoutMs = normalizeAttachedWorkspaceCommandTimeoutMax(maxTimeoutMs);
|
||||
|
|
@ -301,7 +320,8 @@ export function createAttachedWorkspaceBashTool({
|
|||
try {
|
||||
const result = await executeWorkspaceTool({
|
||||
baseURL: baseUrl,
|
||||
authHeaders: await authHeaders(),
|
||||
/** Passed as a supplier: a queued call outlives its minted token. */
|
||||
authHeaders,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'execute_command',
|
||||
|
|
@ -316,6 +336,7 @@ export function createAttachedWorkspaceBashTool({
|
|||
},
|
||||
signal,
|
||||
fetchImpl,
|
||||
...(maxQueueWaitMs == null ? {} : { maxQueueWaitMs }),
|
||||
});
|
||||
if (result.operation !== 'execute_command') {
|
||||
throw new Error('Attached workspace returned an unexpected command result.');
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ describe('workspace admission feedback', () => {
|
|||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
maxQueueWaitMs: 0,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'execute_command',
|
||||
|
|
@ -26,6 +27,261 @@ describe('workspace admission feedback', () => {
|
|||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('keeps one tool call queued when capacity expires before admission', async () => {
|
||||
const unavailable = () =>
|
||||
new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), {
|
||||
status: 503,
|
||||
headers: { 'Retry-After': '0' },
|
||||
});
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(unavailable())
|
||||
.mockResolvedValueOnce(unavailable())
|
||||
.mockResolvedValueOnce(
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
protocolVersion: 1,
|
||||
operation: 'edit_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'src/app.ts',
|
||||
replacements: 1,
|
||||
bytesWritten: 24,
|
||||
}),
|
||||
{ status: 200, headers: { 'Content-Type': 'application/json' } },
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'edit_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'src/app.ts',
|
||||
edits: [{ oldText: 'const old = true;', newText: 'const ready = true;' }],
|
||||
},
|
||||
}),
|
||||
).resolves.toMatchObject({ operation: 'edit_file', replacements: 1 });
|
||||
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(3);
|
||||
expect(fetchImpl.mock.calls.map((call) => call[1]?.body)).toEqual([
|
||||
fetchImpl.mock.calls[0][1]?.body,
|
||||
fetchImpl.mock.calls[0][1]?.body,
|
||||
fetchImpl.mock.calls[0][1]?.body,
|
||||
]);
|
||||
});
|
||||
|
||||
test('mints fresh credentials for every admission attempt', async () => {
|
||||
let minted = 0;
|
||||
const authHeaders = jest.fn(async () => ({ Authorization: `Bearer token-${++minted}` }));
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), {
|
||||
status: 503,
|
||||
headers: { 'Retry-After': '0' },
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
protocolVersion: 1,
|
||||
operation: 'edit_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'src/app.ts',
|
||||
replacements: 1,
|
||||
bytesWritten: 24,
|
||||
}),
|
||||
{ status: 200, headers: { 'Content-Type': 'application/json' } },
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders,
|
||||
fetchImpl,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'edit_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'src/app.ts',
|
||||
edits: [{ oldText: 'const old = true;', newText: 'const ready = true;' }],
|
||||
},
|
||||
}),
|
||||
).resolves.toMatchObject({ operation: 'edit_file' });
|
||||
|
||||
// A token minted before the first capacity window expires within the
|
||||
// admission budget, so reusing it would fail the retry with a 401.
|
||||
expect(authHeaders).toHaveBeenCalledTimes(2);
|
||||
expect(
|
||||
fetchImpl.mock.calls.map(
|
||||
(call) => (call[1]?.headers as Record<string, string>).Authorization,
|
||||
),
|
||||
).toEqual(['Bearer token-1', 'Bearer token-2']);
|
||||
});
|
||||
|
||||
test('never opens another admission window once the queue budget is spent', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const fetchImpl = jest.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), {
|
||||
status: 503,
|
||||
headers: { 'Retry-After': '1' },
|
||||
}),
|
||||
);
|
||||
|
||||
const result = executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
maxQueueWaitMs: 150,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'edit_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'src/app.ts',
|
||||
edits: [{ oldText: 'const old = true;', newText: 'const ready = true;' }],
|
||||
},
|
||||
}).catch((error: Error) => error);
|
||||
|
||||
await jest.advanceTimersByTimeAsync(150);
|
||||
expect(await result).toMatchObject({
|
||||
message: expect.stringContaining('The operation was not started'),
|
||||
});
|
||||
|
||||
// The clamped wait lands on the deadline: a further dispatch could still be
|
||||
// admitted and run a long command after the budget expired.
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test.each([true, false])(
|
||||
'does not dispatch when cancelled before or during credential refresh (%s)',
|
||||
async (beforeRefresh) => {
|
||||
const controller = new AbortController();
|
||||
const reason = new DOMException('Stopped', 'AbortError');
|
||||
const authHeaders = jest.fn(async () => {
|
||||
controller.abort(reason);
|
||||
return {};
|
||||
});
|
||||
const fetchImpl = jest.fn();
|
||||
if (beforeRefresh) controller.abort(reason);
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders,
|
||||
fetchImpl,
|
||||
signal: controller.signal,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'read_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'src/app.ts',
|
||||
},
|
||||
}),
|
||||
).rejects.toBe(reason);
|
||||
expect(fetchImpl).not.toHaveBeenCalled();
|
||||
expect(authHeaders).toHaveBeenCalledTimes(beforeRefresh ? 0 : 1);
|
||||
},
|
||||
);
|
||||
|
||||
test('does not dispatch a retry if credential refresh spends the remaining queue budget', async () => {
|
||||
const now = jest.spyOn(Date, 'now').mockReturnValue(1000);
|
||||
const authHeaders = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce({})
|
||||
.mockImplementationOnce(async () => {
|
||||
now.mockReturnValue(2000);
|
||||
return {};
|
||||
});
|
||||
const fetchImpl = jest.fn().mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), {
|
||||
status: 503,
|
||||
headers: { 'Retry-After': '0' },
|
||||
}),
|
||||
);
|
||||
try {
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders,
|
||||
fetchImpl,
|
||||
maxQueueWaitMs: 1000,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'read_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'src/app.ts',
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow('The operation was not started');
|
||||
expect(authHeaders).toHaveBeenCalledTimes(2);
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
now.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
test('stops waiting for capacity when the chat is cancelled', async () => {
|
||||
const controller = new AbortController();
|
||||
const reason = new DOMException('Stopped', 'AbortError');
|
||||
const fetchImpl = jest.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), {
|
||||
status: 503,
|
||||
headers: { 'Retry-After': '1' },
|
||||
}),
|
||||
);
|
||||
const request = executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
signal: controller.signal,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'read_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'src/app.ts',
|
||||
},
|
||||
});
|
||||
while (fetchImpl.mock.calls.length === 0) await Promise.resolve();
|
||||
controller.abort(reason);
|
||||
|
||||
await expect(request).rejects.toBe(reason);
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('never retries an ambiguous capacity-looking failure', async () => {
|
||||
const fetchImpl = jest.fn().mockResolvedValue(
|
||||
new Response('<html>Gateway unavailable</html>', {
|
||||
status: 503,
|
||||
headers: { 'Retry-After': '0' },
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'edit_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'src/app.ts',
|
||||
edits: [{ oldText: 'const old = true;', newText: 'const ready = true;' }],
|
||||
},
|
||||
}),
|
||||
).rejects.toMatchObject({ upstreamStatus: 503 });
|
||||
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test.each([
|
||||
[504, '{"code":"ASSIGNMENT_EXPIRED"}', false],
|
||||
[503, '<html>Gateway unavailable</html>', false],
|
||||
|
|
@ -42,6 +298,51 @@ describe('workspace admission feedback', () => {
|
|||
});
|
||||
|
||||
describe('executeWorkspaceTool', () => {
|
||||
test.each([0, 100])(
|
||||
'preserves admitted command execution beyond a %i ms retry horizon',
|
||||
async (maxQueueWaitMs) => {
|
||||
const now = jest.spyOn(Date, 'now').mockReturnValue(1000);
|
||||
const timeout = jest.spyOn(AbortSignal, 'timeout');
|
||||
const fetchImpl = jest.fn(async (...[_url, init]: Parameters<CodeBridgeFetch>) => {
|
||||
// The endpoint does not report admission separately. A successful command
|
||||
// may finish after the client retry horizon without being cancelled.
|
||||
now.mockReturnValue(2000);
|
||||
expect(init?.signal?.aborted).toBe(false);
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
protocolVersion: 1,
|
||||
operation: 'execute_command',
|
||||
workspaceId: 'primary',
|
||||
stdout: 'ready',
|
||||
stderr: '',
|
||||
exitCode: 0,
|
||||
timedOut: false,
|
||||
truncated: false,
|
||||
}),
|
||||
{ status: 200 },
|
||||
);
|
||||
});
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
maxQueueWaitMs,
|
||||
fetchImpl,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'execute_command',
|
||||
workspaceId: 'primary',
|
||||
command: 'echo ready',
|
||||
timeoutMs: 300000,
|
||||
},
|
||||
}),
|
||||
).resolves.toMatchObject({ stdout: 'ready', exitCode: 0 });
|
||||
expect(timeout).toHaveBeenCalledTimes(1);
|
||||
expect(timeout).toHaveBeenCalledWith(340000);
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
},
|
||||
);
|
||||
|
||||
test.each<[WorkspaceToolRequest, number]>([
|
||||
[
|
||||
{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' },
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS } from 'librechat-data-provider';
|
||||
import type { CodeBridgeFetch } from './bridge';
|
||||
|
||||
const WORKSPACE_TOOL_TIMEOUT_MS = 30_000;
|
||||
|
|
@ -20,6 +21,9 @@ const MAX_COMMAND_SIGNAL_LENGTH = 32;
|
|||
const WORKSPACE_COMMAND_TRANSPORT_GRACE_MS = 5_000;
|
||||
/** Matches Code API's bounded admission wait and command settlement allowance. */
|
||||
const WORKSPACE_QUEUE_TIMEOUT_MS = 30_000;
|
||||
/** Compatibility export; the deployment schema owns the default and hard cap. */
|
||||
export const WORKSPACE_QUEUE_MAX_WAIT_MS: number = CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS;
|
||||
const WORKSPACE_QUEUE_RETRY_DELAY_MS = 1_000;
|
||||
const WORKSPACE_COMMAND_SETTLEMENT_GRACE_MS = 5_000;
|
||||
const MAX_RESPONSE_BYTES = 4 * 1024 * 1024;
|
||||
const MAX_ERROR_BODY_BYTES = 4096;
|
||||
|
|
@ -282,6 +286,30 @@ function isWorkspaceAdmissionTimeout(status?: number, body?: string, truncated =
|
|||
}
|
||||
}
|
||||
|
||||
function waitForWorkspaceAdmission(delayMs: number, signal?: AbortSignal): Promise<void> {
|
||||
signal?.throwIfAborted();
|
||||
return new Promise((resolve, reject) => {
|
||||
const cleanup = () => signal?.removeEventListener('abort', abort);
|
||||
const finish = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
const abort = () => {
|
||||
clearTimeout(timer);
|
||||
cleanup();
|
||||
reject(signal?.reason);
|
||||
};
|
||||
const timer = setTimeout(finish, delayMs);
|
||||
signal?.addEventListener('abort', abort, { once: true });
|
||||
if (signal?.aborted === true) abort();
|
||||
});
|
||||
}
|
||||
|
||||
function workspaceAdmissionRetryDelay(value: string | null): number {
|
||||
if (value == null || !/^\d+$/.test(value)) return WORKSPACE_QUEUE_RETRY_DELAY_MS;
|
||||
return Math.max(100, Math.min(Number(value) * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS));
|
||||
}
|
||||
|
||||
/** Keep a received HTTP status even if reading its diagnostic body fails or stalls. */
|
||||
async function readErrorBody(
|
||||
response: Response,
|
||||
|
|
@ -703,62 +731,110 @@ function getWorkspaceToolTimeoutMs(request: WorkspaceToolRequest): number {
|
|||
return WORKSPACE_QUEUE_TIMEOUT_MS + executionBudgetMs + WORKSPACE_COMMAND_TRANSPORT_GRACE_MS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Credentials for one admission attempt. A supplier is minted per attempt, so a
|
||||
* call that stays queued past the Code API token TTL presents a fresh token
|
||||
* instead of failing permanently with 401 while capacity is still pending.
|
||||
*/
|
||||
export type WorkspaceToolAuthHeaders =
|
||||
| Record<string, string>
|
||||
| (() => Promise<Record<string, string>> | Record<string, string>);
|
||||
|
||||
export async function executeWorkspaceTool({
|
||||
baseURL,
|
||||
authHeaders,
|
||||
request,
|
||||
signal,
|
||||
fetchImpl = fetch,
|
||||
maxQueueWaitMs = WORKSPACE_QUEUE_MAX_WAIT_MS,
|
||||
}: {
|
||||
baseURL: string;
|
||||
authHeaders: Record<string, string>;
|
||||
authHeaders: WorkspaceToolAuthHeaders;
|
||||
request: WorkspaceToolRequest;
|
||||
signal?: AbortSignal;
|
||||
fetchImpl?: CodeBridgeFetch;
|
||||
maxQueueWaitMs?: number;
|
||||
}): Promise<WorkspaceToolResult> {
|
||||
if (!isValidRequest(request)) {
|
||||
if (
|
||||
!isValidRequest(request) ||
|
||||
!Number.isSafeInteger(maxQueueWaitMs) ||
|
||||
maxQueueWaitMs < 0 ||
|
||||
maxQueueWaitMs > WORKSPACE_QUEUE_MAX_WAIT_MS
|
||||
) {
|
||||
throw new WorkspaceToolHttpError('invalid');
|
||||
}
|
||||
try {
|
||||
const timeoutSignal = AbortSignal.timeout(getWorkspaceToolTimeoutMs(request));
|
||||
const requestSignal =
|
||||
signal != null && typeof AbortSignal.any === 'function'
|
||||
? AbortSignal.any([signal, timeoutSignal])
|
||||
: timeoutSignal;
|
||||
const response = await fetchImpl(
|
||||
`${baseURL.trim().replace(/\/+$/, '')}/workspace-tools/execute`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
...authHeaders,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(request),
|
||||
redirect: 'error',
|
||||
signal: requestSignal,
|
||||
},
|
||||
);
|
||||
if (!response.ok) {
|
||||
const { body, truncated } = await readErrorBody(response, requestSignal);
|
||||
const queueDeadlineAt = Date.now() + maxQueueWaitMs;
|
||||
const body = JSON.stringify(request);
|
||||
let lastAdmissionRejection: WorkspaceToolHttpError | undefined;
|
||||
while (true) {
|
||||
try {
|
||||
signal?.throwIfAborted();
|
||||
throw new WorkspaceToolHttpError('rejected', response.status, body, truncated);
|
||||
/** The endpoint replies only after execution settles. Capping this by the
|
||||
* retry horizon would also abort already-admitted commands, with an unknown
|
||||
* mutation outcome. Server admission and execution retain separate budgets. */
|
||||
const timeoutSignal = AbortSignal.timeout(getWorkspaceToolTimeoutMs(request));
|
||||
const requestSignal =
|
||||
signal != null && typeof AbortSignal.any === 'function'
|
||||
? AbortSignal.any([signal, timeoutSignal])
|
||||
: timeoutSignal;
|
||||
const attemptHeaders = typeof authHeaders === 'function' ? await authHeaders() : authHeaders;
|
||||
requestSignal.throwIfAborted();
|
||||
if (lastAdmissionRejection && Date.now() >= queueDeadlineAt) {
|
||||
throw lastAdmissionRejection;
|
||||
}
|
||||
const response = await fetchImpl(
|
||||
`${baseURL.trim().replace(/\/+$/, '')}/workspace-tools/execute`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
...attemptHeaders,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body,
|
||||
redirect: 'error',
|
||||
signal: requestSignal,
|
||||
},
|
||||
);
|
||||
if (!response.ok) {
|
||||
const { body, truncated } = await readErrorBody(response, requestSignal);
|
||||
signal?.throwIfAborted();
|
||||
const rejection = new WorkspaceToolHttpError('rejected', response.status, body, truncated);
|
||||
if (
|
||||
!isWorkspaceAdmissionTimeout(response.status, body, truncated) ||
|
||||
Date.now() >= queueDeadlineAt
|
||||
) {
|
||||
throw rejection;
|
||||
}
|
||||
const delayMs = Math.min(
|
||||
workspaceAdmissionRetryDelay(response.headers.get('Retry-After')),
|
||||
queueDeadlineAt - Date.now(),
|
||||
);
|
||||
lastAdmissionRejection = rejection;
|
||||
await waitForWorkspaceAdmission(delayMs, signal);
|
||||
/** A clamped delay can land exactly on the deadline: the budget is spent,
|
||||
* so never open another admission window that could still be admitted. */
|
||||
if (Date.now() >= queueDeadlineAt) {
|
||||
throw rejection;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const result = await readBoundedJson(response, requestSignal);
|
||||
if (!isValidResult(request, result)) {
|
||||
throw new WorkspaceToolHttpError('invalid');
|
||||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (error instanceof WorkspaceToolHttpError) throw error;
|
||||
if (
|
||||
signal?.aborted === true &&
|
||||
(error === signal.reason || (isRecord(error) && error.name === 'AbortError'))
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
if (error instanceof Error && error.name === 'TimeoutError') {
|
||||
throw new WorkspaceToolHttpError('timeout');
|
||||
}
|
||||
throw new WorkspaceToolHttpError('failed');
|
||||
}
|
||||
const result = await readBoundedJson(response, requestSignal);
|
||||
if (!isValidResult(request, result)) {
|
||||
throw new WorkspaceToolHttpError('invalid');
|
||||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (error instanceof WorkspaceToolHttpError) throw error;
|
||||
if (
|
||||
signal?.aborted === true &&
|
||||
(error === signal.reason || (isRecord(error) && error.name === 'AbortError'))
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
if (error instanceof Error && error.name === 'TimeoutError') {
|
||||
throw new WorkspaceToolHttpError('timeout');
|
||||
}
|
||||
throw new WorkspaceToolHttpError('failed');
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue