🔁 fix: Retry Code API Rate Limits That Reject Before a Workspace Call Starts (#16453)
Some checks are pending
Backend Unit Tests / Build packages (push) Waiting to run
Backend Unit Tests / Codegraph select (push) Waiting to run
Backend Unit Tests / TypeScript type checks (push) Blocked by required conditions
Backend Unit Tests / Circular dependency checks (push) Waiting to run
Backend Unit Tests / Tests: api (shard 1/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 2/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 3/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: data-provider (push) Blocked by required conditions
Backend Unit Tests / Tests: data-schemas (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 1/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 2/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 3/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 4/4) (push) Blocked by required conditions
Codegraph E2E Votes / vote (full suite) (push) Waiting to run
Docker Dev Branch Images Build / publish (push) Waiting to run
Frontend Unit Tests / Client build recovery regression (push) Waiting to run
Frontend Unit Tests / Codegraph select (push) Waiting to run
Frontend Unit Tests / Build packages (push) Waiting to run
Frontend Unit Tests / TypeScript type checks (client) (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/client (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 1/2) (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 2/2) (push) Blocked by required conditions
Frontend Unit Tests / Vite build verification (push) Blocked by required conditions
Sync Locize Translations & Create Translation PR / Sync Translation Keys with Locize (push) Waiting to run
Sync Locize Translations & Create Translation PR / Create Translation PR on Version Published (push) Blocked by required conditions

* 🔁 fix: Retry Code API Rate Limits That Reject Before a Workspace Call Starts

* 🔁 fix: Bound Workspace Rate Limit Retries by Code API Budget

* 🔗 fix: Isolate Code API Retry Default from Utility Graph

* 🧩 fix: Type the Shared Code API Retry Default

* 🔁 fix: Keep Workspace Admission Retry Budgets Independent

---------

Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
Danny Avila 2026-09-28 15:45:13 -04:00 • committed by GitHub
parent 40bb16edfd
commit f624ad0876
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 779 additions and 44 deletions

View file

@ -164,6 +164,24 @@ function commandResponse(overrides: Record<string, unknown> = {}): Response {
}
describe('createAttachedWorkspaceBashTool', () => {
test('honors a zero Code API rate-limit retry budget', async () => {
const fetchImpl: CodeBridgeFetch = jest.fn(
async () => new Response(JSON.stringify({ error: 'rate_limited' }), { status: 429 }),
);
const bashTool = createAttachedWorkspaceBashTool({
baseUrl: 'https://code.example.com/v1',
authHeaders: () => ({}),
workspaceId: 'project-a',
codeApiMaxRetryWaitMs: 0,
fetchImpl,
});
await expect(bashTool.invoke({ command: 'pwd' })).rejects.toThrow(
'The operation was not started',
);
expect(fetchImpl).toHaveBeenCalledTimes(1);
});
test('dispatches commands to the resolved conversation workspace instance', async () => {
const fetchImpl: CodeBridgeFetch = jest.fn(async () => commandResponse());
const workspaceInstanceId = 'e'.repeat(64);

View file

@ -302,6 +302,7 @@ export function createAttachedWorkspaceBashTool({
gitIdentity,
maxTimeoutMs = WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS,
maxQueueWaitMs,
codeApiMaxRetryWaitMs,
maxRequestTimeoutMs,
fetchImpl,
}: {
@ -315,6 +316,7 @@ export function createAttachedWorkspaceBashTool({
maxTimeoutMs?: number;
/** Retry horizon across typed queue expirations, not an admission budget. */
maxQueueWaitMs?: number;
codeApiMaxRetryWaitMs?: number;
/** Verified total HTTP budget; omission keeps the legacy per-attempt timeout. */
maxRequestTimeoutMs?: number;
fetchImpl?: CodeBridgeFetch;
@ -395,6 +397,7 @@ export function createAttachedWorkspaceBashTool({
signal,
fetchImpl,
...(maxQueueWaitMs == null ? {} : { maxQueueWaitMs }),
...(codeApiMaxRetryWaitMs == null ? {} : { codeApiMaxRetryWaitMs }),
...(maxRequestTimeoutMs == null ? {} : { maxRequestTimeoutMs }),
});
if (result.operation !== 'execute_command') {

View file

@ -37,6 +37,72 @@ const response = () =>
);
describe('repository instruction loading', () => {
it('does not retry a typed 429 when the configured Code API wait budget is zero', async () => {
const fetchImpl = jest
.fn()
.mockResolvedValueOnce(
new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 0 }), {
status: 429,
headers: { 'Retry-After': '0' },
}),
)
.mockResolvedValueOnce(response());
const result = await createRepositoryInstructionLoader()({
enabled: true,
context,
principalId: 'alice',
authHeaders: async () => ({}),
assertContent: jest.fn(),
codeApiMaxRetryWaitMs: 0,
fetchImpl,
});
expect(result).toBeUndefined();
expect(fetchImpl).toHaveBeenCalledTimes(1);
});
it('refreshes instruction-read authorization for a typed 429 retry', async () => {
jest.useFakeTimers();
try {
const fetchImpl = jest
.fn()
.mockResolvedValueOnce(
new Response(JSON.stringify({ error: 'rate_limited' }), {
status: 429,
headers: { 'Retry-After': '0' },
}),
)
.mockResolvedValueOnce(response());
let minted = 0;
const authHeaders = jest.fn(async () => ({ Authorization: `Bearer token-${++minted}` }));
const load = createRepositoryInstructionLoader();
const input = {
enabled: true,
context,
principalId: 'alice',
authHeaders,
fetchImpl,
codeApiMaxRetryWaitMs: 100,
timeoutMs: 500,
assertContent: jest.fn(),
};
const result = load(input);
await jest.advanceTimersByTimeAsync(100);
await expect(result).resolves.toContain('Use the project test command.');
expect(fetchImpl).toHaveBeenCalledTimes(2);
expect(fetchImpl.mock.calls.map((call) => call[1]?.headers.Authorization)).toEqual([
'Bearer token-1',
'Bearer token-2',
]);
await expect(load(input)).resolves.toContain('Use the project test command.');
expect(authHeaders).toHaveBeenCalledTimes(3);
expect(fetchImpl).toHaveBeenCalledTimes(2);
} finally {
jest.useRealTimers();
}
});
it('bounds optional authorization waits and preserves explicit cancellation', async () => {
jest.useFakeTimers();
try {

View file

@ -9,6 +9,7 @@ export interface RepositoryInstructionSource {
context: CodeExecutionContext;
principalId: string;
authHeaders: () => Promise<Record<string, string>>;
codeApiMaxRetryWaitMs?: number;
load: ReturnType<typeof createRepositoryInstructionLoader>;
}
@ -54,6 +55,7 @@ export function createRepositoryInstructionLoader() {
signal,
fetchImpl,
assertContent,
codeApiMaxRetryWaitMs,
timeoutMs = 2000,
}: {
enabled: boolean;
@ -64,6 +66,7 @@ export function createRepositoryInstructionLoader() {
signal?: AbortSignal;
fetchImpl?: CodeBridgeFetch;
assertContent: (content: string) => void;
codeApiMaxRetryWaitMs?: number;
timeoutMs?: number;
}): Promise<string | undefined> => {
const workspace = context.codeWorkspace;
@ -93,13 +96,13 @@ export function createRepositoryInstructionLoader() {
const timer = setTimeout(() => deadline.abort(), budget);
const readSignal = signal ? AbortSignal.any([signal, deadline.signal]) : deadline.signal;
try {
const headers = await abortable(authHeaders(), readSignal);
if (content === undefined) {
const result = await executeWorkspaceTool({
baseURL: context.baseUrl,
authHeaders: headers,
authHeaders: () => abortable(authHeaders(), readSignal),
signal: readSignal,
fetchImpl,
codeApiMaxRetryWaitMs,
request: {
protocolVersion: 1,
operation: 'read_file',
@ -121,6 +124,8 @@ export function createRepositoryInstructionLoader() {
content = result.content;
if (cache.size >= 64) cache.delete(cache.keys().next().value!);
cache.set(key, content);
} else {
await abortable(authHeaders(), readSignal);
}
} catch {
if (signal?.aborted) throw signal.reason;

View file

@ -0,0 +1 @@
export const CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS: number = 20_000;

View file

@ -338,6 +338,514 @@ describe('workspace admission feedback', () => {
expect(fetchImpl).toHaveBeenCalledTimes(1);
});
test('cancels a typed rate-limit wait without replaying the request', async () => {
jest.useFakeTimers();
try {
const controller = new AbortController();
const reason = new DOMException('Stopped', 'AbortError');
const fetchImpl = jest
.fn()
.mockResolvedValueOnce(
new Response(JSON.stringify({ error: 'rate_limited' }), {
status: 429,
headers: { 'Retry-After': '1' },
}),
)
.mockResolvedValueOnce(new Response('{}', { status: 504 }));
const request = executeWorkspaceTool({
baseURL: 'https://code.example/v1',
authHeaders: {},
signal: controller.signal,
fetchImpl,
maxQueueWaitMs: 0,
request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
});
await jest.advanceTimersByTimeAsync(1);
controller.abort(reason);
await expect(request).rejects.toBe(reason);
await jest.advanceTimersByTimeAsync(1_000);
expect(fetchImpl).toHaveBeenCalledTimes(1);
} finally {
jest.useRealTimers();
}
});
test('retries a Code API rate limit, which rejects before the operation starts', async () => {
const rateLimited = () =>
new Response(
JSON.stringify({
error: 'rate_limited',
message: 'Too many CodeAPI execution requests. Please retry in 1 second.',
retry_after_seconds: 1,
}),
{ status: 429, headers: { 'Retry-After': '0' } },
);
const fetchImpl = jest
.fn()
.mockResolvedValueOnce(rateLimited())
.mockResolvedValueOnce(rateLimited())
.mockResolvedValueOnce(
new Response(
JSON.stringify({
protocolVersion: 1,
operation: 'edit_file',
workspaceId: 'primary',
path: 'src/app.ts',
replacements: 1,
bytesWritten: 24,
}),
{ status: 200, headers: { 'Content-Type': 'application/json' } },
),
);
await expect(
executeWorkspaceTool({
baseURL: 'https://code.example/v1',
authHeaders: {},
fetchImpl,
request: {
protocolVersion: 1,
operation: 'edit_file',
workspaceId: 'primary',
path: 'src/app.ts',
edits: [{ oldText: 'const old = true;', newText: 'const ready = true;' }],
},
}),
).resolves.toMatchObject({ operation: 'edit_file', replacements: 1 });
expect(fetchImpl).toHaveBeenCalledTimes(3);
const bodies = fetchImpl.mock.calls.map((call) => call[1]?.body);
expect(new Set(bodies).size).toBe(1);
});
test.each([
['missing', undefined],
['invalid', 'invalid'],
['overflowing', '9'.repeat(400)],
] as const)('uses the typed 429 body delay when Retry-After is %s', async (_case, retryAfter) => {
jest.useFakeTimers();
try {
const fetchImpl = jest
.fn()
.mockResolvedValueOnce(
new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 2 }), {
status: 429,
...(retryAfter ? { headers: { 'Retry-After': retryAfter } } : {}),
}),
)
.mockResolvedValueOnce(new Response('{}', { status: 504 }));
const result = executeWorkspaceTool({
baseURL: 'https://code.example/v1',
authHeaders: {},
fetchImpl,
codeApiMaxRetryWaitMs: 3_000,
request: {
protocolVersion: 1,
operation: 'read_file',
workspaceId: 'primary',
path: 'x',
},
}).catch((error: WorkspaceToolHttpError) => error);
await jest.advanceTimersByTimeAsync(1_999);
expect(fetchImpl).toHaveBeenCalledTimes(1);
await jest.advanceTimersByTimeAsync(1);
await expect(result).resolves.toMatchObject({ upstreamStatus: 504 });
expect(fetchImpl).toHaveBeenCalledTimes(2);
} finally {
jest.useRealTimers();
}
});
test.each([-1, 300_001, 1.5, Number.NaN])(
'rejects an invalid Code API retry limit of %p before transport',
async (codeApiMaxRetryWaitMs) => {
const fetchImpl = jest.fn();
await expect(
executeWorkspaceTool({
baseURL: 'https://code.example/v1',
authHeaders: {},
fetchImpl,
codeApiMaxRetryWaitMs,
request: {
protocolVersion: 1,
operation: 'read_file',
workspaceId: 'primary',
path: 'x',
},
}),
).rejects.toMatchObject({ reason: 'invalid' });
expect(fetchImpl).not.toHaveBeenCalled();
},
);
test.each([0, 150])('bounds typed 429 retries to the %i ms Code API budget', async (budget) => {
jest.useFakeTimers();
try {
const fetchImpl = jest.fn().mockImplementation(
() =>
new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 0 }), {
status: 429,
headers: { 'Retry-After': '0' },
}),
);
const result = executeWorkspaceTool({
baseURL: 'https://code.example/v1',
authHeaders: {},
fetchImpl,
codeApiMaxRetryWaitMs: budget,
request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
}).catch((error: WorkspaceToolHttpError) => error);
await jest.advanceTimersByTimeAsync(budget);
await expect(result).resolves.toMatchObject({
upstreamStatus: 429,
message: expect.stringContaining('The operation was not started'),
});
expect(fetchImpl).toHaveBeenCalledTimes(budget === 0 ? 1 : 2);
} finally {
jest.useRealTimers();
}
});
test('does not wait when a 429 retry hint exceeds the Code API budget', async () => {
const fetchImpl = jest.fn().mockResolvedValue(
new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 30 }), {
status: 429,
}),
);
await expect(
executeWorkspaceTool({
baseURL: 'https://code.example/v1',
authHeaders: {},
fetchImpl,
request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
}),
).rejects.toMatchObject({ upstreamStatus: 429 });
expect(fetchImpl).toHaveBeenCalledTimes(1);
});
test('keeps queue retries enabled when Code API rate-limit retries are disabled', async () => {
jest.useFakeTimers();
try {
const fetchImpl = jest
.fn()
.mockResolvedValueOnce(
new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), {
status: 503,
headers: { 'Retry-After': '0' },
}),
)
.mockResolvedValueOnce(
new Response(JSON.stringify({ error: 'rate_limited' }), { status: 429 }),
);
const result = executeWorkspaceTool({
baseURL: 'https://code.example/v1',
authHeaders: {},
fetchImpl,
codeApiMaxRetryWaitMs: 0,
request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
}).catch((error: WorkspaceToolHttpError) => error);
await jest.advanceTimersByTimeAsync(100);
await expect(result).resolves.toMatchObject({ upstreamStatus: 429 });
expect(fetchImpl).toHaveBeenCalledTimes(2);
} finally {
jest.useRealTimers();
}
});
test('does not charge queue waits to the Code API rate-limit budget', async () => {
jest.useFakeTimers();
try {
const fetchImpl = jest
.fn()
.mockResolvedValueOnce(
new Response(JSON.stringify({ error: 'rate_limited' }), {
status: 429,
headers: { 'Retry-After': '0' },
}),
)
.mockResolvedValueOnce(
new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), {
status: 503,
headers: { 'Retry-After': '0' },
}),
)
.mockResolvedValueOnce(new Response('{}', { status: 504 }));
const result = executeWorkspaceTool({
baseURL: 'https://code.example/v1',
authHeaders: {},
fetchImpl,
codeApiMaxRetryWaitMs: 100,
request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
}).catch((error: WorkspaceToolHttpError) => error);
await jest.advanceTimersByTimeAsync(200);
await expect(result).resolves.toMatchObject({ upstreamStatus: 504 });
expect(fetchImpl).toHaveBeenCalledTimes(3);
} finally {
jest.useRealTimers();
}
});
test.each([0, 50])(
'retries typed 429s after a %i ms queue horizon without re-enabling capacity retries',
async (maxQueueWaitMs) => {
jest.useFakeTimers();
try {
const fetchImpl = jest
.fn()
.mockResolvedValueOnce(
new Response(JSON.stringify({ error: 'rate_limited' }), {
status: 429,
headers: { 'Retry-After': '0' },
}),
)
.mockResolvedValueOnce(new Response('{}', { status: 504 }));
const result = executeWorkspaceTool({
baseURL: 'https://code.example/v1',
authHeaders: {},
fetchImpl,
maxQueueWaitMs,
codeApiMaxRetryWaitMs: 100,
request: {
protocolVersion: 1,
operation: 'read_file',
workspaceId: 'primary',
path: 'x',
},
}).catch((error: WorkspaceToolHttpError) => error);
await jest.advanceTimersByTimeAsync(100);
await expect(result).resolves.toMatchObject({ upstreamStatus: 504 });
expect(fetchImpl).toHaveBeenCalledTimes(2);
} finally {
jest.useRealTimers();
}
},
);
test('continues 429 recovery after a queue retry but never extends its capacity horizon', async () => {
jest.useFakeTimers();
try {
const fetchImpl = jest
.fn()
.mockResolvedValueOnce(
new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), {
status: 503,
headers: { 'Retry-After': '0' },
}),
)
.mockResolvedValueOnce(
new Response(JSON.stringify({ error: 'rate_limited' }), {
status: 429,
headers: { 'Retry-After': '0' },
}),
)
.mockResolvedValueOnce(new Response('{}', { status: 504 }));
const result = executeWorkspaceTool({
baseURL: 'https://code.example/v1',
authHeaders: {},
fetchImpl,
maxQueueWaitMs: 150,
codeApiMaxRetryWaitMs: 100,
request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
}).catch((error: WorkspaceToolHttpError) => error);
await jest.advanceTimersByTimeAsync(200);
await expect(result).resolves.toMatchObject({ upstreamStatus: 504 });
expect(fetchImpl).toHaveBeenCalledTimes(3);
} finally {
jest.useRealTimers();
}
});
test('never borrows the rate-limit horizon to retry a later capacity timeout', async () => {
jest.useFakeTimers();
try {
const fetchImpl = jest
.fn()
.mockResolvedValueOnce(
new Response(JSON.stringify({ error: 'rate_limited' }), {
status: 429,
headers: { 'Retry-After': '0' },
}),
)
.mockResolvedValueOnce(
new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), {
status: 503,
headers: { 'Retry-After': '0' },
}),
)
.mockResolvedValueOnce(new Response('{}', { status: 504 }));
const result = executeWorkspaceTool({
baseURL: 'https://code.example/v1',
authHeaders: {},
fetchImpl,
maxQueueWaitMs: 50,
codeApiMaxRetryWaitMs: 200,
request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
}).catch((error: WorkspaceToolHttpError) => error);
await jest.advanceTimersByTimeAsync(100);
await expect(result).resolves.toMatchObject({ upstreamStatus: 503 });
expect(fetchImpl).toHaveBeenCalledTimes(2);
} finally {
jest.useRealTimers();
}
});
test('retries after an authorized rate-limit wait despite timer jitter', async () => {
jest.useFakeTimers();
const startedAt = Date.now();
let currentTime = startedAt;
jest.spyOn(Date, 'now').mockImplementation(() => currentTime);
try {
const fetchImpl = jest
.fn()
.mockResolvedValueOnce(
new Response(JSON.stringify({ error: 'rate_limited' }), {
status: 429,
headers: { 'Retry-After': '0' },
}),
)
.mockResolvedValueOnce(new Response('{}', { status: 504 }));
const result = executeWorkspaceTool({
baseURL: 'https://code.example/v1',
authHeaders: {},
fetchImpl,
maxQueueWaitMs: 0,
codeApiMaxRetryWaitMs: 100,
request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
}).catch((error: WorkspaceToolHttpError) => error);
await jest.advanceTimersByTimeAsync(0);
expect(fetchImpl).toHaveBeenCalledTimes(1);
currentTime += 101;
await jest.advanceTimersByTimeAsync(100);
await expect(result).resolves.toMatchObject({ upstreamStatus: 504 });
expect(fetchImpl).toHaveBeenCalledTimes(2);
} finally {
jest.useRealTimers();
}
});
test('charges an overslept 429 wait before considering another retry', async () => {
jest.useFakeTimers();
const startedAt = Date.now();
let currentTime = startedAt;
jest.spyOn(Date, 'now').mockImplementation(() => currentTime);
try {
const rateLimited = () =>
new Response(JSON.stringify({ error: 'rate_limited' }), {
status: 429,
headers: { 'Retry-After': '0' },
});
const fetchImpl = jest
.fn()
.mockResolvedValueOnce(rateLimited())
.mockResolvedValueOnce(rateLimited())
.mockResolvedValueOnce(new Response('{}', { status: 504 }));
const result = executeWorkspaceTool({
baseURL: 'https://code.example/v1',
authHeaders: {},
fetchImpl,
maxQueueWaitMs: 0,
codeApiMaxRetryWaitMs: 200,
request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
}).catch((error: WorkspaceToolHttpError) => error);
await jest.advanceTimersByTimeAsync(0);
currentTime += 150;
await jest.advanceTimersByTimeAsync(100);
await expect(result).resolves.toMatchObject({ upstreamStatus: 429 });
expect(fetchImpl).toHaveBeenCalledTimes(2);
} finally {
jest.useRealTimers();
}
});
test('reserves the caller HTTP execution budget when capacity retries are disabled', async () => {
jest.useFakeTimers();
try {
const fetchImpl = jest.fn().mockResolvedValue(
new Response(JSON.stringify({ error: 'rate_limited' }), {
status: 429,
headers: { 'Retry-After': '0' },
}),
);
const result = executeWorkspaceTool({
baseURL: 'https://code.example/v1',
authHeaders: {},
fetchImpl,
maxQueueWaitMs: 0,
codeApiMaxRetryWaitMs: 1_000,
maxRequestTimeoutMs: 35_050,
request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
}).catch((error: WorkspaceToolHttpError) => error);
await jest.advanceTimersByTimeAsync(50);
await expect(result).resolves.toMatchObject({ upstreamStatus: 429 });
expect(fetchImpl).toHaveBeenCalledTimes(1);
} finally {
jest.useRealTimers();
}
});
test('reports a rate limit as not started when its retry budget is disabled', async () => {
const fetchImpl = jest.fn().mockResolvedValue(
new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 1 }), {
status: 429,
headers: { 'Retry-After': '1' },
}),
);
const failure = executeWorkspaceTool({
baseURL: 'https://code.example/v1',
authHeaders: {},
fetchImpl,
codeApiMaxRetryWaitMs: 0,
request: {
protocolVersion: 1,
operation: 'execute_command',
workspaceId: 'primary',
command: 'echo test',
},
});
await expect(failure).rejects.toThrow('The operation was not started');
await expect(failure).rejects.toMatchObject({ upstreamStatus: 429 });
expect(fetchImpl).toHaveBeenCalledTimes(1);
});
test('never retries a 429 that does not carry the typed rate-limit body', async () => {
const fetchImpl = jest.fn().mockResolvedValue(
new Response('<html>Too Many Requests</html>', {
status: 429,
headers: { 'Retry-After': '0' },
}),
);
await expect(
executeWorkspaceTool({
baseURL: 'https://code.example/v1',
authHeaders: {},
fetchImpl,
request: {
protocolVersion: 1,
operation: 'edit_file',
workspaceId: 'primary',
path: 'src/app.ts',
edits: [{ oldText: 'const old = true;', newText: 'const ready = true;' }],
},
}),
).rejects.toMatchObject({ upstreamStatus: 429 });
expect(fetchImpl).toHaveBeenCalledTimes(1);
});
test('never retries an ambiguous capacity-looking failure', async () => {
const fetchImpl = jest.fn().mockResolvedValue(
new Response('<html>Gateway unavailable</html>', {
@ -369,6 +877,10 @@ describe('workspace admission feedback', () => {
[503, '<html>Gateway unavailable</html>', false],
[503, '{"code":"WORKSPACE_QUEUE_TIMEOUT"}', true],
[503, 'null', false],
[503, '{"error":"rate_limited"}', false],
[429, '{"code":"WORKSPACE_QUEUE_TIMEOUT"}', false],
[429, '{"error":"rate_limited"}', true],
[429, '<html>Too Many Requests</html>', false],
] as const)(
'does not infer non-execution from an ambiguous response',
(status, body, truncated) => {

View file

@ -4,6 +4,7 @@ import {
CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS,
} from 'librechat-data-provider';
import type { CodeBridgeFetch } from './bridge';
import { CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS } from './limits';
const WORKSPACE_TOOL_TIMEOUT_MS = 30_000;
const MAX_PATH_LENGTH = 4096;
@ -274,13 +275,18 @@ export class WorkspaceToolHttpError extends Error {
public readonly upstreamBodyTruncated = false,
) {
let message = `Workspace tool request ${reason}`;
if (
reason === 'rejected' &&
isWorkspaceAdmissionTimeout(upstreamStatus, upstreamBody, upstreamBodyTruncated)
) {
const admissionRejection =
reason === 'rejected'
? getWorkspaceAdmissionRejection(upstreamStatus, upstreamBody, upstreamBodyTruncated)
: undefined;
if (admissionRejection === 'queue_timeout') {
message =
'Workspace capacity was unavailable before the queue deadline. The operation was not started. Wait for active work to finish or select an independent workspace on a machine with available capacity.';
}
if (admissionRejection === 'rate_limited') {
message =
'The Code API request rate limit was reached. The operation was not started. Wait a few seconds before retrying, or make fewer concurrent workspace calls.';
}
if (reason === 'insufficient_time') {
message =
'Workspace execution cannot fit within the remaining HTTP budget. The operation was not started.';
@ -295,15 +301,38 @@ export class WorkspaceToolHttpError extends Error {
}
}
function isWorkspaceAdmissionTimeout(status?: number, body?: string, truncated = false): boolean {
if (status !== 503 || !body || truncated || body.length > MAX_ERROR_BODY_BYTES) {
return false;
type WorkspaceAdmissionRejection = 'queue_timeout' | 'rate_limited';
/**
* Rejections the Code API issues before an operation is assigned, so retrying cannot repeat a
* mutation: a queue deadline that expired before admission, or its rate limiter, which runs as
* middleware ahead of the workspace router. Anything else, including an unparsable body under
* either status, keeps an unknown outcome and is never retried.
*/
function getWorkspaceAdmissionRejection(
status?: number,
body?: string,
truncated = false,
): WorkspaceAdmissionRejection | undefined {
if (
(status !== 503 && status !== 429) ||
!body ||
truncated ||
body.length > MAX_ERROR_BODY_BYTES
) {
return undefined;
}
try {
const parsed: { code?: string } | null = JSON.parse(body);
return parsed?.code === 'WORKSPACE_QUEUE_TIMEOUT';
const parsed: { code?: string; error?: string } | null = JSON.parse(body);
if (status === 503 && parsed?.code === 'WORKSPACE_QUEUE_TIMEOUT') {
return 'queue_timeout';
}
if (status === 429 && parsed?.error === 'rate_limited') {
return 'rate_limited';
}
return undefined;
} catch {
return false;
return undefined;
}
}
@ -326,9 +355,25 @@ function waitForWorkspaceAdmission(delayMs: number, signal?: AbortSignal): Promi
});
}
function workspaceAdmissionRetryDelay(value: string | null): number {
if (value == null || !/^\d+$/.test(value)) return WORKSPACE_QUEUE_RETRY_DELAY_MS;
return Math.max(100, Math.min(Number(value) * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS));
function workspaceAdmissionRetryDelay(value: string | null, rateLimitBody?: string): number {
if (value != null && /^\d+$/.test(value)) {
const seconds = Number(value);
if (Number.isFinite(seconds)) {
return Math.max(100, Math.min(seconds * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS));
}
}
if (rateLimitBody) {
try {
const parsed: { retry_after_seconds?: number } | null = JSON.parse(rateLimitBody);
const seconds = parsed?.retry_after_seconds;
if (typeof seconds === 'number' && Number.isFinite(seconds) && seconds >= 0) {
return Math.max(100, Math.min(seconds * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS));
}
} catch {
// Fall back when the delay hint is unusable.
}
}
return WORKSPACE_QUEUE_RETRY_DELAY_MS;
}
/** Keep a received HTTP status even if reading its diagnostic body fails or stalls. */
@ -801,6 +846,7 @@ export async function executeWorkspaceTool({
signal,
fetchImpl = fetch,
maxQueueWaitMs = WORKSPACE_QUEUE_MAX_WAIT_MS,
codeApiMaxRetryWaitMs = CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS,
maxRequestTimeoutMs,
deadlineAtMs,
}: {
@ -810,6 +856,8 @@ export async function executeWorkspaceTool({
signal?: AbortSignal;
fetchImpl?: CodeBridgeFetch;
maxQueueWaitMs?: number;
/** Maximum time waiting for Code API rate-limit admission, independent of queue retries. */
codeApiMaxRetryWaitMs?: number;
/** End-to-end HTTP budget for this call. Omission keeps legacy per-attempt timeouts. */
maxRequestTimeoutMs?: number;
/** Optional earlier caller deadline; a signal alone has no remaining-time value. */
@ -820,6 +868,9 @@ export async function executeWorkspaceTool({
!Number.isSafeInteger(maxQueueWaitMs) ||
maxQueueWaitMs < 0 ||
maxQueueWaitMs > WORKSPACE_QUEUE_MAX_WAIT_MS ||
!Number.isSafeInteger(codeApiMaxRetryWaitMs) ||
codeApiMaxRetryWaitMs < 0 ||
codeApiMaxRetryWaitMs > 300_000 ||
(maxRequestTimeoutMs !== undefined &&
(!Number.isSafeInteger(maxRequestTimeoutMs) ||
maxRequestTimeoutMs < 1 ||
@ -836,9 +887,11 @@ export async function executeWorkspaceTool({
maxRequestTimeoutMs == null ? Infinity : Date.now() + maxRequestTimeoutMs,
);
const queueDeadlineAt = Date.now() + maxQueueWaitMs;
const retryDeadlineAt = Math.min(queueDeadlineAt, callerDeadlineAt - completionReserveMs);
const callerRetryDeadlineAt = callerDeadlineAt - completionReserveMs;
const body = JSON.stringify(request);
let lastAdmissionRejection: WorkspaceToolHttpError | undefined;
let lastRetryDeadlineAt = Infinity;
let rateLimitWaitedMs = 0;
while (true) {
try {
signal?.throwIfAborted();
@ -871,7 +924,7 @@ export async function executeWorkspaceTool({
if (timeoutSignal.aborted) {
throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time');
}
if (lastAdmissionRejection && Date.now() >= retryDeadlineAt) {
if (lastAdmissionRejection && Date.now() >= lastRetryDeadlineAt) {
throw lastAdmissionRejection;
}
const remainingMs = Math.min(
@ -903,20 +956,35 @@ export async function executeWorkspaceTool({
const { body, truncated } = await readErrorBody(response, requestSignal);
signal?.throwIfAborted();
const rejection = new WorkspaceToolHttpError('rejected', response.status, body, truncated);
const admission = getWorkspaceAdmissionRejection(response.status, body, truncated);
if (admission == null) throw rejection;
const retryDeadlineAt = Math.min(
admission === 'queue_timeout' ? queueDeadlineAt : Infinity,
callerRetryDeadlineAt,
);
if (Date.now() >= retryDeadlineAt) throw rejection;
const retryAfterMs = workspaceAdmissionRetryDelay(
response.headers.get('Retry-After'),
admission === 'rate_limited' ? body : undefined,
);
if (
!isWorkspaceAdmissionTimeout(response.status, body, truncated) ||
Date.now() >= queueDeadlineAt
admission === 'rate_limited' &&
retryAfterMs > codeApiMaxRetryWaitMs - rateLimitWaitedMs
) {
throw rejection;
}
const delayMs = Math.min(
workspaceAdmissionRetryDelay(response.headers.get('Retry-After')),
retryDeadlineAt - Date.now(),
);
const delayMs = Math.min(retryAfterMs, retryDeadlineAt - Date.now());
lastAdmissionRejection = rejection;
lastRetryDeadlineAt = retryDeadlineAt;
if (delayMs <= 0) throw rejection;
const waitStartedAt = Date.now();
await waitForWorkspaceAdmission(delayMs, signal);
/** A clamped delay can land exactly on either deadline. Never open
if (admission === 'rate_limited') {
/** A timer may run late. Charge the actual wait, but honor the retry that
* was authorized before waiting; another 429 cannot exceed the balance. */
rateLimitWaitedMs += Math.max(retryAfterMs, Date.now() - waitStartedAt);
}
/** A clamped delay can land exactly on this admission's retry deadline. Never open
* another admission window without the full execution reserve. */
if (Date.now() >= retryDeadlineAt) {
throw rejection;