mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-20 23:13:35 +00:00
🔁 fix: Retry Code API Rate Limits That Reject Before a Workspace Call Starts (#16453)
Some checks are pending
Backend Unit Tests / Build packages (push) Waiting to run
Backend Unit Tests / Codegraph select (push) Waiting to run
Backend Unit Tests / TypeScript type checks (push) Blocked by required conditions
Backend Unit Tests / Circular dependency checks (push) Waiting to run
Backend Unit Tests / Tests: api (shard 1/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 2/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 3/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: data-provider (push) Blocked by required conditions
Backend Unit Tests / Tests: data-schemas (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 1/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 2/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 3/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 4/4) (push) Blocked by required conditions
Codegraph E2E Votes / vote (full suite) (push) Waiting to run
Docker Dev Branch Images Build / publish (push) Waiting to run
Frontend Unit Tests / Client build recovery regression (push) Waiting to run
Frontend Unit Tests / Codegraph select (push) Waiting to run
Frontend Unit Tests / Build packages (push) Waiting to run
Frontend Unit Tests / TypeScript type checks (client) (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/client (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 1/2) (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 2/2) (push) Blocked by required conditions
Frontend Unit Tests / Vite build verification (push) Blocked by required conditions
Sync Locize Translations & Create Translation PR / Sync Translation Keys with Locize (push) Waiting to run
Sync Locize Translations & Create Translation PR / Create Translation PR on Version Published (push) Blocked by required conditions
Some checks are pending
Backend Unit Tests / Build packages (push) Waiting to run
Backend Unit Tests / Codegraph select (push) Waiting to run
Backend Unit Tests / TypeScript type checks (push) Blocked by required conditions
Backend Unit Tests / Circular dependency checks (push) Waiting to run
Backend Unit Tests / Tests: api (shard 1/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 2/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 3/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: data-provider (push) Blocked by required conditions
Backend Unit Tests / Tests: data-schemas (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 1/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 2/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 3/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 4/4) (push) Blocked by required conditions
Codegraph E2E Votes / vote (full suite) (push) Waiting to run
Docker Dev Branch Images Build / publish (push) Waiting to run
Frontend Unit Tests / Client build recovery regression (push) Waiting to run
Frontend Unit Tests / Codegraph select (push) Waiting to run
Frontend Unit Tests / Build packages (push) Waiting to run
Frontend Unit Tests / TypeScript type checks (client) (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/client (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 1/2) (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 2/2) (push) Blocked by required conditions
Frontend Unit Tests / Vite build verification (push) Blocked by required conditions
Sync Locize Translations & Create Translation PR / Sync Translation Keys with Locize (push) Waiting to run
Sync Locize Translations & Create Translation PR / Create Translation PR on Version Published (push) Blocked by required conditions
* 🔁 fix: Retry Code API Rate Limits That Reject Before a Workspace Call Starts * 🔁 fix: Bound Workspace Rate Limit Retries by Code API Budget * 🔗 fix: Isolate Code API Retry Default from Utility Graph * 🧩 fix: Type the Shared Code API Retry Default * 🔁 fix: Keep Workspace Admission Retry Budgets Independent --------- Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
parent
40bb16edfd
commit
f624ad0876
14 changed files with 779 additions and 44 deletions
|
|
@ -164,6 +164,24 @@ function commandResponse(overrides: Record<string, unknown> = {}): Response {
|
|||
}
|
||||
|
||||
describe('createAttachedWorkspaceBashTool', () => {
|
||||
test('honors a zero Code API rate-limit retry budget', async () => {
|
||||
const fetchImpl: CodeBridgeFetch = jest.fn(
|
||||
async () => new Response(JSON.stringify({ error: 'rate_limited' }), { status: 429 }),
|
||||
);
|
||||
const bashTool = createAttachedWorkspaceBashTool({
|
||||
baseUrl: 'https://code.example.com/v1',
|
||||
authHeaders: () => ({}),
|
||||
workspaceId: 'project-a',
|
||||
codeApiMaxRetryWaitMs: 0,
|
||||
fetchImpl,
|
||||
});
|
||||
|
||||
await expect(bashTool.invoke({ command: 'pwd' })).rejects.toThrow(
|
||||
'The operation was not started',
|
||||
);
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('dispatches commands to the resolved conversation workspace instance', async () => {
|
||||
const fetchImpl: CodeBridgeFetch = jest.fn(async () => commandResponse());
|
||||
const workspaceInstanceId = 'e'.repeat(64);
|
||||
|
|
|
|||
|
|
@ -302,6 +302,7 @@ export function createAttachedWorkspaceBashTool({
|
|||
gitIdentity,
|
||||
maxTimeoutMs = WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS,
|
||||
maxQueueWaitMs,
|
||||
codeApiMaxRetryWaitMs,
|
||||
maxRequestTimeoutMs,
|
||||
fetchImpl,
|
||||
}: {
|
||||
|
|
@ -315,6 +316,7 @@ export function createAttachedWorkspaceBashTool({
|
|||
maxTimeoutMs?: number;
|
||||
/** Retry horizon across typed queue expirations, not an admission budget. */
|
||||
maxQueueWaitMs?: number;
|
||||
codeApiMaxRetryWaitMs?: number;
|
||||
/** Verified total HTTP budget; omission keeps the legacy per-attempt timeout. */
|
||||
maxRequestTimeoutMs?: number;
|
||||
fetchImpl?: CodeBridgeFetch;
|
||||
|
|
@ -395,6 +397,7 @@ export function createAttachedWorkspaceBashTool({
|
|||
signal,
|
||||
fetchImpl,
|
||||
...(maxQueueWaitMs == null ? {} : { maxQueueWaitMs }),
|
||||
...(codeApiMaxRetryWaitMs == null ? {} : { codeApiMaxRetryWaitMs }),
|
||||
...(maxRequestTimeoutMs == null ? {} : { maxRequestTimeoutMs }),
|
||||
});
|
||||
if (result.operation !== 'execute_command') {
|
||||
|
|
|
|||
|
|
@ -37,6 +37,72 @@ const response = () =>
|
|||
);
|
||||
|
||||
describe('repository instruction loading', () => {
|
||||
it('does not retry a typed 429 when the configured Code API wait budget is zero', async () => {
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 0 }), {
|
||||
status: 429,
|
||||
headers: { 'Retry-After': '0' },
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(response());
|
||||
|
||||
const result = await createRepositoryInstructionLoader()({
|
||||
enabled: true,
|
||||
context,
|
||||
principalId: 'alice',
|
||||
authHeaders: async () => ({}),
|
||||
assertContent: jest.fn(),
|
||||
codeApiMaxRetryWaitMs: 0,
|
||||
fetchImpl,
|
||||
});
|
||||
|
||||
expect(result).toBeUndefined();
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('refreshes instruction-read authorization for a typed 429 retry', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ error: 'rate_limited' }), {
|
||||
status: 429,
|
||||
headers: { 'Retry-After': '0' },
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(response());
|
||||
let minted = 0;
|
||||
const authHeaders = jest.fn(async () => ({ Authorization: `Bearer token-${++minted}` }));
|
||||
const load = createRepositoryInstructionLoader();
|
||||
const input = {
|
||||
enabled: true,
|
||||
context,
|
||||
principalId: 'alice',
|
||||
authHeaders,
|
||||
fetchImpl,
|
||||
codeApiMaxRetryWaitMs: 100,
|
||||
timeoutMs: 500,
|
||||
assertContent: jest.fn(),
|
||||
};
|
||||
const result = load(input);
|
||||
await jest.advanceTimersByTimeAsync(100);
|
||||
await expect(result).resolves.toContain('Use the project test command.');
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
||||
expect(fetchImpl.mock.calls.map((call) => call[1]?.headers.Authorization)).toEqual([
|
||||
'Bearer token-1',
|
||||
'Bearer token-2',
|
||||
]);
|
||||
await expect(load(input)).resolves.toContain('Use the project test command.');
|
||||
expect(authHeaders).toHaveBeenCalledTimes(3);
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('bounds optional authorization waits and preserves explicit cancellation', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ export interface RepositoryInstructionSource {
|
|||
context: CodeExecutionContext;
|
||||
principalId: string;
|
||||
authHeaders: () => Promise<Record<string, string>>;
|
||||
codeApiMaxRetryWaitMs?: number;
|
||||
load: ReturnType<typeof createRepositoryInstructionLoader>;
|
||||
}
|
||||
|
||||
|
|
@ -54,6 +55,7 @@ export function createRepositoryInstructionLoader() {
|
|||
signal,
|
||||
fetchImpl,
|
||||
assertContent,
|
||||
codeApiMaxRetryWaitMs,
|
||||
timeoutMs = 2000,
|
||||
}: {
|
||||
enabled: boolean;
|
||||
|
|
@ -64,6 +66,7 @@ export function createRepositoryInstructionLoader() {
|
|||
signal?: AbortSignal;
|
||||
fetchImpl?: CodeBridgeFetch;
|
||||
assertContent: (content: string) => void;
|
||||
codeApiMaxRetryWaitMs?: number;
|
||||
timeoutMs?: number;
|
||||
}): Promise<string | undefined> => {
|
||||
const workspace = context.codeWorkspace;
|
||||
|
|
@ -93,13 +96,13 @@ export function createRepositoryInstructionLoader() {
|
|||
const timer = setTimeout(() => deadline.abort(), budget);
|
||||
const readSignal = signal ? AbortSignal.any([signal, deadline.signal]) : deadline.signal;
|
||||
try {
|
||||
const headers = await abortable(authHeaders(), readSignal);
|
||||
if (content === undefined) {
|
||||
const result = await executeWorkspaceTool({
|
||||
baseURL: context.baseUrl,
|
||||
authHeaders: headers,
|
||||
authHeaders: () => abortable(authHeaders(), readSignal),
|
||||
signal: readSignal,
|
||||
fetchImpl,
|
||||
codeApiMaxRetryWaitMs,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'read_file',
|
||||
|
|
@ -121,6 +124,8 @@ export function createRepositoryInstructionLoader() {
|
|||
content = result.content;
|
||||
if (cache.size >= 64) cache.delete(cache.keys().next().value!);
|
||||
cache.set(key, content);
|
||||
} else {
|
||||
await abortable(authHeaders(), readSignal);
|
||||
}
|
||||
} catch {
|
||||
if (signal?.aborted) throw signal.reason;
|
||||
|
|
|
|||
1
packages/api/src/code/limits.ts
Normal file
1
packages/api/src/code/limits.ts
Normal file
|
|
@ -0,0 +1 @@
|
|||
export const CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS: number = 20_000;
|
||||
|
|
@ -338,6 +338,514 @@ describe('workspace admission feedback', () => {
|
|||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('cancels a typed rate-limit wait without replaying the request', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const reason = new DOMException('Stopped', 'AbortError');
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ error: 'rate_limited' }), {
|
||||
status: 429,
|
||||
headers: { 'Retry-After': '1' },
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(new Response('{}', { status: 504 }));
|
||||
const request = executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
signal: controller.signal,
|
||||
fetchImpl,
|
||||
maxQueueWaitMs: 0,
|
||||
request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
|
||||
});
|
||||
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
controller.abort(reason);
|
||||
await expect(request).rejects.toBe(reason);
|
||||
await jest.advanceTimersByTimeAsync(1_000);
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test('retries a Code API rate limit, which rejects before the operation starts', async () => {
|
||||
const rateLimited = () =>
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
error: 'rate_limited',
|
||||
message: 'Too many CodeAPI execution requests. Please retry in 1 second.',
|
||||
retry_after_seconds: 1,
|
||||
}),
|
||||
{ status: 429, headers: { 'Retry-After': '0' } },
|
||||
);
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(rateLimited())
|
||||
.mockResolvedValueOnce(rateLimited())
|
||||
.mockResolvedValueOnce(
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
protocolVersion: 1,
|
||||
operation: 'edit_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'src/app.ts',
|
||||
replacements: 1,
|
||||
bytesWritten: 24,
|
||||
}),
|
||||
{ status: 200, headers: { 'Content-Type': 'application/json' } },
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'edit_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'src/app.ts',
|
||||
edits: [{ oldText: 'const old = true;', newText: 'const ready = true;' }],
|
||||
},
|
||||
}),
|
||||
).resolves.toMatchObject({ operation: 'edit_file', replacements: 1 });
|
||||
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(3);
|
||||
const bodies = fetchImpl.mock.calls.map((call) => call[1]?.body);
|
||||
expect(new Set(bodies).size).toBe(1);
|
||||
});
|
||||
|
||||
test.each([
|
||||
['missing', undefined],
|
||||
['invalid', 'invalid'],
|
||||
['overflowing', '9'.repeat(400)],
|
||||
] as const)('uses the typed 429 body delay when Retry-After is %s', async (_case, retryAfter) => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 2 }), {
|
||||
status: 429,
|
||||
...(retryAfter ? { headers: { 'Retry-After': retryAfter } } : {}),
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(new Response('{}', { status: 504 }));
|
||||
const result = executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
codeApiMaxRetryWaitMs: 3_000,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'read_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'x',
|
||||
},
|
||||
}).catch((error: WorkspaceToolHttpError) => error);
|
||||
|
||||
await jest.advanceTimersByTimeAsync(1_999);
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
await expect(result).resolves.toMatchObject({ upstreamStatus: 504 });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test.each([-1, 300_001, 1.5, Number.NaN])(
|
||||
'rejects an invalid Code API retry limit of %p before transport',
|
||||
async (codeApiMaxRetryWaitMs) => {
|
||||
const fetchImpl = jest.fn();
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
codeApiMaxRetryWaitMs,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'read_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'x',
|
||||
},
|
||||
}),
|
||||
).rejects.toMatchObject({ reason: 'invalid' });
|
||||
expect(fetchImpl).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
test.each([0, 150])('bounds typed 429 retries to the %i ms Code API budget', async (budget) => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const fetchImpl = jest.fn().mockImplementation(
|
||||
() =>
|
||||
new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 0 }), {
|
||||
status: 429,
|
||||
headers: { 'Retry-After': '0' },
|
||||
}),
|
||||
);
|
||||
const result = executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
codeApiMaxRetryWaitMs: budget,
|
||||
request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
|
||||
}).catch((error: WorkspaceToolHttpError) => error);
|
||||
|
||||
await jest.advanceTimersByTimeAsync(budget);
|
||||
await expect(result).resolves.toMatchObject({
|
||||
upstreamStatus: 429,
|
||||
message: expect.stringContaining('The operation was not started'),
|
||||
});
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(budget === 0 ? 1 : 2);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test('does not wait when a 429 retry hint exceeds the Code API budget', async () => {
|
||||
const fetchImpl = jest.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 30 }), {
|
||||
status: 429,
|
||||
}),
|
||||
);
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
|
||||
}),
|
||||
).rejects.toMatchObject({ upstreamStatus: 429 });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('keeps queue retries enabled when Code API rate-limit retries are disabled', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), {
|
||||
status: 503,
|
||||
headers: { 'Retry-After': '0' },
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ error: 'rate_limited' }), { status: 429 }),
|
||||
);
|
||||
const result = executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
codeApiMaxRetryWaitMs: 0,
|
||||
request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
|
||||
}).catch((error: WorkspaceToolHttpError) => error);
|
||||
|
||||
await jest.advanceTimersByTimeAsync(100);
|
||||
await expect(result).resolves.toMatchObject({ upstreamStatus: 429 });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test('does not charge queue waits to the Code API rate-limit budget', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ error: 'rate_limited' }), {
|
||||
status: 429,
|
||||
headers: { 'Retry-After': '0' },
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), {
|
||||
status: 503,
|
||||
headers: { 'Retry-After': '0' },
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(new Response('{}', { status: 504 }));
|
||||
const result = executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
codeApiMaxRetryWaitMs: 100,
|
||||
request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
|
||||
}).catch((error: WorkspaceToolHttpError) => error);
|
||||
|
||||
await jest.advanceTimersByTimeAsync(200);
|
||||
await expect(result).resolves.toMatchObject({ upstreamStatus: 504 });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(3);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test.each([0, 50])(
|
||||
'retries typed 429s after a %i ms queue horizon without re-enabling capacity retries',
|
||||
async (maxQueueWaitMs) => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ error: 'rate_limited' }), {
|
||||
status: 429,
|
||||
headers: { 'Retry-After': '0' },
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(new Response('{}', { status: 504 }));
|
||||
const result = executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
maxQueueWaitMs,
|
||||
codeApiMaxRetryWaitMs: 100,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'read_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'x',
|
||||
},
|
||||
}).catch((error: WorkspaceToolHttpError) => error);
|
||||
|
||||
await jest.advanceTimersByTimeAsync(100);
|
||||
await expect(result).resolves.toMatchObject({ upstreamStatus: 504 });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
test('continues 429 recovery after a queue retry but never extends its capacity horizon', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), {
|
||||
status: 503,
|
||||
headers: { 'Retry-After': '0' },
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ error: 'rate_limited' }), {
|
||||
status: 429,
|
||||
headers: { 'Retry-After': '0' },
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(new Response('{}', { status: 504 }));
|
||||
const result = executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
maxQueueWaitMs: 150,
|
||||
codeApiMaxRetryWaitMs: 100,
|
||||
request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
|
||||
}).catch((error: WorkspaceToolHttpError) => error);
|
||||
|
||||
await jest.advanceTimersByTimeAsync(200);
|
||||
await expect(result).resolves.toMatchObject({ upstreamStatus: 504 });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(3);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test('never borrows the rate-limit horizon to retry a later capacity timeout', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ error: 'rate_limited' }), {
|
||||
status: 429,
|
||||
headers: { 'Retry-After': '0' },
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), {
|
||||
status: 503,
|
||||
headers: { 'Retry-After': '0' },
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(new Response('{}', { status: 504 }));
|
||||
const result = executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
maxQueueWaitMs: 50,
|
||||
codeApiMaxRetryWaitMs: 200,
|
||||
request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
|
||||
}).catch((error: WorkspaceToolHttpError) => error);
|
||||
|
||||
await jest.advanceTimersByTimeAsync(100);
|
||||
await expect(result).resolves.toMatchObject({ upstreamStatus: 503 });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test('retries after an authorized rate-limit wait despite timer jitter', async () => {
|
||||
jest.useFakeTimers();
|
||||
const startedAt = Date.now();
|
||||
let currentTime = startedAt;
|
||||
jest.spyOn(Date, 'now').mockImplementation(() => currentTime);
|
||||
try {
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ error: 'rate_limited' }), {
|
||||
status: 429,
|
||||
headers: { 'Retry-After': '0' },
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(new Response('{}', { status: 504 }));
|
||||
const result = executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
maxQueueWaitMs: 0,
|
||||
codeApiMaxRetryWaitMs: 100,
|
||||
request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
|
||||
}).catch((error: WorkspaceToolHttpError) => error);
|
||||
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
currentTime += 101;
|
||||
await jest.advanceTimersByTimeAsync(100);
|
||||
await expect(result).resolves.toMatchObject({ upstreamStatus: 504 });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test('charges an overslept 429 wait before considering another retry', async () => {
|
||||
jest.useFakeTimers();
|
||||
const startedAt = Date.now();
|
||||
let currentTime = startedAt;
|
||||
jest.spyOn(Date, 'now').mockImplementation(() => currentTime);
|
||||
try {
|
||||
const rateLimited = () =>
|
||||
new Response(JSON.stringify({ error: 'rate_limited' }), {
|
||||
status: 429,
|
||||
headers: { 'Retry-After': '0' },
|
||||
});
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(rateLimited())
|
||||
.mockResolvedValueOnce(rateLimited())
|
||||
.mockResolvedValueOnce(new Response('{}', { status: 504 }));
|
||||
const result = executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
maxQueueWaitMs: 0,
|
||||
codeApiMaxRetryWaitMs: 200,
|
||||
request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
|
||||
}).catch((error: WorkspaceToolHttpError) => error);
|
||||
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
currentTime += 150;
|
||||
await jest.advanceTimersByTimeAsync(100);
|
||||
await expect(result).resolves.toMatchObject({ upstreamStatus: 429 });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test('reserves the caller HTTP execution budget when capacity retries are disabled', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const fetchImpl = jest.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify({ error: 'rate_limited' }), {
|
||||
status: 429,
|
||||
headers: { 'Retry-After': '0' },
|
||||
}),
|
||||
);
|
||||
const result = executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
maxQueueWaitMs: 0,
|
||||
codeApiMaxRetryWaitMs: 1_000,
|
||||
maxRequestTimeoutMs: 35_050,
|
||||
request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
|
||||
}).catch((error: WorkspaceToolHttpError) => error);
|
||||
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
await expect(result).resolves.toMatchObject({ upstreamStatus: 429 });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test('reports a rate limit as not started when its retry budget is disabled', async () => {
|
||||
const fetchImpl = jest.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 1 }), {
|
||||
status: 429,
|
||||
headers: { 'Retry-After': '1' },
|
||||
}),
|
||||
);
|
||||
|
||||
const failure = executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
codeApiMaxRetryWaitMs: 0,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'execute_command',
|
||||
workspaceId: 'primary',
|
||||
command: 'echo test',
|
||||
},
|
||||
});
|
||||
|
||||
await expect(failure).rejects.toThrow('The operation was not started');
|
||||
await expect(failure).rejects.toMatchObject({ upstreamStatus: 429 });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('never retries a 429 that does not carry the typed rate-limit body', async () => {
|
||||
const fetchImpl = jest.fn().mockResolvedValue(
|
||||
new Response('<html>Too Many Requests</html>', {
|
||||
status: 429,
|
||||
headers: { 'Retry-After': '0' },
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example/v1',
|
||||
authHeaders: {},
|
||||
fetchImpl,
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
operation: 'edit_file',
|
||||
workspaceId: 'primary',
|
||||
path: 'src/app.ts',
|
||||
edits: [{ oldText: 'const old = true;', newText: 'const ready = true;' }],
|
||||
},
|
||||
}),
|
||||
).rejects.toMatchObject({ upstreamStatus: 429 });
|
||||
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('never retries an ambiguous capacity-looking failure', async () => {
|
||||
const fetchImpl = jest.fn().mockResolvedValue(
|
||||
new Response('<html>Gateway unavailable</html>', {
|
||||
|
|
@ -369,6 +877,10 @@ describe('workspace admission feedback', () => {
|
|||
[503, '<html>Gateway unavailable</html>', false],
|
||||
[503, '{"code":"WORKSPACE_QUEUE_TIMEOUT"}', true],
|
||||
[503, 'null', false],
|
||||
[503, '{"error":"rate_limited"}', false],
|
||||
[429, '{"code":"WORKSPACE_QUEUE_TIMEOUT"}', false],
|
||||
[429, '{"error":"rate_limited"}', true],
|
||||
[429, '<html>Too Many Requests</html>', false],
|
||||
] as const)(
|
||||
'does not infer non-execution from an ambiguous response',
|
||||
(status, body, truncated) => {
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import {
|
|||
CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS,
|
||||
} from 'librechat-data-provider';
|
||||
import type { CodeBridgeFetch } from './bridge';
|
||||
import { CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS } from './limits';
|
||||
|
||||
const WORKSPACE_TOOL_TIMEOUT_MS = 30_000;
|
||||
const MAX_PATH_LENGTH = 4096;
|
||||
|
|
@ -274,13 +275,18 @@ export class WorkspaceToolHttpError extends Error {
|
|||
public readonly upstreamBodyTruncated = false,
|
||||
) {
|
||||
let message = `Workspace tool request ${reason}`;
|
||||
if (
|
||||
reason === 'rejected' &&
|
||||
isWorkspaceAdmissionTimeout(upstreamStatus, upstreamBody, upstreamBodyTruncated)
|
||||
) {
|
||||
const admissionRejection =
|
||||
reason === 'rejected'
|
||||
? getWorkspaceAdmissionRejection(upstreamStatus, upstreamBody, upstreamBodyTruncated)
|
||||
: undefined;
|
||||
if (admissionRejection === 'queue_timeout') {
|
||||
message =
|
||||
'Workspace capacity was unavailable before the queue deadline. The operation was not started. Wait for active work to finish or select an independent workspace on a machine with available capacity.';
|
||||
}
|
||||
if (admissionRejection === 'rate_limited') {
|
||||
message =
|
||||
'The Code API request rate limit was reached. The operation was not started. Wait a few seconds before retrying, or make fewer concurrent workspace calls.';
|
||||
}
|
||||
if (reason === 'insufficient_time') {
|
||||
message =
|
||||
'Workspace execution cannot fit within the remaining HTTP budget. The operation was not started.';
|
||||
|
|
@ -295,15 +301,38 @@ export class WorkspaceToolHttpError extends Error {
|
|||
}
|
||||
}
|
||||
|
||||
function isWorkspaceAdmissionTimeout(status?: number, body?: string, truncated = false): boolean {
|
||||
if (status !== 503 || !body || truncated || body.length > MAX_ERROR_BODY_BYTES) {
|
||||
return false;
|
||||
type WorkspaceAdmissionRejection = 'queue_timeout' | 'rate_limited';
|
||||
|
||||
/**
|
||||
* Rejections the Code API issues before an operation is assigned, so retrying cannot repeat a
|
||||
* mutation: a queue deadline that expired before admission, or its rate limiter, which runs as
|
||||
* middleware ahead of the workspace router. Anything else, including an unparsable body under
|
||||
* either status, keeps an unknown outcome and is never retried.
|
||||
*/
|
||||
function getWorkspaceAdmissionRejection(
|
||||
status?: number,
|
||||
body?: string,
|
||||
truncated = false,
|
||||
): WorkspaceAdmissionRejection | undefined {
|
||||
if (
|
||||
(status !== 503 && status !== 429) ||
|
||||
!body ||
|
||||
truncated ||
|
||||
body.length > MAX_ERROR_BODY_BYTES
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
const parsed: { code?: string } | null = JSON.parse(body);
|
||||
return parsed?.code === 'WORKSPACE_QUEUE_TIMEOUT';
|
||||
const parsed: { code?: string; error?: string } | null = JSON.parse(body);
|
||||
if (status === 503 && parsed?.code === 'WORKSPACE_QUEUE_TIMEOUT') {
|
||||
return 'queue_timeout';
|
||||
}
|
||||
if (status === 429 && parsed?.error === 'rate_limited') {
|
||||
return 'rate_limited';
|
||||
}
|
||||
return undefined;
|
||||
} catch {
|
||||
return false;
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -326,9 +355,25 @@ function waitForWorkspaceAdmission(delayMs: number, signal?: AbortSignal): Promi
|
|||
});
|
||||
}
|
||||
|
||||
function workspaceAdmissionRetryDelay(value: string | null): number {
|
||||
if (value == null || !/^\d+$/.test(value)) return WORKSPACE_QUEUE_RETRY_DELAY_MS;
|
||||
return Math.max(100, Math.min(Number(value) * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS));
|
||||
function workspaceAdmissionRetryDelay(value: string | null, rateLimitBody?: string): number {
|
||||
if (value != null && /^\d+$/.test(value)) {
|
||||
const seconds = Number(value);
|
||||
if (Number.isFinite(seconds)) {
|
||||
return Math.max(100, Math.min(seconds * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS));
|
||||
}
|
||||
}
|
||||
if (rateLimitBody) {
|
||||
try {
|
||||
const parsed: { retry_after_seconds?: number } | null = JSON.parse(rateLimitBody);
|
||||
const seconds = parsed?.retry_after_seconds;
|
||||
if (typeof seconds === 'number' && Number.isFinite(seconds) && seconds >= 0) {
|
||||
return Math.max(100, Math.min(seconds * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS));
|
||||
}
|
||||
} catch {
|
||||
// Fall back when the delay hint is unusable.
|
||||
}
|
||||
}
|
||||
return WORKSPACE_QUEUE_RETRY_DELAY_MS;
|
||||
}
|
||||
|
||||
/** Keep a received HTTP status even if reading its diagnostic body fails or stalls. */
|
||||
|
|
@ -801,6 +846,7 @@ export async function executeWorkspaceTool({
|
|||
signal,
|
||||
fetchImpl = fetch,
|
||||
maxQueueWaitMs = WORKSPACE_QUEUE_MAX_WAIT_MS,
|
||||
codeApiMaxRetryWaitMs = CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS,
|
||||
maxRequestTimeoutMs,
|
||||
deadlineAtMs,
|
||||
}: {
|
||||
|
|
@ -810,6 +856,8 @@ export async function executeWorkspaceTool({
|
|||
signal?: AbortSignal;
|
||||
fetchImpl?: CodeBridgeFetch;
|
||||
maxQueueWaitMs?: number;
|
||||
/** Maximum time waiting for Code API rate-limit admission, independent of queue retries. */
|
||||
codeApiMaxRetryWaitMs?: number;
|
||||
/** End-to-end HTTP budget for this call. Omission keeps legacy per-attempt timeouts. */
|
||||
maxRequestTimeoutMs?: number;
|
||||
/** Optional earlier caller deadline; a signal alone has no remaining-time value. */
|
||||
|
|
@ -820,6 +868,9 @@ export async function executeWorkspaceTool({
|
|||
!Number.isSafeInteger(maxQueueWaitMs) ||
|
||||
maxQueueWaitMs < 0 ||
|
||||
maxQueueWaitMs > WORKSPACE_QUEUE_MAX_WAIT_MS ||
|
||||
!Number.isSafeInteger(codeApiMaxRetryWaitMs) ||
|
||||
codeApiMaxRetryWaitMs < 0 ||
|
||||
codeApiMaxRetryWaitMs > 300_000 ||
|
||||
(maxRequestTimeoutMs !== undefined &&
|
||||
(!Number.isSafeInteger(maxRequestTimeoutMs) ||
|
||||
maxRequestTimeoutMs < 1 ||
|
||||
|
|
@ -836,9 +887,11 @@ export async function executeWorkspaceTool({
|
|||
maxRequestTimeoutMs == null ? Infinity : Date.now() + maxRequestTimeoutMs,
|
||||
);
|
||||
const queueDeadlineAt = Date.now() + maxQueueWaitMs;
|
||||
const retryDeadlineAt = Math.min(queueDeadlineAt, callerDeadlineAt - completionReserveMs);
|
||||
const callerRetryDeadlineAt = callerDeadlineAt - completionReserveMs;
|
||||
const body = JSON.stringify(request);
|
||||
let lastAdmissionRejection: WorkspaceToolHttpError | undefined;
|
||||
let lastRetryDeadlineAt = Infinity;
|
||||
let rateLimitWaitedMs = 0;
|
||||
while (true) {
|
||||
try {
|
||||
signal?.throwIfAborted();
|
||||
|
|
@ -871,7 +924,7 @@ export async function executeWorkspaceTool({
|
|||
if (timeoutSignal.aborted) {
|
||||
throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time');
|
||||
}
|
||||
if (lastAdmissionRejection && Date.now() >= retryDeadlineAt) {
|
||||
if (lastAdmissionRejection && Date.now() >= lastRetryDeadlineAt) {
|
||||
throw lastAdmissionRejection;
|
||||
}
|
||||
const remainingMs = Math.min(
|
||||
|
|
@ -903,20 +956,35 @@ export async function executeWorkspaceTool({
|
|||
const { body, truncated } = await readErrorBody(response, requestSignal);
|
||||
signal?.throwIfAborted();
|
||||
const rejection = new WorkspaceToolHttpError('rejected', response.status, body, truncated);
|
||||
const admission = getWorkspaceAdmissionRejection(response.status, body, truncated);
|
||||
if (admission == null) throw rejection;
|
||||
const retryDeadlineAt = Math.min(
|
||||
admission === 'queue_timeout' ? queueDeadlineAt : Infinity,
|
||||
callerRetryDeadlineAt,
|
||||
);
|
||||
if (Date.now() >= retryDeadlineAt) throw rejection;
|
||||
const retryAfterMs = workspaceAdmissionRetryDelay(
|
||||
response.headers.get('Retry-After'),
|
||||
admission === 'rate_limited' ? body : undefined,
|
||||
);
|
||||
if (
|
||||
!isWorkspaceAdmissionTimeout(response.status, body, truncated) ||
|
||||
Date.now() >= queueDeadlineAt
|
||||
admission === 'rate_limited' &&
|
||||
retryAfterMs > codeApiMaxRetryWaitMs - rateLimitWaitedMs
|
||||
) {
|
||||
throw rejection;
|
||||
}
|
||||
const delayMs = Math.min(
|
||||
workspaceAdmissionRetryDelay(response.headers.get('Retry-After')),
|
||||
retryDeadlineAt - Date.now(),
|
||||
);
|
||||
const delayMs = Math.min(retryAfterMs, retryDeadlineAt - Date.now());
|
||||
lastAdmissionRejection = rejection;
|
||||
lastRetryDeadlineAt = retryDeadlineAt;
|
||||
if (delayMs <= 0) throw rejection;
|
||||
const waitStartedAt = Date.now();
|
||||
await waitForWorkspaceAdmission(delayMs, signal);
|
||||
/** A clamped delay can land exactly on either deadline. Never open
|
||||
if (admission === 'rate_limited') {
|
||||
/** A timer may run late. Charge the actual wait, but honor the retry that
|
||||
* was authorized before waiting; another 429 cannot exceed the balance. */
|
||||
rateLimitWaitedMs += Math.max(retryAfterMs, Date.now() - waitStartedAt);
|
||||
}
|
||||
/** A clamped delay can land exactly on this admission's retry deadline. Never open
|
||||
* another admission window without the full execution reserve. */
|
||||
if (Date.now() >= retryDeadlineAt) {
|
||||
throw rejection;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue