LibreChat/packages/api/src/utils/code.ts
Danny Avila f624ad0876
Some checks are pending
Backend Unit Tests / Build packages (push) Waiting to run
Backend Unit Tests / Codegraph select (push) Waiting to run
Backend Unit Tests / TypeScript type checks (push) Blocked by required conditions
Backend Unit Tests / Circular dependency checks (push) Waiting to run
Backend Unit Tests / Tests: api (shard 1/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 2/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 3/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: data-provider (push) Blocked by required conditions
Backend Unit Tests / Tests: data-schemas (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 1/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 2/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 3/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 4/4) (push) Blocked by required conditions
Codegraph E2E Votes / vote (full suite) (push) Waiting to run
Docker Dev Branch Images Build / publish (push) Waiting to run
Frontend Unit Tests / Client build recovery regression (push) Waiting to run
Frontend Unit Tests / Codegraph select (push) Waiting to run
Frontend Unit Tests / Build packages (push) Waiting to run
Frontend Unit Tests / TypeScript type checks (client) (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/client (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 1/2) (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 2/2) (push) Blocked by required conditions
Frontend Unit Tests / Vite build verification (push) Blocked by required conditions
Sync Locize Translations & Create Translation PR / Sync Translation Keys with Locize (push) Waiting to run
Sync Locize Translations & Create Translation PR / Create Translation PR on Version Published (push) Blocked by required conditions
🔁 fix: Retry Code API Rate Limits That Reject Before a Workspace Call Starts (#16453)
* 🔁 fix: Retry Code API Rate Limits That Reject Before a Workspace Call Starts

* 🔁 fix: Bound Workspace Rate Limit Retries by Code API Budget

* 🔗 fix: Isolate Code API Retry Default from Utility Graph

* 🧩 fix: Type the Shared Code API Retry Default

* 🔁 fix: Keep Workspace Admission Retry Budgets Independent

---------

Co-authored-by: Lia <lia@librechat.ai>
2026-09-28 15:45:13 -04:00

410 lines
13 KiB
TypeScript

import http from 'http';
import https from 'https';
import { isAxiosError } from 'axios';
import { setTimeout as delay } from 'node:timers/promises';
import type { ServerRequest } from '~/types';
import { CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS } from '~/code/limits';
/**
* Dedicated agents for code-server requests, preventing socket pool contamination.
* follow-redirects (used by axios) leaks `socket.destroy` as a timeout listener;
* on Node 19+ (keepAlive: true by default), tainted sockets re-enter the global pool
* and kill unrelated requests (e.g., node-fetch in CodeExecutor) after the idle timeout.
*/
export const codeServerHttpAgent: http.Agent = new http.Agent({ keepAlive: false });
export const codeServerHttpsAgent: https.Agent = new https.Agent({ keepAlive: false });
/**
* Wait implied by a Code API 429, in milliseconds, or `null` when the error
* is not a rate-limit response. The service answers with both the standard
* `Retry-After` header and a `retry_after_seconds` body field; the header is
* preferred so a body the caller asked for as a stream/buffer never has to be
* parsed, and the body is the fallback for proxies that drop the header.
*/
export function getCodeApiRetryAfterMs(error: unknown): number | null {
if (!isAxiosError(error) || error.response?.status !== 429) {
return null;
}
const header = error.response.headers?.['retry-after'];
const raw = Array.isArray(header) ? header[0] : header;
/* `Number('')` is 0, which would read as "retry immediately" and skip the
* body fallback; treat a blank header as absent. */
const headerSeconds = typeof raw === 'string' && raw.trim() === '' ? NaN : Number(raw);
if (Number.isFinite(headerSeconds) && headerSeconds >= 0) {
return headerSeconds * 1000;
}
const body = error.response.data;
if (typeof body === 'object' && body !== null && 'retry_after_seconds' in body) {
const bodySeconds = Number(body.retry_after_seconds);
if (Number.isFinite(bodySeconds) && bodySeconds >= 0) {
return bodySeconds * 1000;
}
}
return null;
}
/** Total time one operation may spend waiting out Code API rate limits.
* Keeps recovery bounded inside a live chat turn. */
export const MAX_CODE_API_RATE_LIMIT_WAIT_MS: number = CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS;
export const CODE_API_UPLOAD_CONCURRENCY_DEFAULT = 3;
export interface CodeApiUploadRegistry {
scopes: Map<string, UploadScopeState>;
}
export function createCodeApiUploadRegistry(): CodeApiUploadRegistry {
return { scopes: new Map() };
}
export function createCodeApiUploadScope(params: {
route: string;
principalId: string;
tenantId?: string;
}): string {
return JSON.stringify([params.route, params.tenantId ?? 'legacy', params.principalId]);
}
export function getCodeApiUploadOptions(
req: ServerRequest,
route: string,
): { scope: string; concurrency: number; retryWaitMs: number } {
const user = req.user as typeof req.user & {
_id?: { toString(): string } | string;
tenantId?: unknown;
orgId?: unknown;
};
const principalId = user?.id ?? user?._id?.toString();
if (!principalId) {
throw new Error('Code API upload recovery requires an authenticated principal');
}
return {
scope: createCodeApiUploadScope({
route,
principalId,
tenantId:
user.tenantId == null && user.orgId == null
? undefined
: String(user.tenantId ?? user.orgId),
}),
concurrency:
req.config?.endpoints?.agents?.codeApiUploadConcurrency ??
CODE_API_UPLOAD_CONCURRENCY_DEFAULT,
retryWaitMs:
req.config?.endpoints?.agents?.codeApiMaxRetryWaitMs ?? MAX_CODE_API_RATE_LIMIT_WAIT_MS,
};
}
interface QueuedUpload {
run: () => void;
reject: (error: unknown) => void;
signal?: AbortSignal;
onAbort?: () => void;
}
interface UploadScopeState {
active: number;
concurrency: number;
queue: QueuedUpload[];
}
interface CodeApiRateLimitWaitToken {
endAt: number;
}
function abortReason(signal: AbortSignal): unknown {
try {
signal.throwIfAborted();
} catch (error) {
return error;
}
return new Error('The operation was aborted');
}
function drainCodeApiUploadScope(
registry: CodeApiUploadRegistry,
scope: string,
state: UploadScopeState,
): void {
while (state.active < state.concurrency && state.queue.length > 0) {
const queued = state.queue.shift();
if (!queued) {
break;
}
queued.signal?.removeEventListener('abort', queued.onAbort!);
if (queued.signal?.aborted) {
queued.reject(abortReason(queued.signal));
continue;
}
queued.run();
}
if (state.active === 0 && state.queue.length === 0) {
registry.scopes.delete(scope);
}
}
/** Limits uploads within one Code API route and principal. A throttled bucket
* holds only its own slots, and canceled callers leave the queue immediately. */
export function withCodeApiUploadSlot<T>(params: {
registry: CodeApiUploadRegistry;
task: () => Promise<T>;
scope: string;
concurrency?: number;
signal?: AbortSignal;
}): Promise<T> {
const {
registry,
task,
scope,
concurrency = CODE_API_UPLOAD_CONCURRENCY_DEFAULT,
signal,
} = params;
if (!Number.isInteger(concurrency) || concurrency < 1) {
throw new Error(`Code API upload concurrency must be a positive integer (got ${concurrency})`);
}
signal?.throwIfAborted();
let state = registry.scopes.get(scope);
if (!state) {
state = { active: 0, concurrency, queue: [] };
registry.scopes.set(scope, state);
} else {
state.concurrency = concurrency;
}
return new Promise<T>((resolve, reject) => {
const run = (): void => {
state.active++;
Promise.resolve()
.then(task)
.then(resolve, reject)
.finally(() => {
state.active--;
drainCodeApiUploadScope(registry, scope, state);
});
};
if (state.active < state.concurrency) {
run();
return;
}
const queued: QueuedUpload = { run, reject, signal };
queued.onAbort = () => {
const index = state.queue.indexOf(queued);
if (index >= 0) {
state.queue.splice(index, 1);
reject(abortReason(signal!));
drainCodeApiUploadScope(registry, scope, state);
}
};
signal?.addEventListener('abort', queued.onAbort, { once: true });
state.queue.push(queued);
});
}
/** Actual rate-limit wait time shared by every request of one operation. */
export interface CodeApiRateLimitBudget {
limitMs: number;
waitedMs: number;
waitingSince?: number;
activeWaitEnds: Set<CodeApiRateLimitWaitToken>;
}
export function createCodeApiRateLimitBudget(
totalMs: number = MAX_CODE_API_RATE_LIMIT_WAIT_MS,
): CodeApiRateLimitBudget {
return { limitMs: totalMs, waitedMs: 0, activeWaitEnds: new Set() };
}
function beginCodeApiRateLimitWait(
budget: CodeApiRateLimitBudget,
waitMs: number,
): CodeApiRateLimitWaitToken | null {
const now = Date.now();
const activeElapsed = budget.waitingSince == null ? 0 : now - budget.waitingSince;
const currentWaitEnd = Math.max(
now,
...Array.from(budget.activeWaitEnds, (token) => token.endAt),
);
const projectedWaitedMs =
budget.waitedMs + activeElapsed + Math.max(currentWaitEnd, now + waitMs) - now;
if (projectedWaitedMs > budget.limitMs) {
return null;
}
const token = { endAt: now + waitMs };
if (budget.activeWaitEnds.size === 0) {
budget.waitingSince = now;
}
budget.activeWaitEnds.add(token);
return token;
}
function finishCodeApiRateLimitWait(
budget: CodeApiRateLimitBudget,
token: CodeApiRateLimitWaitToken,
): void {
if (!budget.activeWaitEnds.delete(token)) {
return;
}
if (budget.activeWaitEnds.size === 0 && budget.waitingSince != null) {
budget.waitedMs += Date.now() - budget.waitingSince;
budget.waitingSince = undefined;
}
}
function codeApiRateLimitError(label: string, cause: unknown, retryAfterMs?: number): Error {
const retrySuffix = retryAfterMs == null ? '' : ` (retry in ${Math.ceil(retryAfterMs / 1000)}s)`;
return Object.assign(
new Error(`Code API rate limit reached while ${label}${retrySuffix}.`, { cause }),
{
name: 'CodeApiRateLimitError',
code: 'CODE_API_RATE_LIMITED',
status: 429,
statusCode: 429,
...(retryAfterMs == null ? {} : { retryAfterMs }),
},
);
}
/**
* Runs a Code API request, waiting out a rate limit whenever the shared
* budget still covers the server's `Retry-After`. Multi-request operations
* (a windowed image read, a batch upload) would otherwise abandon the work
* already done the moment a limiter window closes mid-flight. A 429 the
* budget cannot absorb throws a named error rather than a bare
* "status code 429", which reads as an unspecified failure instead of one
* that clears on its own.
*/
export async function withCodeApiRateLimit<T>(params: {
attempt: () => Promise<T>;
label: string;
budget?: CodeApiRateLimitBudget;
onWait?: (waitMs: number) => void;
signal?: AbortSignal;
/** Narrows retry classification when an attempt also performs non-CodeAPI work. */
isRateLimit?: (error: unknown) => boolean;
}): Promise<T> {
const {
attempt,
label,
budget,
onWait,
signal,
isRateLimit = (error) => isAxiosError(error) && error.response?.status === 429,
} = params;
for (;;) {
signal?.throwIfAborted();
try {
return await attempt();
} catch (error) {
if (!isRateLimit(error)) {
throw error;
}
const retryAfterMs = getCodeApiRetryAfterMs(error);
if (retryAfterMs == null) {
throw codeApiRateLimitError(label, error);
}
/* A zero delay would spin. Wait at least a second; the actual-wait
* budget still guarantees that repeated zero hints terminate. */
const waitMs = Math.max(retryAfterMs, 1000);
if (budget == null) {
throw codeApiRateLimitError(label, error, waitMs);
}
const rateLimitBudget = budget;
const waitToken = beginCodeApiRateLimitWait(rateLimitBudget, waitMs);
if (waitToken == null) {
throw codeApiRateLimitError(label, error, waitMs);
}
onWait?.(waitMs);
try {
await delay(waitMs, undefined, signal ? { signal } : undefined);
} finally {
finishCodeApiRateLimitWait(rateLimitBudget, waitToken);
}
}
}
}
/** Owns upload admission, retry waiting, and source reopening for every Code API
* upload producer. Callers supply storage and transport adapters only. */
export function withCodeApiUploadRecovery<S, T>(params: {
registry: CodeApiUploadRegistry;
scope: string;
concurrency?: number;
budget?: CodeApiRateLimitBudget;
signal?: AbortSignal;
label: string;
onWait?: (waitMs: number) => void;
openSource: () => Promise<S>;
upload: (source: S) => Promise<T>;
}): Promise<T> {
const { registry, scope, concurrency, budget, signal, label, onWait, openSource, upload } =
params;
return withCodeApiUploadSlot({
registry,
scope,
concurrency,
signal,
task: () => {
let uploadStarted = false;
return withCodeApiRateLimit({
budget,
signal,
label,
onWait,
/* Storage backends can also answer 429. Only the CodeAPI transport owns this
* retry policy and its operator-facing error identity. */
isRateLimit: (error) =>
uploadStarted && isAxiosError(error) && error.response?.status === 429,
attempt: async () => {
uploadStarted = false;
let source: S | undefined;
try {
source = await openSource();
signal?.throwIfAborted();
uploadStarted = true;
return await upload(source);
} catch (error) {
if (source !== undefined) {
disposeCodeApiUploadSource(source);
}
throw error;
}
},
});
},
});
}
/** Best-effort disposal for the stream shapes accepted by upload adapters: a
* single readable or a batch of `{ stream }` entries. */
function disposeCodeApiUploadSource(source: unknown): void {
try {
if (Array.isArray(source)) {
source.forEach(disposeCodeApiUploadSource);
return;
}
if (source == null || typeof source !== 'object') {
return;
}
if ('stream' in source) {
disposeCodeApiUploadSource((source as { stream?: unknown }).stream);
return;
}
const destroy = (source as { destroy?: unknown }).destroy;
if (typeof destroy === 'function') {
destroy.call(source);
}
} catch {
/* Cleanup must not replace the upload or cancellation error being propagated. */
}
}
export function isCodeApiRateLimitError(error: unknown): error is Error & {
code: 'CODE_API_RATE_LIMITED';
status: 429;
} {
return (
error != null &&
typeof error === 'object' &&
(error as { code?: unknown }).code === 'CODE_API_RATE_LIMITED' &&
(error as { status?: unknown }).status === 429
);
}