mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-16 21:08:17 +00:00
Some checks are pending
Backend Unit Tests / Build packages (push) Waiting to run
Backend Unit Tests / Codegraph select (push) Waiting to run
Backend Unit Tests / TypeScript type checks (push) Blocked by required conditions
Backend Unit Tests / Circular dependency checks (push) Waiting to run
Backend Unit Tests / Tests: api (shard 1/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 2/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 3/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: data-provider (push) Blocked by required conditions
Backend Unit Tests / Tests: data-schemas (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 1/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 2/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 3/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 4/4) (push) Blocked by required conditions
Codegraph E2E Votes / vote (full suite) (push) Waiting to run
Docker Dev Branch Images Build / publish (push) Waiting to run
Frontend Unit Tests / Client build recovery regression (push) Waiting to run
Frontend Unit Tests / Codegraph select (push) Waiting to run
Frontend Unit Tests / Build packages (push) Waiting to run
Frontend Unit Tests / TypeScript type checks (client) (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/client (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 1/2) (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 2/2) (push) Blocked by required conditions
Frontend Unit Tests / Vite build verification (push) Blocked by required conditions
Sync Locize Translations & Create Translation PR / Sync Translation Keys with Locize (push) Waiting to run
Sync Locize Translations & Create Translation PR / Create Translation PR on Version Published (push) Blocked by required conditions
* 🔁 fix: Retry Code API Rate Limits That Reject Before a Workspace Call Starts * 🔁 fix: Bound Workspace Rate Limit Retries by Code API Budget * 🔗 fix: Isolate Code API Retry Default from Utility Graph * 🧩 fix: Type the Shared Code API Retry Default * 🔁 fix: Keep Workspace Admission Retry Budgets Independent --------- Co-authored-by: Lia <lia@librechat.ai>
410 lines
13 KiB
TypeScript
410 lines
13 KiB
TypeScript
import http from 'http';
|
|
import https from 'https';
|
|
import { isAxiosError } from 'axios';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import type { ServerRequest } from '~/types';
|
|
import { CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS } from '~/code/limits';
|
|
|
|
/**
|
|
* Dedicated agents for code-server requests, preventing socket pool contamination.
|
|
* follow-redirects (used by axios) leaks `socket.destroy` as a timeout listener;
|
|
* on Node 19+ (keepAlive: true by default), tainted sockets re-enter the global pool
|
|
* and kill unrelated requests (e.g., node-fetch in CodeExecutor) after the idle timeout.
|
|
*/
|
|
export const codeServerHttpAgent: http.Agent = new http.Agent({ keepAlive: false });
|
|
export const codeServerHttpsAgent: https.Agent = new https.Agent({ keepAlive: false });
|
|
|
|
/**
|
|
* Wait implied by a Code API 429, in milliseconds, or `null` when the error
|
|
* is not a rate-limit response. The service answers with both the standard
|
|
* `Retry-After` header and a `retry_after_seconds` body field; the header is
|
|
* preferred so a body the caller asked for as a stream/buffer never has to be
|
|
* parsed, and the body is the fallback for proxies that drop the header.
|
|
*/
|
|
export function getCodeApiRetryAfterMs(error: unknown): number | null {
|
|
if (!isAxiosError(error) || error.response?.status !== 429) {
|
|
return null;
|
|
}
|
|
const header = error.response.headers?.['retry-after'];
|
|
const raw = Array.isArray(header) ? header[0] : header;
|
|
/* `Number('')` is 0, which would read as "retry immediately" and skip the
|
|
* body fallback; treat a blank header as absent. */
|
|
const headerSeconds = typeof raw === 'string' && raw.trim() === '' ? NaN : Number(raw);
|
|
if (Number.isFinite(headerSeconds) && headerSeconds >= 0) {
|
|
return headerSeconds * 1000;
|
|
}
|
|
const body = error.response.data;
|
|
if (typeof body === 'object' && body !== null && 'retry_after_seconds' in body) {
|
|
const bodySeconds = Number(body.retry_after_seconds);
|
|
if (Number.isFinite(bodySeconds) && bodySeconds >= 0) {
|
|
return bodySeconds * 1000;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/** Total time one operation may spend waiting out Code API rate limits.
|
|
* Keeps recovery bounded inside a live chat turn. */
|
|
export const MAX_CODE_API_RATE_LIMIT_WAIT_MS: number = CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS;
|
|
export const CODE_API_UPLOAD_CONCURRENCY_DEFAULT = 3;
|
|
|
|
export interface CodeApiUploadRegistry {
|
|
scopes: Map<string, UploadScopeState>;
|
|
}
|
|
|
|
export function createCodeApiUploadRegistry(): CodeApiUploadRegistry {
|
|
return { scopes: new Map() };
|
|
}
|
|
|
|
export function createCodeApiUploadScope(params: {
|
|
route: string;
|
|
principalId: string;
|
|
tenantId?: string;
|
|
}): string {
|
|
return JSON.stringify([params.route, params.tenantId ?? 'legacy', params.principalId]);
|
|
}
|
|
|
|
export function getCodeApiUploadOptions(
|
|
req: ServerRequest,
|
|
route: string,
|
|
): { scope: string; concurrency: number; retryWaitMs: number } {
|
|
const user = req.user as typeof req.user & {
|
|
_id?: { toString(): string } | string;
|
|
tenantId?: unknown;
|
|
orgId?: unknown;
|
|
};
|
|
const principalId = user?.id ?? user?._id?.toString();
|
|
if (!principalId) {
|
|
throw new Error('Code API upload recovery requires an authenticated principal');
|
|
}
|
|
return {
|
|
scope: createCodeApiUploadScope({
|
|
route,
|
|
principalId,
|
|
tenantId:
|
|
user.tenantId == null && user.orgId == null
|
|
? undefined
|
|
: String(user.tenantId ?? user.orgId),
|
|
}),
|
|
concurrency:
|
|
req.config?.endpoints?.agents?.codeApiUploadConcurrency ??
|
|
CODE_API_UPLOAD_CONCURRENCY_DEFAULT,
|
|
retryWaitMs:
|
|
req.config?.endpoints?.agents?.codeApiMaxRetryWaitMs ?? MAX_CODE_API_RATE_LIMIT_WAIT_MS,
|
|
};
|
|
}
|
|
|
|
interface QueuedUpload {
|
|
run: () => void;
|
|
reject: (error: unknown) => void;
|
|
signal?: AbortSignal;
|
|
onAbort?: () => void;
|
|
}
|
|
|
|
interface UploadScopeState {
|
|
active: number;
|
|
concurrency: number;
|
|
queue: QueuedUpload[];
|
|
}
|
|
|
|
interface CodeApiRateLimitWaitToken {
|
|
endAt: number;
|
|
}
|
|
|
|
function abortReason(signal: AbortSignal): unknown {
|
|
try {
|
|
signal.throwIfAborted();
|
|
} catch (error) {
|
|
return error;
|
|
}
|
|
return new Error('The operation was aborted');
|
|
}
|
|
|
|
function drainCodeApiUploadScope(
|
|
registry: CodeApiUploadRegistry,
|
|
scope: string,
|
|
state: UploadScopeState,
|
|
): void {
|
|
while (state.active < state.concurrency && state.queue.length > 0) {
|
|
const queued = state.queue.shift();
|
|
if (!queued) {
|
|
break;
|
|
}
|
|
queued.signal?.removeEventListener('abort', queued.onAbort!);
|
|
if (queued.signal?.aborted) {
|
|
queued.reject(abortReason(queued.signal));
|
|
continue;
|
|
}
|
|
queued.run();
|
|
}
|
|
if (state.active === 0 && state.queue.length === 0) {
|
|
registry.scopes.delete(scope);
|
|
}
|
|
}
|
|
|
|
/** Limits uploads within one Code API route and principal. A throttled bucket
|
|
* holds only its own slots, and canceled callers leave the queue immediately. */
|
|
export function withCodeApiUploadSlot<T>(params: {
|
|
registry: CodeApiUploadRegistry;
|
|
task: () => Promise<T>;
|
|
scope: string;
|
|
concurrency?: number;
|
|
signal?: AbortSignal;
|
|
}): Promise<T> {
|
|
const {
|
|
registry,
|
|
task,
|
|
scope,
|
|
concurrency = CODE_API_UPLOAD_CONCURRENCY_DEFAULT,
|
|
signal,
|
|
} = params;
|
|
if (!Number.isInteger(concurrency) || concurrency < 1) {
|
|
throw new Error(`Code API upload concurrency must be a positive integer (got ${concurrency})`);
|
|
}
|
|
signal?.throwIfAborted();
|
|
let state = registry.scopes.get(scope);
|
|
if (!state) {
|
|
state = { active: 0, concurrency, queue: [] };
|
|
registry.scopes.set(scope, state);
|
|
} else {
|
|
state.concurrency = concurrency;
|
|
}
|
|
|
|
return new Promise<T>((resolve, reject) => {
|
|
const run = (): void => {
|
|
state.active++;
|
|
Promise.resolve()
|
|
.then(task)
|
|
.then(resolve, reject)
|
|
.finally(() => {
|
|
state.active--;
|
|
drainCodeApiUploadScope(registry, scope, state);
|
|
});
|
|
};
|
|
if (state.active < state.concurrency) {
|
|
run();
|
|
return;
|
|
}
|
|
const queued: QueuedUpload = { run, reject, signal };
|
|
queued.onAbort = () => {
|
|
const index = state.queue.indexOf(queued);
|
|
if (index >= 0) {
|
|
state.queue.splice(index, 1);
|
|
reject(abortReason(signal!));
|
|
drainCodeApiUploadScope(registry, scope, state);
|
|
}
|
|
};
|
|
signal?.addEventListener('abort', queued.onAbort, { once: true });
|
|
state.queue.push(queued);
|
|
});
|
|
}
|
|
|
|
/** Actual rate-limit wait time shared by every request of one operation. */
|
|
export interface CodeApiRateLimitBudget {
|
|
limitMs: number;
|
|
waitedMs: number;
|
|
waitingSince?: number;
|
|
activeWaitEnds: Set<CodeApiRateLimitWaitToken>;
|
|
}
|
|
|
|
export function createCodeApiRateLimitBudget(
|
|
totalMs: number = MAX_CODE_API_RATE_LIMIT_WAIT_MS,
|
|
): CodeApiRateLimitBudget {
|
|
return { limitMs: totalMs, waitedMs: 0, activeWaitEnds: new Set() };
|
|
}
|
|
|
|
function beginCodeApiRateLimitWait(
|
|
budget: CodeApiRateLimitBudget,
|
|
waitMs: number,
|
|
): CodeApiRateLimitWaitToken | null {
|
|
const now = Date.now();
|
|
const activeElapsed = budget.waitingSince == null ? 0 : now - budget.waitingSince;
|
|
const currentWaitEnd = Math.max(
|
|
now,
|
|
...Array.from(budget.activeWaitEnds, (token) => token.endAt),
|
|
);
|
|
const projectedWaitedMs =
|
|
budget.waitedMs + activeElapsed + Math.max(currentWaitEnd, now + waitMs) - now;
|
|
if (projectedWaitedMs > budget.limitMs) {
|
|
return null;
|
|
}
|
|
const token = { endAt: now + waitMs };
|
|
if (budget.activeWaitEnds.size === 0) {
|
|
budget.waitingSince = now;
|
|
}
|
|
budget.activeWaitEnds.add(token);
|
|
return token;
|
|
}
|
|
|
|
function finishCodeApiRateLimitWait(
|
|
budget: CodeApiRateLimitBudget,
|
|
token: CodeApiRateLimitWaitToken,
|
|
): void {
|
|
if (!budget.activeWaitEnds.delete(token)) {
|
|
return;
|
|
}
|
|
if (budget.activeWaitEnds.size === 0 && budget.waitingSince != null) {
|
|
budget.waitedMs += Date.now() - budget.waitingSince;
|
|
budget.waitingSince = undefined;
|
|
}
|
|
}
|
|
|
|
function codeApiRateLimitError(label: string, cause: unknown, retryAfterMs?: number): Error {
|
|
const retrySuffix = retryAfterMs == null ? '' : ` (retry in ${Math.ceil(retryAfterMs / 1000)}s)`;
|
|
return Object.assign(
|
|
new Error(`Code API rate limit reached while ${label}${retrySuffix}.`, { cause }),
|
|
{
|
|
name: 'CodeApiRateLimitError',
|
|
code: 'CODE_API_RATE_LIMITED',
|
|
status: 429,
|
|
statusCode: 429,
|
|
...(retryAfterMs == null ? {} : { retryAfterMs }),
|
|
},
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Runs a Code API request, waiting out a rate limit whenever the shared
|
|
* budget still covers the server's `Retry-After`. Multi-request operations
|
|
* (a windowed image read, a batch upload) would otherwise abandon the work
|
|
* already done the moment a limiter window closes mid-flight. A 429 the
|
|
* budget cannot absorb throws a named error rather than a bare
|
|
* "status code 429", which reads as an unspecified failure instead of one
|
|
* that clears on its own.
|
|
*/
|
|
export async function withCodeApiRateLimit<T>(params: {
|
|
attempt: () => Promise<T>;
|
|
label: string;
|
|
budget?: CodeApiRateLimitBudget;
|
|
onWait?: (waitMs: number) => void;
|
|
signal?: AbortSignal;
|
|
/** Narrows retry classification when an attempt also performs non-CodeAPI work. */
|
|
isRateLimit?: (error: unknown) => boolean;
|
|
}): Promise<T> {
|
|
const {
|
|
attempt,
|
|
label,
|
|
budget,
|
|
onWait,
|
|
signal,
|
|
isRateLimit = (error) => isAxiosError(error) && error.response?.status === 429,
|
|
} = params;
|
|
for (;;) {
|
|
signal?.throwIfAborted();
|
|
try {
|
|
return await attempt();
|
|
} catch (error) {
|
|
if (!isRateLimit(error)) {
|
|
throw error;
|
|
}
|
|
const retryAfterMs = getCodeApiRetryAfterMs(error);
|
|
if (retryAfterMs == null) {
|
|
throw codeApiRateLimitError(label, error);
|
|
}
|
|
/* A zero delay would spin. Wait at least a second; the actual-wait
|
|
* budget still guarantees that repeated zero hints terminate. */
|
|
const waitMs = Math.max(retryAfterMs, 1000);
|
|
if (budget == null) {
|
|
throw codeApiRateLimitError(label, error, waitMs);
|
|
}
|
|
const rateLimitBudget = budget;
|
|
const waitToken = beginCodeApiRateLimitWait(rateLimitBudget, waitMs);
|
|
if (waitToken == null) {
|
|
throw codeApiRateLimitError(label, error, waitMs);
|
|
}
|
|
onWait?.(waitMs);
|
|
try {
|
|
await delay(waitMs, undefined, signal ? { signal } : undefined);
|
|
} finally {
|
|
finishCodeApiRateLimitWait(rateLimitBudget, waitToken);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/** Owns upload admission, retry waiting, and source reopening for every Code API
|
|
* upload producer. Callers supply storage and transport adapters only. */
|
|
export function withCodeApiUploadRecovery<S, T>(params: {
|
|
registry: CodeApiUploadRegistry;
|
|
scope: string;
|
|
concurrency?: number;
|
|
budget?: CodeApiRateLimitBudget;
|
|
signal?: AbortSignal;
|
|
label: string;
|
|
onWait?: (waitMs: number) => void;
|
|
openSource: () => Promise<S>;
|
|
upload: (source: S) => Promise<T>;
|
|
}): Promise<T> {
|
|
const { registry, scope, concurrency, budget, signal, label, onWait, openSource, upload } =
|
|
params;
|
|
return withCodeApiUploadSlot({
|
|
registry,
|
|
scope,
|
|
concurrency,
|
|
signal,
|
|
task: () => {
|
|
let uploadStarted = false;
|
|
return withCodeApiRateLimit({
|
|
budget,
|
|
signal,
|
|
label,
|
|
onWait,
|
|
/* Storage backends can also answer 429. Only the CodeAPI transport owns this
|
|
* retry policy and its operator-facing error identity. */
|
|
isRateLimit: (error) =>
|
|
uploadStarted && isAxiosError(error) && error.response?.status === 429,
|
|
attempt: async () => {
|
|
uploadStarted = false;
|
|
let source: S | undefined;
|
|
try {
|
|
source = await openSource();
|
|
signal?.throwIfAborted();
|
|
uploadStarted = true;
|
|
return await upload(source);
|
|
} catch (error) {
|
|
if (source !== undefined) {
|
|
disposeCodeApiUploadSource(source);
|
|
}
|
|
throw error;
|
|
}
|
|
},
|
|
});
|
|
},
|
|
});
|
|
}
|
|
|
|
/** Best-effort disposal for the stream shapes accepted by upload adapters: a
|
|
* single readable or a batch of `{ stream }` entries. */
|
|
function disposeCodeApiUploadSource(source: unknown): void {
|
|
try {
|
|
if (Array.isArray(source)) {
|
|
source.forEach(disposeCodeApiUploadSource);
|
|
return;
|
|
}
|
|
if (source == null || typeof source !== 'object') {
|
|
return;
|
|
}
|
|
if ('stream' in source) {
|
|
disposeCodeApiUploadSource((source as { stream?: unknown }).stream);
|
|
return;
|
|
}
|
|
const destroy = (source as { destroy?: unknown }).destroy;
|
|
if (typeof destroy === 'function') {
|
|
destroy.call(source);
|
|
}
|
|
} catch {
|
|
/* Cleanup must not replace the upload or cancellation error being propagated. */
|
|
}
|
|
}
|
|
|
|
export function isCodeApiRateLimitError(error: unknown): error is Error & {
|
|
code: 'CODE_API_RATE_LIMITED';
|
|
status: 429;
|
|
} {
|
|
return (
|
|
error != null &&
|
|
typeof error === 'object' &&
|
|
(error as { code?: unknown }).code === 'CODE_API_RATE_LIMITED' &&
|
|
(error as { status?: unknown }).status === 429
|
|
);
|
|
}
|