🛟 fix: Recover CodeAPI File Uploads After Throttling (#15784)

* fix: recover CodeAPI uploads after throttling

* fix: cover eager CodeAPI uploads

* fix: scope CodeAPI upload recovery

* test: clarify upload recovery deadlines

* fix: isolate CodeAPI recovery state

* fix: cancel CodeAPI recovery waits

* test: provide CodeAPI registry in handler fixture

* fix: propagate CodeAPI recovery controls

* fix: bound CodeAPI recovery by run

* fix: preserve skill upload cancellation

* test: update deferred loader signal contract

* fix: preserve recovery cancellation

* fix: cancel recovery source reads

* fix: classify Axios upload cancellation

* fix: preserve lazy provisioning cancellation

* fix: close resource recovery boundaries

* fix: preserve provisioning failures compatibly
This commit is contained in:
Danny Avila 2026-09-09 15:48:11 -04:00 • committed by GitHub
parent f0c9dee669
commit 65bb64bc68
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
30 changed files with 1792 additions and 161 deletions

View file

@ -388,6 +388,7 @@ const loadTools = async ({
});
const { files, toolContext } = await primeCodeFiles({
...options,
signal,
agentId: agent?.id,
codeApiBaseUrl: codeExecutionContext.baseUrl,
executionProfile: codeExecutionContext.executionProfile,

View file

@ -374,11 +374,13 @@ describe('Tool Handlers', () => {
it('routes code file priming to the selected bridge worker', async () => {
const bridgeWorkerId = 'principal-worker';
const controller = new AbortController();
const toolMap = await loadTools({
user: fakeUser._id.toString(),
tools: [Tools.execute_code],
returnMap: true,
agent: { id: 'agent-1' },
signal: controller.signal,
options: {
codeExecutionContext: {
baseUrl: 'https://code.example.com/v1',
@ -394,6 +396,7 @@ describe('Tool Handlers', () => {
agentId: 'agent-1',
codeApiBaseUrl: 'https://code.example.com/v1',
bridgeWorkerId,
signal: controller.signal,
}),
);
});

View file

@ -51,6 +51,7 @@ const {
createAgentEventTerminalHandler,
startCodeEnvironmentLifecycleReconciler,
waitForKeyvRedisClient,
createCodeApiUploadRegistry,
} = require('@librechat/api');
const { connectDb, indexSync } = require('~/db');
const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager');
@ -356,6 +357,7 @@ if (cluster.isMaster) {
* Each worker runs a full Express server instance
*/
const app = express();
app.locals.codeApiUploadRegistry = createCodeApiUploadRegistry();
// The clustered entrypoint deliberately does not arm the v1 schedule engine,
// but an already-fired scheduled generation can still reach HITL here. Settle
// its durable run when the generic approval runtime expires it.

View file

@ -58,6 +58,7 @@ const {
startCodeEnvironmentLifecycleReconciler,
waitForKeyvRedisClient,
warnOnUnreachableDeliveryPaths,
createCodeApiUploadRegistry,
} = require('@librechat/api');
const { connectDb, indexSync } = require('~/db');
const {
@ -101,6 +102,7 @@ const host = HOST || 'localhost';
const trusted_proxy = Number(TRUST_PROXY) || 1; /* trust first proxy by default */
const app = express();
app.locals.codeApiUploadRegistry = createCodeApiUploadRegistry();
let serverReady = false;
/** @type {import('@librechat/api').ScheduleEngineState} */
let scheduleEngineState = 'starting';

View file

@ -413,7 +413,7 @@ const initializeClient = async ({
runSignal: signal,
foregroundRunId,
ordinaryToolCancellation: ordinaryToolCancellationEnabled,
loadTools: async (toolNames, agentId, _configurable, callerCapabilityProjection) => {
loadTools: async (toolNames, agentId, _configurable, callerCapabilityProjection, runSignal) => {
const ctx = agentToolContexts.get(agentId) ?? {};
logger.debug(`[ON_TOOL_EXECUTE] ctx found: ${!!ctx.userMCPAuthMap}, agent: ${ctx.agent?.id}`);
logger.debug(`[ON_TOOL_EXECUTE] toolRegistry size: ${ctx.toolRegistry?.size ?? 'undefined'}`);
@ -421,7 +421,7 @@ const initializeClient = async ({
const result = await loadToolsForExecution({
req,
res,
signal,
signal: runSignal ?? signal,
streamId,
conversationId,
requestBody: runtimeRequestBody,
@ -1599,6 +1599,7 @@ const initializeClient = async ({
req,
payload,
skillNames,
signal,
accessibleSkillIds,
executionProfiles: codeExecutionProfiles,
...getSkillToolDeps(),

View file

@ -4,6 +4,7 @@ const { getCodeBaseURL } = require('@librechat/agents');
const { EModelEndpoint, getCodeEnvRefs } = require('librechat-data-provider');
const {
logAxiosError,
wrapCodeApiUploadError,
appendCodeEnvFile,
createAxiosInstance,
codeServerHttpAgent,
@ -183,6 +184,7 @@ async function deleteCodeEnvFile(req, file) {
* @param {string} [params.codeApiBaseUrl] - Trusted per-agent Code API endpoint.
* @param {'default'|'stateful'} [params.executionProfile] - Trusted execution profile.
* @param {string} [params.bridgeWorkerId] - Trusted worker selected for this execution.
* @param {AbortSignal} [params.signal] - Effective cancellation signal.
* @returns {Promise<{ storage_session_id: string; file_id: string }>}
* The codeapi storage location of the uploaded file.
* @throws {Error} If there's an error during the upload process.
@ -237,12 +239,7 @@ async function uploadCodeEnvFile({
file_id: result.files[0].fileId,
};
} catch (error) {
throw new Error(
logAxiosError({
message: `Error uploading code environment file: ${error.message}`,
error,
}),
);
throw wrapCodeApiUploadError(error, `Error uploading code environment file: ${error.message}`);
}
}
@ -281,6 +278,7 @@ async function batchUploadCodeEnvFiles({
codeApiBaseUrl,
executionProfile,
bridgeWorkerId,
signal,
}) {
const form = new FormData();
appendCodeEnvFileIdentity(form, { kind, id, version });
@ -308,6 +306,7 @@ async function batchUploadCodeEnvFiles({
timeout: 120000,
maxContentLength: MAX_FILE_SIZE,
maxBodyLength: MAX_FILE_SIZE,
signal,
};
const response = await axios.post(`${baseURL}/upload/batch`, form, options);

View file

@ -42,6 +42,14 @@ jest.mock('@librechat/api', () => {
form.append('id', identity.id);
if (identity.version != null) form.append('version', String(identity.version));
}),
wrapCodeApiUploadError: jest.fn((error, message) => {
const wrapped = new Error(`${message} ${error.message}`, { cause: error });
if (error?.isAxiosError === true) {
wrapped.isAxiosError = true;
wrapped.response = error.response;
}
return wrapped;
}),
buildCodeEnvDownloadQuery: jest.fn((identity) => {
validateIdentity(identity, 'buildCodeEnvDownloadQuery');
const params = new URLSearchParams({ kind: identity.kind, id: identity.id });
@ -561,10 +569,25 @@ describe('Code CRUD', () => {
await expect(uploadCodeEnvFile(baseUploadParams)).rejects.toThrow();
});
it('preserves rate-limit metadata through the upload logging boundary', async () => {
const rateLimit = Object.assign(new Error('Too Many Requests'), {
isAxiosError: true,
response: { status: 429, headers: { 'retry-after': '7' } },
});
mockAxios.post.mockRejectedValue(rateLimit);
await expect(uploadCodeEnvFile(baseUploadParams)).rejects.toMatchObject({
isAxiosError: true,
response: { status: 429, headers: { 'retry-after': '7' } },
cause: rateLimit,
});
});
});
describe('batchUploadCodeEnvFiles', () => {
it('routes batch uploads through the selected bridge worker', async () => {
const controller = new AbortController();
const req = { user: { id: 'user-123' } };
mockAxios.post.mockResolvedValue({
data: {
@ -584,6 +607,7 @@ describe('Code CRUD', () => {
codeApiBaseUrl: 'https://stateful-code.example.com',
executionProfile: 'stateful',
bridgeWorkerId: 'personal-worker-1',
signal: controller.signal,
});
const [url, , callConfig] = mockAxios.post.mock.calls[0];
@ -591,6 +615,7 @@ describe('Code CRUD', () => {
expect(getCodeApiAuthHeaders).toHaveBeenCalledWith(req, 'personal-worker-1');
expect(callConfig.headers['X-CodeAPI-Expected-Profile']).toBe('stateful');
expect(callConfig.headers['X-LibreChat-Code-Worker-ID']).toBe('personal-worker-1');
expect(callConfig.signal).toBe(controller.signal);
});
});
});

View file

@ -11,7 +11,10 @@ const {
flattenArtifactPath,
createAxiosInstance,
getCodeApiAuthHeaders,
isAbortError,
getCodeApiUploadOptions,
withCodeApiRateLimit,
withCodeApiUploadRecovery,
classifyCodeArtifact,
isMissingSandboxPathError,
parseSandboxImageChunk,
@ -1138,15 +1141,18 @@ function checkIfActive(dateString) {
* @param {ServerRequest} [req] - Current authenticated request, used to mint Code API auth.
* @param {{baseUrl?: string, executionProfile?: 'default'|'stateful', bridgeWorkerId?: string}} [route]
* Trusted host-selected Code API route.
* @param {AbortSignal} [signal] - Effective run cancellation signal.
*
* @returns {Promise<string|null>}
* A promise that resolves to the `lastModified` time string of the file if successful, or null if there is an
* error in initialization or fetching the info.
*/
async function getSessionInfo(ref, req, route = {}) {
async function getSessionInfo(ref, req, route = {}, signal) {
try {
signal?.throwIfAborted();
const baseURL = route.baseUrl ?? getCodeBaseURL();
const authHeaders = await getCodeApiAuthHeaders(req, route.bridgeWorkerId);
signal?.throwIfAborted();
/* `/sessions/.../objects/...` is gated by codeapi's `sessionAuth`
* middleware (post-Phase C). The middleware reconstructs the
* sessionKey from the URL query (`kind`/`id`/`version?`) plus the
@ -1174,10 +1180,15 @@ async function getSessionInfo(ref, req, route = {}) {
httpAgent: codeServerHttpAgent,
httpsAgent: codeServerHttpsAgent,
timeout: 5000,
signal,
});
signal?.throwIfAborted();
return response.data?.lastModified;
} catch (_error) {
} catch (error) {
if (signal?.aborted && isAbortError(error)) {
throw error;
}
logger.debug('[getSessionInfo] session lookup failed (treating as cache miss)');
return null;
}
@ -1275,6 +1286,9 @@ const getReuploadFailureCategory = (error) => {
) {
return 'resource_access_denied';
}
if (status === 429 || code === 'CODE_API_RATE_LIMITED') {
return 'rate_limited';
}
return 'reupload_failed';
};
@ -1285,6 +1299,7 @@ const getReuploadFailureCategory = (error) => {
* @param {Agent['tool_resources']} options.tool_resources
* @param {string} [options.agentId] - The agent ID for file access control
* @param {string} [options.agentResourceType] - Permission resource type for the authorized agent route
* @param {AbortSignal} [options.signal] - Effective run cancellation signal
* @returns {Promise<{
* files: Array<{ id: string; session_id: string; name: string }>,
* toolContext: string,
@ -1300,6 +1315,7 @@ const primeFiles = async (options) => {
executionProfile = 'default',
executionRouteKey = executionProfile,
bridgeWorkerId,
signal,
} = options;
const codeApiRoute = { baseUrl: codeApiBaseUrl, executionProfile, bridgeWorkerId };
const file_ids = tool_resources?.[EToolResources.execute_code]?.file_ids ?? [];
@ -1342,6 +1358,9 @@ const primeFiles = async (options) => {
const files = [];
const sessions = new Map();
const uploadOptions = getCodeApiUploadOptions(req, executionRouteKey);
/** All stale-file reuploads in this prime share one live-turn wait cap. */
const uploadRateLimitBudget = createCodeApiRateLimitBudget(uploadOptions.retryWaitMs);
let toolContext = '';
/* Claim order decides which record keeps the bare `/mnt/data/<name>` path
@ -1442,22 +1461,40 @@ const primeFiles = async (options) => {
const { handleFileUpload: uploadCodeEnvFile } = getStrategyFunctions(
FileSources.execute_code,
);
const stream = await getDownloadStream(options.req, resolveDownloadPath(file));
/* Reupload preserves the resource identity from the existing
* ref so codeapi re-buckets under the same sessionKey shape
* (skill stays skill, user stays user). Without this, a
* skill-cache-miss reupload would land in the user bucket
* and never re-shareable cross-user. */
const uploaded = await uploadCodeEnvFile({
req: options.req,
stream,
filename: getDestination(),
kind: sourceRef.kind,
id: sourceRef.id,
...(sourceRef.kind === 'skill' ? { version: sourceRef.version } : {}),
codeApiBaseUrl,
executionProfile,
bridgeWorkerId,
const uploaded = await withCodeApiUploadRecovery({
registry: req.app?.locals?.codeApiUploadRegistry,
scope: uploadOptions.scope,
concurrency: uploadOptions.concurrency,
label: `re-uploading file ${file.file_id} to the code environment`,
budget: uploadRateLimitBudget,
signal,
onWait: (waitMs) =>
logger.warn(
`[primeCodeFiles] Rate-limited reupload requestId=${getPrimingCorrelation(req).requestId} ` +
`runId=${getPrimingCorrelation(req).runId}; retrying in ${waitMs}ms`,
),
openSource: async () => {
signal?.throwIfAborted();
return getDownloadStream(options.req, resolveDownloadPath(file), { signal });
},
upload: (stream) =>
uploadCodeEnvFile({
req: options.req,
stream,
filename: getDestination(),
kind: sourceRef.kind,
id: sourceRef.id,
...(sourceRef.kind === 'skill' ? { version: sourceRef.version } : {}),
codeApiBaseUrl,
executionProfile,
bridgeWorkerId,
signal,
}),
});
/**
@ -1497,6 +1534,10 @@ const primeFiles = async (options) => {
`oldSession=${session_id} newSession=${newRef.storage_session_id} newFileId=${newRef.file_id}`,
);
} catch (error) {
/* Cancellation is an operation outcome, not a recoverable per-file
* miss. Swallowing it here would keep walking and could dispatch more
* uploads after the foreground run has ended. */
signal?.throwIfAborted();
reuploadFailures += 1;
const failureCategory = getReuploadFailureCategory(error);
reuploadFailureCategories.add(failureCategory);
@ -1522,7 +1563,8 @@ const primeFiles = async (options) => {
pushFile();
continue;
}
const uploadTime = await getSessionInfo(ref, req, codeApiRoute);
const uploadTime = await getSessionInfo(ref, req, codeApiRoute, signal);
signal?.throwIfAborted();
if (!uploadTime) {
logger.debug(
`[primeCodeFiles] file=${file.file_id} path=reupload reason=no-uploadtime ` +
@ -1927,6 +1969,7 @@ async function previewWorkspaceEdit({
* @param {string} [params.runtime_session_hint] - Per-conversation stateful runtime-session hint.
* @param {number} [params.maxBytes] - In-sandbox size cap; larger files return `{ tooLarge, bytes }`.
* @param {ServerRequest} [params.req] - Current authenticated request, used to mint Code API auth.
* @param {AbortSignal} [params.signal] - Foreground run cancellation.
* @param {string} [params.executionRouteKey] - Trusted deployment-local route identity.
* @param {string} [params.bridgeWorkerId] - Trusted bridge worker selected for this execution.
* @returns {Promise<{base64: string, bytes: number}
@ -1944,6 +1987,7 @@ async function readSandboxImage({
executionRouteKey,
maxBytes,
req,
signal,
}) {
const limit = typeof maxBytes === 'number' && maxBytes > 0 ? maxBytes : 5 * megabyte;
const preparedBuffer = getPreparedCodeOutputBuffer({
@ -1970,7 +2014,9 @@ async function readSandboxImage({
/** Every window is one `/exec` call against the Code API's per-user
* execution limiter, so the read shares one wait budget: a window that
* resets mid-read is worth pausing for, an exhausted budget is not. */
const rateLimit = createCodeApiRateLimitBudget();
const rateLimit = createCodeApiRateLimitBudget(
req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
);
return readWindowedSandboxImage({
filePath: file_path,
baseUrl: baseURL,
@ -1987,6 +2033,7 @@ async function readSandboxImage({
files,
req,
rateLimit,
signal,
}),
});
}
@ -2009,6 +2056,7 @@ async function execSandboxImageChunk({
files,
req,
rateLimit,
signal,
}) {
/** @type {Record<string, unknown>} */
const postData = { lang: 'bash', code };
@ -2026,6 +2074,7 @@ async function execSandboxImageChunk({
const response = await withCodeApiRateLimit({
label: `reading "${file_path}" from the sandbox`,
budget: rateLimit,
signal,
onWait: (waitMs) =>
logger.warn(
`[readSandboxImage] Rate-limited reading "${file_path}"; retrying in ${waitMs}ms`,
@ -2045,6 +2094,7 @@ async function execSandboxImageChunk({
httpAgent: codeServerHttpAgent,
httpsAgent: codeServerHttpsAgent,
timeout: 15000,
signal,
});
},
});

View file

@ -90,13 +90,43 @@ jest.mock('@librechat/api', () => {
: undefined,
executeWorkspaceTool: (...args) => mockExecuteWorkspaceTool(...args),
getCodeApiAuthHeaders: jest.fn(async () => ({})),
isAbortError: (error) =>
error?.name === 'AbortError' || error?.code === 'ABORT_ERR' || error?.code === 'ERR_CANCELED',
/* Windowing, sizing and rate-limit policy are real code in
* `packages/api` with their own tests (`files/code/image.spec.ts`,
* `utils/code.spec.ts`). These stand-ins are deliberately inert
* passthroughs — they assert nothing about that behavior, so the
* transport tests below cover only what this file still owns. */
createCodeApiRateLimitBudget: jest.fn(() => ({ remainingMs: 20_000 })),
withCodeApiRateLimit: jest.fn(({ attempt }) => attempt()),
createCodeApiRateLimitBudget: jest.fn(() => ({
limitMs: 20_000,
waitedMs: 0,
activeWaitEnds: new Set(),
})),
getCodeApiUploadOptions: jest.fn(() => ({
scope: 'default:user-123',
concurrency: 3,
retryWaitMs: 20_000,
})),
withCodeApiRateLimit: jest.fn(async ({ attempt }) => {
try {
return await attempt();
} catch (error) {
if (error?.response?.status !== 429) {
throw error;
}
return attempt();
}
}),
withCodeApiUploadRecovery: jest.fn(async ({ openSource, upload }) => {
try {
return await upload(await openSource());
} catch (error) {
if (error?.response?.status !== 429) {
throw error;
}
return upload(await openSource());
}
}),
buildSandboxImageReaderCode: jest.fn(
({ filePath, limit, offset, chunkBytes }) =>
`read ${filePath} ${limit} ${offset} ${chunkBytes}`,
@ -1475,6 +1505,33 @@ describe('Code Process', () => {
expect(callConfig.httpsAgent.keepAlive).toBe(false);
});
it('forwards cancellation to the freshness request and preserves its error', async () => {
const controller = new AbortController();
const canceled = Object.assign(new Error('canceled'), {
name: 'CanceledError',
code: 'ERR_CANCELED',
});
mockAxios.mockImplementationOnce(async (config) => {
expect(config.signal).toBe(controller.signal);
controller.abort();
throw canceled;
});
await expect(
getSessionInfo(
{
kind: 'user',
id: 'user-1',
storage_session_id: 'sess',
file_id: 'fid',
},
mockReq,
{},
controller.signal,
),
).rejects.toBe(canceled);
});
it('forwards Code API auth headers when checking session object freshness', async () => {
getCodeApiAuthHeaders.mockResolvedValueOnce({ Authorization: 'Bearer freshness-token' });
mockAxios.mockResolvedValue({
@ -2585,6 +2642,112 @@ describe('Code Process', () => {
return { handleFileUpload, getDownloadStream };
}
it('recovers a throttled stale-file reupload with a fresh stream', async () => {
const controller = new AbortController();
const dbFile = {
file_id: 'rate-limited-file',
filename: 'report.csv',
filepath: '/uploads/report.csv',
source: 'local',
context: 'execute_code',
metadata: {
codeEnvRef: {
kind: 'user',
id: 'user-123',
storage_session_id: 'OLD_SESSION',
file_id: 'OLD_ID',
},
},
};
const rateLimit = Object.assign(new Error('Request failed with status code 429'), {
isAxiosError: true,
response: { status: 429, headers: { 'retry-after': '0' } },
});
const handleFileUpload = jest
.fn()
.mockRejectedValueOnce(rateLimit)
.mockResolvedValueOnce({ storage_session_id: 'NEW_SESSION', file_id: 'NEW_ID' });
const getDownloadStream = jest
.fn()
.mockResolvedValueOnce('first-stream')
.mockResolvedValueOnce('second-stream');
getStrategyFunctions.mockImplementation((source) =>
source === 'execute_code' ? { handleFileUpload } : { getDownloadStream },
);
getFiles.mockResolvedValue([dbFile]);
filterFilesByAgentAccess.mockImplementation(({ files }) => Promise.resolve(files));
mockAxios.mockResolvedValue({ data: null });
updateFile.mockResolvedValue({});
const result = await primeFiles({
req: { user: { id: 'user-123', role: 'USER' } },
tool_resources: { execute_code: { file_ids: [dbFile.file_id], files: [] } },
agentId: 'agent-id',
signal: controller.signal,
});
const { withCodeApiUploadRecovery } = require('@librechat/api');
expect(withCodeApiUploadRecovery).toHaveBeenCalledWith(
expect.objectContaining({ signal: controller.signal }),
);
expect(handleFileUpload).toHaveBeenCalledTimes(2);
expect(handleFileUpload).toHaveBeenCalledWith(
expect.objectContaining({ signal: controller.signal }),
);
expect(getDownloadStream).toHaveBeenCalledTimes(2);
expect(getDownloadStream).toHaveBeenCalledWith(expect.anything(), '/uploads/report.csv', {
signal: controller.signal,
});
expect(handleFileUpload.mock.calls.map(([args]) => args.stream)).toEqual([
'first-stream',
'second-stream',
]);
expect(result.files).toEqual([
expect.objectContaining({ id: 'NEW_ID', storage_session_id: 'NEW_SESSION' }),
]);
});
it('propagates cancellation from a stale-file reupload', async () => {
const controller = new AbortController();
const dbFile = {
file_id: 'canceled-file',
filename: 'report.csv',
filepath: '/uploads/report.csv',
source: 'local',
context: 'execute_code',
metadata: {
codeEnvRef: {
kind: 'user',
id: 'user-123',
storage_session_id: 'OLD_SESSION',
file_id: 'OLD_ID',
},
},
};
const handleFileUpload = jest.fn(async () => {
controller.abort();
controller.signal.throwIfAborted();
});
getStrategyFunctions.mockImplementation((source) =>
source === 'execute_code'
? { handleFileUpload }
: { getDownloadStream: jest.fn().mockResolvedValue('stream') },
);
getFiles.mockResolvedValue([dbFile]);
filterFilesByAgentAccess.mockImplementation(({ files }) => Promise.resolve(files));
mockAxios.mockResolvedValue({ data: null });
await expect(
primeFiles({
req: { user: { id: 'user-123', role: 'USER' } },
tool_resources: { execute_code: { file_ids: [dbFile.file_id], files: [] } },
agentId: 'agent-id',
signal: controller.signal,
}),
).rejects.toMatchObject({ name: 'AbortError' });
expect(handleFileUpload).toHaveBeenCalledTimes(1);
});
it('uses the permission resource type established by the calling route', async () => {
const files = [
{
@ -2711,6 +2874,7 @@ describe('Code Process', () => {
expect(getDownloadStream).toHaveBeenCalledWith(
{ user: { id: 'user-123', role: 'USER' } },
'/uploads/trusted.txt',
{ signal: undefined },
);
expect(handleFileUpload).toHaveBeenCalledTimes(1);
});
@ -3022,6 +3186,7 @@ describe('Code Process', () => {
],
['Azure BlobNotFound', { code: 'BlobNotFound', statusCode: 404 }, 'missing_backing_object'],
['storage access denied', { code: 'AccessDenied', status: 403 }, 'resource_access_denied'],
['Code API throttling', { code: 'CODE_API_RATE_LIMITED', status: 429 }, 'rate_limited'],
])(
'fails with a typed recovery error for %s',
async (_errorShape, downloadError, expectedCategory) => {
@ -3550,6 +3715,7 @@ describe('Code Process', () => {
});
it('forwards the sandbox session, runtime hint and profile header', async () => {
const controller = new AbortController();
mockAxios.mockResolvedValue({ data: { stdout: '{}' } });
await readSandboxImage({
@ -3559,6 +3725,7 @@ describe('Code Process', () => {
files: [{ id: 'file-1', name: 'seed.csv' }],
codeApiBaseUrl: 'https://code-stateful.example.com',
executionProfile: 'stateful',
signal: controller.signal,
});
expect(mockAxios).toHaveBeenCalledTimes(1);
@ -3571,6 +3738,11 @@ describe('Code Process', () => {
files: [{ id: 'file-1', name: 'seed.csv' }],
});
expect(call.headers['X-CodeAPI-Expected-Profile']).toBe('stateful');
expect(call.signal).toBe(controller.signal);
const { withCodeApiRateLimit } = require('@librechat/api');
expect(withCodeApiRateLimit).toHaveBeenCalledWith(
expect.objectContaining({ signal: controller.signal }),
);
});
it('omits the profile header and optional session fields when unset', async () => {

View file

@ -45,6 +45,9 @@ const {
sweepExpiredFiles: sweepExpiredFilesWithDeps,
startExpiredFileSweep: startExpiredFileSweepWithDeps,
resolveToolRoleGrants,
createCodeApiRateLimitBudget,
getCodeApiUploadOptions,
withCodeApiUploadRecovery,
} = require('@librechat/api');
const {
convertImage,
@ -1221,20 +1224,34 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => {
/* Upload from the persisted representation. Images have already been converted here,
* so both the bytes and extension advertised to the sandbox describe the same file. */
const downloadPath = storageResult.storageKey ?? storageResult.filepath;
const stream = getDownloadStream
? await getDownloadStream(req, downloadPath)
: fs.createReadStream(file.path);
const codeKind = messageAttachment === true ? 'user' : 'agent';
const codeId = messageAttachment === true ? req.user.id : agent_id;
const sandboxFilename = resolveSandboxFilename(sanitizeFilename(file.originalname), storedType);
const uploadOptions = getCodeApiUploadOptions(req, 'default');
let uploaded;
try {
uploaded = await uploadCodeEnvFile({
req,
stream,
filename: sandboxFilename,
kind: codeKind,
id: codeId,
uploaded = await withCodeApiUploadRecovery({
registry: req.app?.locals?.codeApiUploadRegistry,
scope: uploadOptions.scope,
concurrency: uploadOptions.concurrency,
label: `uploading "${sandboxFilename}" to the code environment`,
budget: createCodeApiRateLimitBudget(uploadOptions.retryWaitMs),
onWait: (waitMs) =>
logger.warn(
`[processAgentFileUpload] Rate-limited Code API upload; retrying in ${waitMs}ms`,
),
openSource: () =>
getDownloadStream
? getDownloadStream(req, downloadPath)
: Promise.resolve(fs.createReadStream(file.path)),
upload: (stream) =>
uploadCodeEnvFile({
req,
stream,
filename: sandboxFilename,
kind: codeKind,
id: codeId,
}),
});
} catch (error) {
const { deleteFile } = getStrategyFunctions(source);

View file

@ -37,6 +37,26 @@ jest.mock('@librechat/api', () => {
const actualDataProvider = jest.requireActual('librechat-data-provider');
const RetentionMode = actualDataProvider.RetentionMode ?? { ALL: 'all', TEMPORARY: 'temporary' };
const getRetentionExpiry = jest.fn(() => ({}));
const createCodeApiRateLimitBudget = jest.fn(() => ({
limitMs: 20_000,
waitedMs: 0,
activeWaitEnds: new Set(),
}));
const getCodeApiUploadOptions = jest.fn(() => ({
scope: 'default:user-1',
concurrency: 3,
retryWaitMs: 20_000,
}));
const withCodeApiUploadRecovery = jest.fn(async ({ openSource, upload }) => {
try {
return await upload(await openSource());
} catch (error) {
if (error?.response?.status !== 429) {
throw error;
}
return upload(await openSource());
}
});
const UPLOAD_EXTRACTED_TEXT_PLANS = {
configuredOCR: 'configured_ocr',
configuredRAG: 'configured_rag',
@ -131,6 +151,9 @@ jest.mock('@librechat/api', () => {
}),
getStorageMetadata: jest.fn(() => ({})),
getRetentionExpiry,
createCodeApiRateLimitBudget,
getCodeApiUploadOptions,
withCodeApiUploadRecovery,
getAgentFileRetentionExpiry: jest.fn(({ req, messageAttachment, toolResource }) => {
const interfaceConfig = req?.config?.interfaceConfig;
if (
@ -259,6 +282,9 @@ const {
extractInspectableFileText,
assertExtractedTextInspectable,
contentFilterBlockResponse,
createCodeApiRateLimitBudget,
getCodeApiUploadOptions,
withCodeApiUploadRecovery,
} = require('@librechat/api');
const PDF_MIME = 'application/pdf';
@ -1348,6 +1374,33 @@ describe('processAgentFileUpload', () => {
expect(codeEnvUpload).toHaveBeenCalled();
});
it('retries a throttled eager upload with a fresh persisted stream', async () => {
const rateLimited = Object.assign(new Error('rate limited'), {
isAxiosError: true,
response: { status: 429, headers: { 'retry-after': '1' } },
});
const codeEnvUpload = setupCodeEnvUpload({ storage_session_id: 'sess-y', file_id: 'fid-y' });
codeEnvUpload
.mockRejectedValueOnce(rateLimited)
.mockResolvedValueOnce({ storage_session_id: 'sess-retry', file_id: 'fid-retry' });
await processAgentFileUpload({
req: agentsZipReq(),
res: mockRes,
metadata: {
agent_id: 'agent-abc',
tool_resource: EToolResources.execute_code,
file_id: 'file-throttled',
},
}).catch(() => {});
expect(getCodeApiUploadOptions).toHaveBeenCalledTimes(1);
expect(withCodeApiUploadRecovery).toHaveBeenCalledTimes(1);
expect(createCodeApiRateLimitBudget).toHaveBeenCalledTimes(1);
expect(codeEnvUpload).toHaveBeenCalledTimes(2);
expect(codeEnvUpload.mock.calls[0][0].stream).not.toBe(codeEnvUpload.mock.calls[1][0].stream);
});
it('defers an inferred file-search destination until tool execution', async () => {
const { uploadVectors } = require('~/server/services/Files/VectorDB/crud');
setupStoredFileUpload();

View file

@ -773,6 +773,7 @@ const isBuiltInTool = (toolName) =>
* @param {string} [params.agentResourceType] - Permission resource type for the authorized agent route
* @param {string|null} [params.streamId] - Stream ID for resumable mode
* @param {number} [params.jobCreatedAt] - The generation epoch that owns emitted tool events
* @param {AbortSignal} [params.signal] - Effective run cancellation signal
* @returns {Promise<{
* toolDefinitions?: import('@librechat/api').LCTool[];
* toolRegistry?: Map<string, import('@librechat/api').LCTool>;
@ -792,6 +793,7 @@ async function loadToolDefinitionsWrapper({
tool_resources,
codeExecutionContext,
accessibleMcpServerNames,
signal,
}) {
if (!agent.tools || agent.tools.length === 0) {
return { toolDefinitions: [] };
@ -1448,6 +1450,7 @@ async function loadToolDefinitionsWrapper({
tool_resources,
agentId: agent.id,
agentResourceType,
signal,
codeApiBaseUrl: resolvedCodeExecutionContext.baseUrl,
executionProfile: resolvedCodeExecutionContext.executionProfile,
executionRouteKey: resolvedCodeExecutionContext.executionRouteKey,
@ -1581,6 +1584,7 @@ async function loadAgentTools({
tool_resources,
codeExecutionContext: providedCodeExecutionContext,
accessibleMcpServerNames,
signal,
});
} catch (error) {
if (

View file

@ -583,6 +583,10 @@ endpoints:
# # (optional) Maximum explicit subagents per agent, for both the flat list and
# # graph definitions. Defaults to 10; hard cap 50.
# maxSubagents: 20
# # (optional) Maximum concurrent Code API uploads per route and user. Defaults to 3.
# codeApiUploadConcurrency: 3
# # (optional) Maximum total time spent waiting on Code API rate limits. Defaults to 20000 ms.
# codeApiMaxRetryWaitMs: 20000
# # (optional) Cap the number of active accessible skills shown in the model-visible catalog.
# # Useful for large organizations where many department-specific skills may be available.
# skills:

View file

@ -20,6 +20,7 @@ import {
import { markSandboxReady } from './prewarm';
import { ContentFilterError } from '../middleware/contentFilter';
import { WorkspaceToolHttpError } from '../code/workspace';
import { createCodeApiUploadRegistry } from '~/utils';
function createMockTool(
name: string,
@ -415,6 +416,35 @@ describe('createToolExecuteHandler', () => {
expect(results[0].status).toBe('error');
});
it('forwards the effective batch signal into deferred tool loading', async () => {
const controller = new AbortController();
const tool = {
name: 'loaded_tool',
invoke: jest.fn(async () => ({ content: 'done' })),
};
const loadTools: ToolExecuteOptions['loadTools'] = jest.fn(async () => ({
loadedTools: [tool] as never[],
}));
const handler = createToolExecuteHandler({ loadTools });
await new Promise<ToolExecuteResult[]>((resolve, reject) => {
handler.handle('on_tool_execute', {
toolCalls: [{ id: 'call-load', name: tool.name, args: {} }],
signal: controller.signal,
resolve,
reject,
} as ToolExecuteBatchRequest);
});
expect(loadTools).toHaveBeenCalledWith(
[tool.name],
undefined,
undefined,
undefined,
controller.signal,
);
});
it('uses the host-owned run signal when an SDK event omits its signal', async () => {
const controller = new AbortController();
const tool = abortingTool();
@ -803,7 +833,13 @@ describe('createToolExecuteHandler', () => {
);
expect(loadTools).toHaveBeenCalledTimes(1);
expect(loadTools).toHaveBeenCalledWith(['allowed_tool'], undefined, configurable, undefined);
expect(loadTools).toHaveBeenCalledWith(
['allowed_tool'],
undefined,
configurable,
undefined,
undefined,
);
expect(JSON.stringify(jest.mocked(loadTools).mock.calls)).not.toContain(protectedName);
expect(results[0]).toEqual(
expect.objectContaining({
@ -1333,6 +1369,7 @@ describe('createToolExecuteHandler', () => {
undefined,
undefined,
callerCapabilityProjection,
undefined,
);
expect(capturedConfigs[0].toolDefs).toEqual([
{ name: 'active_programmatic_tool', allowed_callers: ['code_execution'] },
@ -1829,12 +1866,20 @@ describe('createToolExecuteHandler', () => {
configurable: {
accessibleSkillIds: skillsInScope(),
codeEnvAvailable: true,
req: { user: { id: 'user-1' } },
req: {
user: { id: 'user-1', tenantId: 'tenant-1' },
app: { locals: { codeApiUploadRegistry: createCodeApiUploadRegistry() } },
config: {
endpoints: {
agents: { codeApiUploadConcurrency: 1, codeApiMaxRetryWaitMs: 1_000 },
},
},
},
},
}));
const getSkillByName: ToolExecuteOptions['getSkillByName'] = jest.fn(async () => ({
_id: `${skillName}-id` as unknown as never,
name: skillName,
const getSkillByName: ToolExecuteOptions['getSkillByName'] = jest.fn(async (name) => ({
_id: `${name ?? skillName}-id` as unknown as never,
name: name ?? skillName,
body: 'skill body',
fileCount: 1,
version: 1,
@ -2173,6 +2218,7 @@ describe('createToolExecuteHandler', () => {
});
it('omits the unavailability note when file priming succeeds', async () => {
const controller = new AbortController();
const batchUploadCodeEnvFiles = jest.fn(async () => ({
storage_session_id: 'session-ok',
files: [
@ -2182,14 +2228,24 @@ describe('createToolExecuteHandler', () => {
}));
const handler = createPrimingSkillHandler('note-ok-skill', batchUploadCodeEnvFiles);
const [result] = await invokeHandler(handler, [
{
id: 'call_prime_ok',
name: Constants.SKILL_TOOL,
args: { skillName: 'note-ok-skill' },
},
]);
const [result] = await new Promise<ToolExecuteResult[]>((resolve, reject) => {
handler.handle('on_tool_execute', {
toolCalls: [
{
id: 'call_prime_ok',
name: Constants.SKILL_TOOL,
args: { skillName: 'note-ok-skill' },
},
],
signal: controller.signal,
resolve,
reject,
} as ToolExecuteBatchRequest);
});
expect(batchUploadCodeEnvFiles).toHaveBeenCalledWith(
expect.objectContaining({ signal: controller.signal }),
);
expect(result.status).toBe('success');
expect(result.content).not.toContain('could not be loaded');
expect(result.artifact).toEqual(
@ -2206,6 +2262,74 @@ describe('createToolExecuteHandler', () => {
);
});
it('shares one retry-wait budget across skill calls in a tool batch', async () => {
const attempts = new Map<string, number>();
const batchUploadCodeEnvFiles = jest.fn(async ({ id }: { id: string }) => {
const attempt = (attempts.get(id) ?? 0) + 1;
attempts.set(id, attempt);
if (attempt === 1) {
const error = Object.assign(new Error('Request failed with status code 429'), {
isAxiosError: true,
response: { status: 429, headers: { 'retry-after': '0' } },
});
throw error;
}
const name = id.replace(/-id$/, '');
return {
storage_session_id: `session-${name}`,
files: [{ fileId: `file-${name}`, filename: `skills/${name}/references/style.md` }],
};
});
const handler = createPrimingSkillHandler('fallback-skill', batchUploadCodeEnvFiles);
const results = await invokeHandler(handler, [
{
id: 'call_first_skill',
name: Constants.SKILL_TOOL,
args: { skillName: 'first-skill' },
},
{
id: 'call_second_skill',
name: Constants.SKILL_TOOL,
args: { skillName: 'second-skill' },
},
]);
expect(batchUploadCodeEnvFiles).toHaveBeenCalledTimes(3);
expect(results.filter((result) => result.artifact != null)).toHaveLength(1);
});
it('returns cancellation instead of a successful skill when upload recovery is aborted', async () => {
const controller = new AbortController();
const batchUploadCodeEnvFiles = jest.fn(
({ signal }: { signal?: AbortSignal }) =>
new Promise<never>((_resolve, reject) => {
signal?.addEventListener('abort', () => reject(signal.reason), { once: true });
}),
);
const handler = createPrimingSkillHandler('cancelled-skill', batchUploadCodeEnvFiles);
const resultPromise = new Promise<ToolExecuteResult[]>((resolve, reject) => {
handler.handle('on_tool_execute', {
toolCalls: [
{
id: 'call_cancelled_skill',
name: Constants.SKILL_TOOL,
args: { skillName: 'cancelled-skill' },
},
],
signal: controller.signal,
resolve,
reject,
} as ToolExecuteBatchRequest);
});
setTimeout(() => controller.abort(), 10);
const [result] = await resultPromise;
expect(result.status).toBe('error');
expect(result.content).not.toContain('Skill "cancelled-skill" loaded');
});
it("read_file pins lookup to the primed skill's _id when manually invoked this turn (no shadowing on collision)", async () => {
/* Same-name collision corner: the resolver primed a specific doc
(its `_id` is in `skillPrimedIdsByName`). If read_file used
@ -5248,6 +5372,7 @@ describe('createToolExecuteHandler', () => {
listWorkspaceFiles?: ToolExecuteOptions['listWorkspaceFiles'];
readSandboxFile?: ToolExecuteOptions['readSandboxFile'];
readSandboxImage?: ToolExecuteOptions['readSandboxImage'];
runSignal?: AbortSignal;
getSkillByName?: ToolExecuteOptions['getSkillByName'];
getAuthorSkillByName?: ToolExecuteOptions['getAuthorSkillByName'];
}) {
@ -5265,6 +5390,7 @@ describe('createToolExecuteHandler', () => {
}));
return createToolExecuteHandler({
loadTools,
runSignal: params.runSignal,
getSkillByName: params.getSkillByName,
getAuthorSkillByName: params.getAuthorSkillByName,
readWorkspaceFile: params.readWorkspaceFile,
@ -6900,6 +7026,7 @@ describe('createToolExecuteHandler', () => {
* bytes, which was the matplotlib-shape mojibake regression.
*/
it('returns a sandbox image as an image_url artifact the model can see', async () => {
const controller = new AbortController();
const readSandboxFile = jest.fn();
const readSandboxImage = jest.fn(async () => ({ base64: PNG_B64, bytes: pngBytes }));
const handler = makeReadFileHandler({
@ -6909,14 +7036,21 @@ describe('createToolExecuteHandler', () => {
readSandboxImage,
});
const [result] = await invokeHandler(handler, [
{
id: 'call_png',
name: Constants.READ_FILE,
args: { path: '/mnt/data/simple_graph.png' },
codeSessionContext: { session_id: 'sess-Z', files: [] },
} as unknown as ToolCallRequest,
]);
const [result] = await new Promise<ToolExecuteResult[]>((resolve, reject) => {
handler.handle('on_tool_execute', {
toolCalls: [
{
id: 'call_png',
name: Constants.READ_FILE,
args: { path: '/mnt/data/simple_graph.png' },
codeSessionContext: { session_id: 'sess-Z', files: [] },
} as unknown as ToolCallRequest,
],
signal: controller.signal,
resolve,
reject,
} as ToolExecuteBatchRequest);
});
expect(readSandboxFile).not.toHaveBeenCalled();
expect(readSandboxImage).toHaveBeenCalledWith(
@ -6924,6 +7058,7 @@ describe('createToolExecuteHandler', () => {
file_path: '/mnt/data/simple_graph.png',
session_id: 'sess-Z',
maxBytes: expect.any(Number),
signal: controller.signal,
}),
);
expect(result.status).toBe('success');

View file

@ -78,6 +78,14 @@ import {
SEARCH_WORKSPACE_TOOL_NAME,
isCodeSessionToolName,
} from './tools';
import {
createCodeApiRateLimitBudget,
isAbortError,
logAxiosError,
truncateMiddle,
runOutsideTracing,
getSafeErrorMetadata,
} from '~/utils';
import {
ContentFilterError,
contentFilterModelBoundBlockResponse,
@ -87,13 +95,6 @@ import {
BACKGROUND_TASK_ABORT_GRACE_MS,
BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS,
} from './backgroundCompletion';
import {
isAbortError,
logAxiosError,
truncateMiddle,
runOutsideTracing,
getSafeErrorMetadata,
} from '~/utils';
import {
WorkspaceToolHttpError,
WORKSPACE_EDIT_MAX_COUNT,
@ -237,6 +238,8 @@ export interface ToolExecuteOptions {
configurable?: Record<string, unknown>,
/** SDK-owned live caller capability projection for this agent context. */
callerCapabilityProjection?: CallerCapabilityProjectionSnapshot,
/** Effective cancellation signal for this tool-execute batch. */
signal?: AbortSignal,
) => Promise<{
loadedTools: StructuredToolInterface[];
/** Additional configurable properties to merge (e.g., userMCPAuthMap) */
@ -466,6 +469,7 @@ export interface ToolExecuteOptions {
codeApiBaseUrl?: string;
executionProfile?: CodeExecutionContext['executionProfile'];
bridgeWorkerId?: string;
signal?: AbortSignal;
}) => Promise<{
storage_session_id: string;
files: Array<{ fileId: string; filename: string }>;
@ -626,6 +630,7 @@ export interface ToolExecuteOptions {
/** In-sandbox size cap; files larger than this return `tooLarge` without transferring bytes. */
maxBytes?: number;
req?: ServerRequest;
signal?: AbortSignal;
}) => Promise<
| { base64: string; bytes: number }
/** `size`: over `maxBytes`. `round_trips`: within the byte cap, but more
@ -2119,6 +2124,7 @@ async function handleSandboxImageRead(
req?: ServerRequest,
codeExecutionContext?: CodeExecutionContext,
onSuccess?: () => void,
signal?: AbortSignal,
): Promise<ToolExecuteResult> {
const filtered = filteredBinaryFileResult(tc, req, filePath);
if (filtered != null) {
@ -2153,6 +2159,7 @@ async function handleSandboxImageRead(
session_id: ctx?.session_id,
files: ctx?.files,
maxBytes: MAX_SANDBOX_INLINE_IMAGE_BYTES,
...(signal ? { signal } : {}),
...codeExecutionRequestParams(codeExecutionContext),
...(req ? { req } : {}),
});
@ -2235,10 +2242,20 @@ async function handleSandboxFileFallback(
req?: ServerRequest,
codeExecutionContext?: CodeExecutionContext,
onSuccess?: () => void,
signal?: AbortSignal,
): Promise<ToolExecuteResult> {
const ext = lowercaseExtension(filePath);
if (SANDBOX_IMAGE_EXTENSIONS.has(ext)) {
return handleSandboxImageRead(tc, filePath, ext, options, req, codeExecutionContext, onSuccess);
return handleSandboxImageRead(
tc,
filePath,
ext,
options,
req,
codeExecutionContext,
onSuccess,
signal,
);
}
const filteredName = filteredFileNameResult(tc, req, filePath);
if (filteredName != null) {
@ -4366,6 +4383,7 @@ async function handleReadFileCall(
req,
codeExecutionContext,
onSandboxReadSuccess,
signal,
);
}
return {
@ -4402,6 +4420,7 @@ async function handleReadFileCall(
req,
codeExecutionContext,
onSandboxReadSuccess,
signal,
);
}
return {
@ -4429,6 +4448,7 @@ async function handleReadFileCall(
req,
codeExecutionContext,
onSandboxReadSuccess,
signal,
);
}
return {
@ -4498,6 +4518,7 @@ async function handleReadFileCall(
req,
codeExecutionContext,
onSandboxReadSuccess,
signal,
);
}
return {
@ -4544,6 +4565,7 @@ async function handleReadFileCall(
req,
codeExecutionContext,
onSandboxReadSuccess,
signal,
);
}
return {
@ -4869,6 +4891,8 @@ async function handleSkillToolCall(
options: ToolExecuteOptions,
agentId?: string,
req?: ServerRequest,
signal?: AbortSignal,
rateLimitBudget?: import('~/utils').CodeApiRateLimitBudget,
): Promise<ToolExecuteResult> {
const {
getSkillByName,
@ -4984,7 +5008,9 @@ async function handleSkillToolCall(
) {
let primeResult: PrimeSkillFilesResult | null = null;
try {
signal?.throwIfAborted();
const skillFiles = await listSkillFiles(skill._id);
signal?.throwIfAborted();
primeResult = await primeSkillFiles({
skill,
skillFiles,
@ -4995,6 +5021,8 @@ async function handleSkillToolCall(
checkIfActive,
updateSkillFileCodeEnvIds,
codeExecutionContext,
signal,
rateLimitBudget,
});
if (primeResult) {
/* `session_id` at the top of the artifact is the (representative)
@ -5019,6 +5047,9 @@ async function handleSkillToolCall(
};
}
} catch (error) {
if (isAbortError(error)) {
throw error;
}
if (isContentFilterError(error)) {
return error instanceof ContentFilterError
? errorResult(tc, modelBoundContentFilterErrorMessage(error.body))
@ -5303,6 +5334,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
agentId,
sourceConfigurable,
callerCapabilityProjection,
runSignal,
);
const toolMap = new Map(loadedTools.map((t) => [t.name, t]));
const loadedConfigurable = toolConfigurable as Record<string, unknown> | undefined;
@ -6186,6 +6218,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
};
};
const batchReq = mergedConfigurable?.req as ServerRequest | undefined;
const batchCodeApiRateLimitBudget = createCodeApiRateLimitBudget(
batchReq?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
);
const results: ToolExecuteResult[] = await Promise.all(
toolCalls.map(async (tc: ToolCallRequest) => {
const preloadedNameBlock = preloadedNameBlocks.get(tc);
@ -6478,6 +6514,8 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
options,
agentId,
req,
runSignal,
batchCodeApiRateLimitBudget,
);
} else if (tc.name === Constants.READ_FILE) {
handlerResult = await handleReadFileCall(

View file

@ -255,6 +255,7 @@ describe('createAgentChatCompletion - MCP permission user propagation', () => {
'agent_test',
expect.objectContaining({ requestBody: runArgs.requestBody }),
undefined,
undefined,
);
});

View file

@ -14,6 +14,7 @@ jest.mock('./run', () => ({
import { Readable } from 'stream';
import { Types } from 'mongoose';
import { createCodeApiUploadRegistry } from '~/utils';
import { primeInvokedSkills, primeInvokedSkillsForProfiles, primeSkillFiles } from './skillFiles';
import type {
PrimeInvokedSkillsDeps,
@ -23,6 +24,7 @@ import type {
const SKILL_ID = new Types.ObjectId();
const SKILL_VERSION = 7;
const uploadRegistry = createCodeApiUploadRegistry();
function makeDeps(overrides: Partial<PrimeInvokedSkillsDeps> = {}): PrimeInvokedSkillsDeps {
const listSkillFiles = jest.fn().mockResolvedValue([]);
@ -31,7 +33,10 @@ function makeDeps(overrides: Partial<PrimeInvokedSkillsDeps> = {}): PrimeInvoked
files: [],
});
return {
req: { user: { id: 'user-1' } } as PrimeInvokedSkillsDeps['req'],
req: {
user: { id: 'user-1', tenantId: 'tenant-1' },
app: { locals: { codeApiUploadRegistry: uploadRegistry } },
} as unknown as PrimeInvokedSkillsDeps['req'],
payload: [{ role: 'assistant', content: [] }],
accessibleSkillIds: [SKILL_ID],
codeEnvAvailable: true,
@ -90,6 +95,66 @@ describe('primeInvokedSkills — execute_code capability gate', () => {
expect(deps.listSkillFiles).toHaveBeenCalledWith(SKILL_ID);
});
it('shares one retry-wait budget across historical skill uploads', async () => {
const skillIds = [new Types.ObjectId(), new Types.ObjectId()];
const skillNames = ['first-skill', 'second-skill'];
mockExtract.mockReturnValue(new Set(skillNames));
const getSkillByName = jest.fn(async (name: string) => {
const index = skillNames.indexOf(name);
return {
_id: skillIds[index],
name,
body: `${name} body`,
version: 1,
fileCount: 1,
};
});
const listSkillFiles = jest.fn(async () => [
{
relativePath: 'references/style.md',
filename: 'style.md',
filepath: '/storage/style.md',
source: 's3',
bytes: 5,
},
]);
const attempts = new Map<string, number>();
const batchUploadCodeEnvFiles = jest.fn(async ({ id }: { id: string }) => {
const attempt = (attempts.get(id) ?? 0) + 1;
attempts.set(id, attempt);
if (attempt === 1) {
const error = new Error('Request failed with status code 429') as Error & {
isAxiosError: boolean;
response: { status: number; headers: Record<string, string> };
};
error.isAxiosError = true;
error.response = { status: 429, headers: { 'retry-after': '0' } };
throw error;
}
const name = skillNames[skillIds.findIndex((skillId) => skillId.toString() === id)];
return {
storage_session_id: `session-${name}`,
files: [{ fileId: `file-${name}`, filename: `skills/${name}/references/style.md` }],
};
});
const deps = makeDeps({
getSkillByName,
listSkillFiles,
getStrategyFunctions: jest.fn().mockReturnValue({
getDownloadStream: jest.fn().mockResolvedValue(Readable.from(Buffer.from('style'))),
}),
batchUploadCodeEnvFiles,
});
deps.req.config = {
endpoints: { agents: { codeApiUploadConcurrency: 1, codeApiMaxRetryWaitMs: 1_000 } },
} as never;
const result = await primeInvokedSkills(deps);
expect(batchUploadCodeEnvFiles).toHaveBeenCalledTimes(3);
expect(result.initialSessions?.get('execute_code')?.files).toHaveLength(1);
});
it('calls batchUploadCodeEnvFiles without an apiKey when files are returned', async () => {
const fileRecords = [
{
@ -527,6 +592,7 @@ describe('primeInvokedSkills — execute_code capability gate', () => {
},
deps.req,
undefined,
undefined,
);
const codeSession = result.initialSessions?.get('execute_code');
expect(codeSession?.files).toEqual([
@ -698,6 +764,82 @@ describe('primeInvokedSkillsForProfiles', () => {
).toEqual(['stateful:first', 'stateful:second']);
});
it('shares one retry-wait budget across execution profiles', async () => {
const file = {
relativePath: 'references/style.md',
filename: 'style.md',
filepath: '/storage/style.md',
source: 's3',
bytes: 5,
};
const attempts = new Map<string, number>();
const batchUploadCodeEnvFiles = jest.fn(
async ({ executionProfile }: { executionProfile?: string }) => {
const profile = executionProfile ?? 'default';
const attempt = (attempts.get(profile) ?? 0) + 1;
attempts.set(profile, attempt);
if (attempt === 1) {
if (profile === 'stateful') {
await new Promise((resolve) => setTimeout(resolve, 500));
}
const error = Object.assign(new Error('Request failed with status code 429'), {
isAxiosError: true,
response: { status: 429, headers: { 'retry-after': '0' } },
});
throw error;
}
return {
storage_session_id: `${profile}-session`,
files: [
{
fileId: `${profile}-file`,
filename: 'skills/brand-guidelines/references/style.md',
},
],
};
},
);
const { codeEnvAvailable: _codeEnvAvailable, ...baseDeps } = makeDeps({
listSkillFiles: jest.fn().mockResolvedValue([file]),
getStrategyFunctions: jest.fn().mockReturnValue({
getDownloadStream: jest.fn().mockResolvedValue(Readable.from(Buffer.from('style'))),
}),
batchUploadCodeEnvFiles,
});
baseDeps.req.config = {
endpoints: { agents: { codeApiMaxRetryWaitMs: 1_000 } },
} as never;
const result = await primeInvokedSkillsForProfiles({
...baseDeps,
executionProfiles: [
{
codeExecutionContext: {
baseUrl: 'https://default.example.com/v1',
codeSessionKey: 'execute_code',
executionProfile: 'default',
statefulSessions: false,
},
codeSessionKeys: ['execute_code'],
},
{
codeExecutionContext: {
baseUrl: 'https://stateful.example.com/v1',
codeSessionKey: 'execute_code:stateful',
executionProfile: 'stateful',
executionRouteKey: 'stateful:one',
statefulSessions: true,
},
codeSessionKeys: ['execute_code:stateful'],
},
],
});
expect(batchUploadCodeEnvFiles).toHaveBeenCalledTimes(3);
expect(result.initialSessions?.has('execute_code')).toBe(true);
expect(result.initialSessions?.has('execute_code:stateful')).toBe(false);
});
it('keeps a successful profile Skill body and identity after another profile lookup fails', async () => {
const {
codeEnvAvailable: _codeEnvAvailable,
@ -768,7 +910,10 @@ function makeSkillFilesDeps(overrides: Partial<PrimeSkillFilesParams> = {}): Pri
version: SKILL_VERSION,
},
skillFiles: [],
req: { user: { id: 'user-1' } } as PrimeSkillFilesParams['req'],
req: {
user: { id: 'user-1', tenantId: 'tenant-1' },
app: { locals: { codeApiUploadRegistry: uploadRegistry } },
} as unknown as PrimeSkillFilesParams['req'],
getStrategyFunctions: jest.fn().mockReturnValue({
getDownloadStream: jest.fn().mockResolvedValue(Readable.from(Buffer.from(''))),
}),
@ -861,10 +1006,15 @@ describe('primeSkillFiles — resource identity propagation', () => {
const result = await primeSkillFiles(deps);
expect(batchUploadCodeEnvFiles).not.toHaveBeenCalled();
expect(deps.getSessionInfo).toHaveBeenCalledWith(cachedRef, deps.req, {
baseUrl: 'https://stateful-code.example.com',
executionProfile: 'stateful',
});
expect(deps.getSessionInfo).toHaveBeenCalledWith(
cachedRef,
deps.req,
{
baseUrl: 'https://stateful-code.example.com',
executionProfile: 'stateful',
},
undefined,
);
expect(result?.files).toEqual([
{
id: 'file-cached',
@ -969,7 +1119,8 @@ describe('primeSkillFiles — resource identity propagation', () => {
const batchUploadCodeEnvFiles = jest.fn();
const deps = makeSkillFilesDeps({
req: {
user: { id: 'user-1' },
user: { id: 'user-1', tenantId: 'tenant-1' },
app: { locals: { codeApiUploadRegistry: uploadRegistry } },
config: {
filters: {
skills: {
@ -980,7 +1131,7 @@ describe('primeSkillFiles — resource identity propagation', () => {
},
},
},
} as PrimeSkillFilesParams['req'],
} as unknown as PrimeSkillFilesParams['req'],
skillFiles: [
{
relativePath: 'references/style.md',
@ -1019,7 +1170,8 @@ describe('primeSkillFiles — resource identity propagation', () => {
const batchUploadCodeEnvFiles = jest.fn();
const deps = makeSkillFilesDeps({
req: {
user: { id: 'user-1' },
user: { id: 'user-1', tenantId: 'tenant-1' },
app: { locals: { codeApiUploadRegistry: uploadRegistry } },
config: {
filters: {
files: {
@ -1029,7 +1181,7 @@ describe('primeSkillFiles — resource identity propagation', () => {
},
},
},
} as PrimeSkillFilesParams['req'],
} as unknown as PrimeSkillFilesParams['req'],
skillFiles: [
{
relativePath: 'references/style.md',
@ -1077,7 +1229,7 @@ describe('primeSkillFiles — resource identity propagation', () => {
},
},
},
} as PrimeSkillFilesParams['req'],
} as unknown as PrimeSkillFilesParams['req'],
skillFiles: [
{
relativePath: 'references/sk-private-name.md',
@ -1123,7 +1275,7 @@ describe('primeSkillFiles — resource identity propagation', () => {
},
},
},
} as PrimeSkillFilesParams['req'],
} as unknown as PrimeSkillFilesParams['req'],
skillFiles: [
{
relativePath: 'references/style.md',
@ -1374,7 +1526,7 @@ describe('primeSkillFiles — resource identity propagation', () => {
},
},
},
} as PrimeSkillFilesParams['req'],
} as unknown as PrimeSkillFilesParams['req'],
skillFiles: [
{
relativePath: 'references/unavailable.txt',
@ -1505,7 +1657,8 @@ describe('primeSkillFiles — resource identity propagation', () => {
});
const deps = makeSkillFilesDeps({
req: {
user: { id: 'user-1' },
user: { id: 'user-1', tenantId: 'tenant-1' },
app: { locals: { codeApiUploadRegistry: uploadRegistry } },
config: {
filters: {
files: {
@ -1517,7 +1670,7 @@ describe('primeSkillFiles — resource identity propagation', () => {
},
},
},
} as PrimeSkillFilesParams['req'],
} as unknown as PrimeSkillFilesParams['req'],
skillFiles: [
{
relativePath: 'references/archive.bin',
@ -1679,7 +1832,72 @@ describe('primeSkillFiles — upload rate-limit resilience', () => {
expect(result).toBeNull();
});
it('bounds concurrent batch uploads to 3 process-wide slots', async () => {
it('does not upload a partial skill bundle when stream acquisition is canceled', async () => {
const controller = new AbortController();
const batchUploadCodeEnvFiles = jest.fn();
const acquiredStream = Readable.from(Buffer.from('style'));
const destroy = jest.spyOn(acquiredStream, 'destroy');
const getDownloadStream = jest.fn(async () => {
controller.abort();
return acquiredStream;
});
await expect(
primeSkillFiles(
makeSkillFilesDeps({
signal: controller.signal,
skillFiles: [styleFileRecord()],
batchUploadCodeEnvFiles,
getStrategyFunctions: jest.fn().mockReturnValue({ getDownloadStream }),
}),
),
).rejects.toMatchObject({ name: 'AbortError' });
expect(getDownloadStream).toHaveBeenCalledWith(expect.anything(), expect.any(String), {
signal: controller.signal,
});
expect(destroy).toHaveBeenCalled();
expect(batchUploadCodeEnvFiles).not.toHaveBeenCalled();
});
it('does not return a cache hit when cancellation arrives during its liveness check', async () => {
const controller = new AbortController();
const batchUploadCodeEnvFiles = jest.fn();
const getSessionInfo = jest.fn(async () => {
controller.abort();
return '2026-05-06T00:00:00Z';
});
const cachedFile = {
...styleFileRecord(),
codeEnvRef: {
kind: 'skill' as const,
id: SKILL_ID.toString(),
storage_session_id: 'session-cached',
file_id: 'file-cached',
version: SKILL_VERSION,
},
};
await expect(
primeSkillFiles(
makeSkillFilesDeps({
signal: controller.signal,
skillFiles: [cachedFile],
batchUploadCodeEnvFiles,
getSessionInfo,
checkIfActive: jest.fn().mockReturnValue(true),
}),
),
).rejects.toMatchObject({ name: 'AbortError' });
expect(getSessionInfo).toHaveBeenCalledWith(
expect.any(Object),
expect.any(Object),
undefined,
controller.signal,
);
expect(batchUploadCodeEnvFiles).not.toHaveBeenCalled();
});
it('bounds concurrent batch uploads to 3 application-scoped slots', async () => {
const gates = Array.from({ length: 5 }, () => deferred<ReturnType<typeof uploadResult>>());
let uploadIndex = 0;
const batchUploadCodeEnvFiles = jest

View file

@ -18,7 +18,14 @@ import {
inspectContent,
UninspectableFileError,
} from '~/protection';
import { createConcurrencyLimiter, getCodeApiRetryAfterMs, getSafeErrorMetadata } from '~/utils';
import {
createCodeApiRateLimitBudget,
getCodeApiUploadOptions,
getSafeErrorMetadata,
isAbortError,
type CodeApiRateLimitBudget,
withCodeApiUploadRecovery,
} from '~/utils';
import { seedCodeFilesIntoSessions, type CodeExecutionProfileRoute } from './codeFilesSession';
import { ContentFilterError, isContentFilterError } from '~/middleware/contentFilter';
import { getCodeExecutionRouteKey, type CodeExecutionContext } from './execution';
@ -55,8 +62,16 @@ export interface PrimeSkillFilesParams {
};
skillFiles: SkillFileRecord[];
req: ServerRequest;
/** Optional operation-wide wait allowance shared by sibling skill primes. */
rateLimitBudget?: CodeApiRateLimitBudget;
/** Effective foreground cancellation signal for acquisition, waits, and transport. */
signal?: AbortSignal;
getStrategyFunctions: (source: string) => {
getDownloadStream?: (req: ServerRequest, filepath: string) => Promise<NodeJS.ReadableStream>;
getDownloadStream?: (
req: ServerRequest,
filepath: string,
options?: { signal?: AbortSignal },
) => Promise<NodeJS.ReadableStream>;
[key: string]: unknown;
};
batchUploadCodeEnvFiles: (params: {
@ -76,6 +91,7 @@ export interface PrimeSkillFilesParams {
codeApiBaseUrl?: string;
executionProfile?: CodeExecutionContext['executionProfile'];
bridgeWorkerId?: string;
signal?: AbortSignal;
}) => Promise<{
storage_session_id: string;
files: Array<{ fileId: string; filename: string }>;
@ -89,6 +105,7 @@ export interface PrimeSkillFilesParams {
executionProfile?: CodeExecutionContext['executionProfile'];
bridgeWorkerId?: string;
},
signal?: AbortSignal,
) => Promise<string | null>;
/** Trusted Code API route selected for the executing agent. */
codeExecutionContext?: Pick<
@ -127,16 +144,21 @@ export interface PrimeSkillFilesResult {
}>;
}
/** Cap on concurrent skill batch uploads per process. Bounds burst pressure
* on codeapi's per-user upload limiter (default 30 requests / 5 min). */
const SKILL_UPLOAD_CONCURRENCY = 3;
/** Retry a 429'd upload only when the server's Retry-After fits under this
* cap; a longer wait would stall a live chat turn worse than degrading. */
const MAX_RETRY_AFTER_MS = 15_000;
const uploadSlots = createConcurrencyLimiter(SKILL_UPLOAD_CONCURRENCY);
const inflightPrimes = new Map<string, Promise<PrimeSkillFilesResult | null>>();
const signalIds = new WeakMap<AbortSignal, number>();
let nextSignalId = 1;
function getSignalScope(signal?: AbortSignal): string {
if (!signal) {
return 'unscoped';
}
let id = signalIds.get(signal);
if (id == null) {
id = nextSignalId++;
signalIds.set(signal, id);
}
return String(id);
}
type SkillUploadFiles = Array<{ stream: NodeJS.ReadableStream; filename: string }>;
type SkillCodeEnvRef = Extract<CodeEnvRef, { kind: 'skill' }>;
@ -148,19 +170,16 @@ function isCurrentSkillRef(
return ref?.kind === 'skill' && ref.version === skillVersion;
}
/** Single retry on 429, honoring Retry-After up to MAX_RETRY_AFTER_MS.
* Runs inside an upload slot so the wait also brakes queued uploads. */
async function retryOn429<T>(attempt: () => Promise<T>, label: string): Promise<T> {
function destroySkillUploadStream(stream: NodeJS.ReadableStream): void {
try {
return await attempt();
} catch (error) {
const retryAfterMs = getCodeApiRetryAfterMs(error);
if (retryAfterMs == null || retryAfterMs > MAX_RETRY_AFTER_MS) {
throw error;
if (
'destroy' in stream &&
typeof (stream as NodeJS.ReadableStream & { destroy?: () => void }).destroy === 'function'
) {
(stream as NodeJS.ReadableStream & { destroy: () => void }).destroy();
}
logger.warn(`[primeSkillFiles] Rate-limited priming ${label}; retrying in ${retryAfterMs}ms`);
await new Promise((resolve) => setTimeout(resolve, retryAfterMs));
return attempt();
} catch {
/* Preserve the run cancellation even if a provider stream rejects cleanup. */
}
}
@ -171,7 +190,8 @@ async function collectSkillUploadFiles(
params: PrimeSkillFilesParams,
inspectedBuffers: ReadonlyMap<SkillFileRecord, Buffer>,
): Promise<SkillUploadFiles> {
const { skill, skillFiles, req, getStrategyFunctions } = params;
const { skill, skillFiles, req, getStrategyFunctions, signal } = params;
signal?.throwIfAborted();
const filesToUpload: SkillUploadFiles = [];
// SKILL.md from the skill body
@ -184,6 +204,7 @@ async function collectSkillUploadFiles(
// Bundled files from storage (parallel stream acquisition)
const streamResults = await Promise.allSettled(
skillFiles.map(async (file) => {
signal?.throwIfAborted();
const inspected = inspectedBuffers.get(file);
if (inspected != null) {
return {
@ -198,10 +219,28 @@ async function collectSkillUploadFiles(
);
return null;
}
const stream = await strategy.getDownloadStream(req, resolveDownloadPath(file));
const stream = await strategy.getDownloadStream(req, resolveDownloadPath(file), { signal });
if (signal?.aborted) {
destroySkillUploadStream(stream);
signal.throwIfAborted();
}
return { stream, filename: `${SKILL_FILE_PREFIX}${skill.name}/${file.relativePath}` };
}),
);
/* Do not let allSettled turn foreground cancellation into a skipped bundle
* member. A partial skill upload can look successful while leaving required
* files unavailable to the sandbox. */
if (signal?.aborted) {
for (const file of filesToUpload) {
destroySkillUploadStream(file.stream);
}
for (const result of streamResults) {
if (result.status === 'fulfilled' && result.value) {
destroySkillUploadStream(result.value.stream);
}
}
signal.throwIfAborted();
}
for (const result of streamResults) {
if (result.status === 'fulfilled' && result.value) {
filesToUpload.push(result.value);
@ -308,10 +347,14 @@ function throwIfStoredSkillFileMustBeInspectable(req: ServerRequest): void {
}
}
async function bufferSkillFileStream(stream: NodeJS.ReadableStream): Promise<Buffer | null> {
async function bufferSkillFileStream(
stream: NodeJS.ReadableStream,
signal?: AbortSignal,
): Promise<Buffer | null> {
const chunks: Buffer[] = [];
let bytes = 0;
for await (const chunk of stream as AsyncIterable<Uint8Array | string>) {
signal?.throwIfAborted();
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
bytes += buffer.length;
if (bytes > MAX_INSPECTABLE_SKILL_FILE_BYTES) {
@ -319,6 +362,7 @@ async function bufferSkillFileStream(stream: NodeJS.ReadableStream): Promise<Buf
}
chunks.push(buffer);
}
signal?.throwIfAborted();
return Buffer.concat(chunks);
}
@ -333,8 +377,8 @@ async function bufferSkillFileStream(stream: NodeJS.ReadableStream): Promise<Buf
* documents for future freshness checks.
*
* Rate-limit resilience: concurrent primes of the same (skill, version)
* share one flight, uploads are bounded process-wide, and a 429 retries
* once per the server's Retry-After.
* share one flight, uploads are bounded per Code API route and principal,
* and 429 responses retry within a capped wait budget.
*/
export async function primeSkillFiles(
params: PrimeSkillFilesParams,
@ -348,7 +392,9 @@ export async function primeSkillFiles(
const executionRouteKey = params.codeExecutionContext
? getCodeExecutionRouteKey(params.codeExecutionContext)
: 'default';
const flightKey = `${executionRouteKey}:${params.skill._id}:v:${params.skill.version}`;
/* A flight may share cancellation only with callers from the same run.
* Cross-run sharing would let one user's Stop abort another live request. */
const flightKey = `${executionRouteKey}:${params.skill._id}:v:${params.skill.version}:run:${getSignalScope(params.signal)}`;
const inflight = inflightPrimes.get(flightKey);
if (inflight) {
return inflight;
@ -373,7 +419,10 @@ async function executePrimeSkillFiles(
checkIfActive,
updateSkillFileCodeEnvIds,
codeExecutionContext,
rateLimitBudget,
signal,
} = params;
signal?.throwIfAborted();
const executionProfile = codeExecutionContext?.executionProfile ?? 'default';
const executionRouteKey = codeExecutionContext
? getCodeExecutionRouteKey(codeExecutionContext)
@ -386,12 +435,14 @@ async function executePrimeSkillFiles(
if (inspectStoredMetadata) {
for (const file of skillFiles) {
signal?.throwIfAborted();
assertStoredSkillFileNameAllowed(file, req);
}
}
if (inspectBundledFileContent) {
for (const file of skillFiles) {
signal?.throwIfAborted();
if (file.bytes > MAX_INSPECTABLE_SKILL_FILE_BYTES) {
throwIfStoredSkillFileMustBeInspectable(req);
continue;
@ -403,8 +454,10 @@ async function executePrimeSkillFiles(
logger.warn('[primeSkillFiles] No download stream for stored skill file');
continue;
}
const sourceStream = await strategy.getDownloadStream(req, resolveDownloadPath(file));
const buffer = await bufferSkillFileStream(sourceStream);
const sourceStream = await strategy.getDownloadStream(req, resolveDownloadPath(file), {
signal,
});
const buffer = await bufferSkillFileStream(sourceStream, signal);
if (buffer == null) {
throwIfStoredSkillFileMustBeInspectable(req);
continue;
@ -412,6 +465,9 @@ async function executePrimeSkillFiles(
assertStoredSkillFileAllowed(file, buffer, req);
inspectedBuffers.set(file, buffer);
} catch (error) {
if (isAbortError(error)) {
throw error;
}
if (isContentFilterError(error)) {
throw error;
}
@ -445,10 +501,11 @@ async function executePrimeSkillFiles(
try {
const checkResults = await Promise.all(
Array.from(refsBySession.values()).map(async (ref) => {
const lastModified = await getSessionInfo(ref, req, codeExecutionContext);
const lastModified = await getSessionInfo(ref, req, codeExecutionContext, signal);
return !!(lastModified && checkIfActive(lastModified));
}),
);
signal?.throwIfAborted();
const allActive = checkResults.every(Boolean);
if (allActive) {
@ -480,7 +537,10 @@ async function executePrimeSkillFiles(
return { storage_session_id: files[0].storage_session_id, files };
}
}
} catch {
} catch (error) {
if (isAbortError(error)) {
throw error;
}
// Session check failed — fall through to re-upload
}
}
@ -488,12 +548,20 @@ async function executePrimeSkillFiles(
const entityId = skill._id.toString();
try {
/* Streams open inside the slot (not while queued) and inside the retry
* closure (a failed attempt consumes them). The slot bounds concurrent
* uploads process-wide across both prime call sites. */
const uploaded = await uploadSlots(() =>
retryOn429(async () => {
const filesToUpload = await collectSkillUploadFiles(params, inspectedBuffers);
const uploadOptions = getCodeApiUploadOptions(req, executionRouteKey);
const uploaded = await withCodeApiUploadRecovery({
registry: req.app.locals.codeApiUploadRegistry,
scope: uploadOptions.scope,
concurrency: uploadOptions.concurrency,
label: `priming skill "${skill.name}"`,
budget: rateLimitBudget ?? createCodeApiRateLimitBudget(uploadOptions.retryWaitMs),
signal,
onWait: (waitMs) =>
logger.warn(
`[primeSkillFiles] Rate-limited priming skill "${skill.name}"; retrying in ${waitMs}ms`,
),
openSource: () => collectSkillUploadFiles(params, inspectedBuffers),
upload: async (filesToUpload) => {
if (filesToUpload.length === 0) {
return null;
}
@ -503,7 +571,7 @@ async function executePrimeSkillFiles(
/* Resource identity for codeapi's sessionKey: skill files share
* cross-user-within-tenant under `<tenant>:skill:<id>:v:<version>`.
* Bumping `skill.version` on edit naturally invalidates the prior
* cache entry under the new sessionKey. */
* cache entry under the new version's sessionKey. */
kind: 'skill',
id: entityId,
version: skill.version,
@ -518,10 +586,11 @@ async function executePrimeSkillFiles(
codeApiBaseUrl: codeExecutionContext?.baseUrl,
executionProfile: codeExecutionContext?.executionProfile,
bridgeWorkerId: codeExecutionContext?.bridgeWorkerId,
signal,
});
return { filesToUpload, result };
}, `skill "${skill.name}"`),
);
},
});
if (uploaded == null) {
return null;
}
@ -608,6 +677,9 @@ async function executePrimeSkillFiles(
return { storage_session_id: result.storage_session_id, files };
} catch (error) {
if (isAbortError(error)) {
throw error;
}
logger.error('[primeSkillFiles] Batch upload failed', getSafeErrorMetadata(error));
return null;
}
@ -615,6 +687,8 @@ async function executePrimeSkillFiles(
export interface PrimeInvokedSkillsDeps {
req: ServerRequest;
/** Effective cancellation signal for the historical priming operation. */
signal?: AbortSignal;
/** Raw message payload (before formatAgentMessages). Used to extract invoked skill names. */
payload?: Array<Partial<{ role: string; content: unknown }>>;
/** Explicit durable names used by a validated event-actor preflight. */
@ -642,6 +716,7 @@ export interface PrimeInvokedSkillsDeps {
checkIfActive?: PrimeSkillFilesParams['checkIfActive'];
updateSkillFileCodeEnvIds?: PrimeSkillFilesParams['updateSkillFileCodeEnvIds'];
codeExecutionContext?: PrimeSkillFilesParams['codeExecutionContext'];
rateLimitBudget?: CodeApiRateLimitBudget;
}
export interface PrimeInvokedSkillsResult {
@ -781,13 +856,19 @@ export async function primeInvokedSkills(
ref,
deps.req,
deps.codeExecutionContext,
deps.signal,
);
deps.signal?.throwIfAborted();
return !!(lastModified && deps.checkIfActive?.(lastModified));
} catch {
} catch (error) {
if (isAbortError(error)) {
throw error;
}
return false;
}
}),
);
deps.signal?.throwIfAborted();
const allActive = checkResults.every(Boolean);
if (allActive) {
@ -844,6 +925,9 @@ export async function primeInvokedSkills(
kind: 'skill';
version: number;
}> = [];
const uploadOptions = getCodeApiUploadOptions(deps.req, executionRouteKey);
const rateLimitBudget =
deps.rateLimitBudget ?? createCodeApiRateLimitBudget(uploadOptions.retryWaitMs);
const primeResults = await Promise.allSettled(
fileListResults.map(async ({ skill, files }) => {
const result = await primeSkillFiles({
@ -856,6 +940,8 @@ export async function primeInvokedSkills(
checkIfActive: deps.checkIfActive,
updateSkillFileCodeEnvIds: deps.updateSkillFileCodeEnvIds,
codeExecutionContext: deps.codeExecutionContext,
rateLimitBudget,
signal: deps.signal,
});
return { skill, result };
}),
@ -876,7 +962,7 @@ export async function primeInvokedSkills(
});
}
} else if (r.status === 'rejected') {
if (isContentFilterError(r.reason)) {
if (isContentFilterError(r.reason) || isAbortError(r.reason)) {
throw r.reason;
}
logger.warn(
@ -921,6 +1007,10 @@ export async function primeInvokedSkillsForProfiles(
return primeInvokedSkills({ ...deps, codeEnvAvailable: false });
}
const rateLimitBudget = createCodeApiRateLimitBudget(
deps.req.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
);
const profileResults = await Promise.all(
deps.executionProfiles.map(async (profile) => ({
profile,
@ -929,6 +1019,8 @@ export async function primeInvokedSkillsForProfiles(
codeEnvAvailable: true,
codeExecutionContext: profile.codeExecutionContext,
updateSkillFileCodeEnvIds: deps.updateSkillFileCodeEnvIds,
rateLimitBudget,
signal: deps.signal,
}),
})),
);

View file

@ -353,6 +353,83 @@ describe('createProvisionFilesCallback', () => {
expect(provisionToCodeEnv).not.toHaveBeenCalled();
});
it('preserves cancellation raised during code provisioning and skips search work', async () => {
const controller = new AbortController();
const codeImpl = jest.fn(async () => {
controller.abort();
controller.signal.throwIfAborted();
});
const vectorImpl = jest.fn();
const { provisionFiles } = buildHarness({
contexts: [
['agent-a', { provisionState: state([makeFile()], [makeFile({ file_id: 'search-1' })]) }],
],
codeImpl,
vectorImpl,
});
await expect(
provisionFiles([Constants.EXECUTE_CODE, 'file_search'], 'agent-a', controller.signal),
).rejects.toMatchObject({ name: 'AbortError' });
expect(vectorImpl).not.toHaveBeenCalled();
});
it('preserves cancellation raised during vector provisioning', async () => {
const controller = new AbortController();
const vectorImpl = jest.fn(async () => {
controller.abort();
controller.signal.throwIfAborted();
});
const { provisionFiles } = buildHarness({
contexts: [['agent-a', { provisionState: state([], [makeFile()]) }]],
vectorImpl,
});
await expect(
provisionFiles(['file_search'], 'agent-a', controller.signal),
).rejects.toMatchObject({ name: 'AbortError' });
});
it('preserves a homogeneous Code API rate-limit failure from lazy provisioning', async () => {
const rateLimit = Object.assign(new Error('rate limited'), {
name: 'CodeApiRateLimitError',
code: 'CODE_API_RATE_LIMITED',
status: 429,
statusCode: 429,
retryAfterMs: 30_000,
});
const { provisionFiles } = buildHarness({
contexts: [['agent-a', { provisionState: state([makeFile()], []) }]],
codeImpl: jest.fn().mockRejectedValue(rateLimit),
});
await expect(provisionFiles([Constants.EXECUTE_CODE], 'agent-a')).rejects.toBe(rateLimit);
});
it('retains every failure when lazy code provisioning has mixed causes', async () => {
const first = new Error('storage unavailable');
const second = new Error('code api unavailable');
const { provisionFiles } = buildHarness({
contexts: [
[
'agent-a',
{
provisionState: state(
[makeFile({ file_id: 'first' }), makeFile({ file_id: 'second' })],
[],
),
},
],
],
codeImpl: jest.fn().mockRejectedValueOnce(first).mockRejectedValueOnce(second),
});
await expect(provisionFiles([Constants.EXECUTE_CODE], 'agent-a')).rejects.toMatchObject({
cause: first,
errors: [first, second],
});
});
it('shares embedding work across agents queueing the same search file', async () => {
const shared = makeFile();
const { provisionFiles, provisionToVectorDB, addEmbeddedEntity } = buildHarness({

View file

@ -7,6 +7,7 @@ import type { CodeEnvRefUpdate, CodeExecutionRoute, ProvisionService } from './s
import type { ProvisionState } from '~/agents/resources';
import type { ServerRequest } from '~/types';
import { claimCodeDestination, createCodeDestinationSet } from '~/files/code/destinations';
import { createCodeApiRateLimitBudget, isCodeApiRateLimitError } from '~/utils';
import { isCodeFileToolName } from '~/agents/tools';
/** Deferred database write produced by a successful provisioning call. */
@ -238,6 +239,7 @@ export function createProvisionFilesCallback({
/** Files whose provisioning rejected this turn; kept queued so a transient
* outage can retry next turn instead of being silently dropped. */
const failedCodeFiles: TFile[] = [];
const codeFailureReasons: unknown[] = [];
const failedVectorFiles: TFile[] = [];
/* The graph seeded each tool call's code-session context from the sessions that
* existed at run start, which predate this upload. Returning the refs lets the
@ -248,6 +250,11 @@ export function createProvisionFilesCallback({
: [];
if (needsCode && provisionState.codeEnvFiles.length > 0) {
const queuedCodeFiles = provisionState.codeEnvFiles;
/** Every file in this tool-load batch shares one wait allowance. This
* prevents a large recovery set from multiplying the live-turn delay. */
const codeApiRateLimitBudget = createCodeApiRateLimitBudget(
req.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
);
const destinations = createCodeDestinationSet();
const existingCodeFiles = (
ctx.tool_resources as Record<string, { files?: TFile[] } | undefined>
@ -287,6 +294,7 @@ export function createProvisionFilesCallback({
route: ctx.codeExecutionContext,
sandboxFilename,
signal,
rateLimitBudget: codeApiRateLimitBudget,
});
signal?.throwIfAborted();
/* primeCodeFiles re-reads the database and skips files without a stored
@ -334,10 +342,15 @@ export function createProvisionFilesCallback({
}
}),
);
/* allSettled keeps independent file failures retryable, but cancellation belongs
* to the whole tool run. Preserve it before translating rejections into queued
* files or starting search provisioning. */
signal?.throwIfAborted();
results.forEach((result, index) => {
if (result.status === 'rejected') {
logger.error('[provisionFiles] Code env provisioning failed', result.reason);
failedCodeFiles.push(queuedCodeFiles[index]);
codeFailureReasons.push(result.reason);
}
});
provisionState.codeEnvFiles = failedCodeFiles;
@ -403,6 +416,7 @@ export function createProvisionFilesCallback({
throw new Error(`Vector store did not embed "${file.filename}" (${file.file_id})`);
}),
);
signal?.throwIfAborted();
results.forEach((result, index) => {
if (result.status === 'rejected') {
logger.error('[provisionFiles] Vector DB provisioning failed', result.reason);
@ -419,8 +433,15 @@ export function createProvisionFilesCallback({
/* Provisioning failed outright, so the sandbox or vector store does not have the
* attachment; running the tool anyway would answer from missing input. */
if (failedCodeFiles.length > 0) {
throw new Error(
`Failed to provision ${failedCodeFiles.length} file(s) to the code environment; aborting tool execution rather than running without them`,
if (codeFailureReasons.length > 0 && codeFailureReasons.every(isCodeApiRateLimitError)) {
throw codeFailureReasons[0];
}
throw Object.assign(
new Error(
`Failed to provision ${failedCodeFiles.length} file(s) to the code environment; aborting tool execution rather than running without them`,
codeFailureReasons[0] !== undefined ? { cause: codeFailureReasons[0] } : undefined,
),
{ errors: codeFailureReasons },
);
}
if (failedVectorFiles.length > 0) {

View file

@ -1,4 +1,5 @@
import { Readable } from 'node:stream';
import { AxiosError, AxiosHeaders } from 'axios';
import type { TFile } from 'librechat-data-provider';
import type { ServerRequest } from '~/types';
@ -20,9 +21,13 @@ jest.mock('@librechat/agents', () => ({
getCodeBaseURL: () => 'http://code.test/v1',
}));
import { createCodeApiUploadRegistry } from '~/utils';
import { createProvisionService } from './service';
const req = { user: { id: 'u1' } } as unknown as ServerRequest;
const req = {
user: { id: 'u1' },
app: { locals: { codeApiUploadRegistry: createCodeApiUploadRegistry() } },
} as unknown as ServerRequest;
const makeFile = (overrides: Partial<TFile> = {}): TFile =>
({
@ -73,6 +78,48 @@ describe('createProvisionService', () => {
});
describe('provisionToCodeEnv', () => {
it('waits out Code API throttling and reopens the upload stream', async () => {
const rateLimit = new AxiosError('Request failed', 'ERR_BAD_REQUEST');
rateLimit.response = {
status: 429,
statusText: 'Too Many Requests',
headers: new AxiosHeaders({ 'retry-after': '0' }),
config: { headers: new AxiosHeaders() },
data: {},
};
const streams = [Readable.from('first'), Readable.from('second')];
const getDownloadStream = jest
.fn()
.mockResolvedValueOnce(streams[0])
.mockResolvedValueOnce(streams[1]);
const uploadCodeEnvFile = jest
.fn()
.mockRejectedValueOnce(rateLimit)
.mockResolvedValueOnce({ storage_session_id: 's1', file_id: 'remote-1' });
const { service } = buildService({
getStrategyFunctions: (source: string) =>
source === 'execute_code'
? { handleFileUpload: uploadCodeEnvFile }
: { getDownloadStream },
});
await expect(
service.provisionToCodeEnv({
req,
file: makeFile(),
rateLimitBudget: {
limitMs: 2_000,
waitedMs: 0,
activeWaitEnds: new Set(),
},
}),
).resolves.toMatchObject({ referenceSet: { codeEnvRef: { file_id: 'remote-1' } } });
expect(uploadCodeEnvFile).toHaveBeenCalledTimes(2);
expect(getDownloadStream).toHaveBeenCalledTimes(2);
expect(uploadCodeEnvFile.mock.calls.map(([args]) => args.stream)).toEqual(streams);
});
it('renames a converted image to match its stored MIME type', async () => {
const { service, uploadCodeEnvFile } = buildService();
@ -112,9 +159,14 @@ describe('createProvisionService', () => {
it('cancels storage acquisition and destroys a stream returned after cancellation', async () => {
const controller = new AbortController();
let resolveDownload!: (stream: Readable) => void;
let markDownloadStarted!: () => void;
const downloadStarted = new Promise<void>((resolve) => {
markDownloadStarted = resolve;
});
const getDownloadStream = jest.fn(
() =>
new Promise<Readable>((resolve) => {
markDownloadStarted();
resolveDownload = resolve;
}),
);
@ -128,6 +180,7 @@ describe('createProvisionService', () => {
signal: controller.signal,
});
await downloadStarted;
controller.abort();
const stream = new Readable({ read() {} });
const destroy = jest.spyOn(stream, 'destroy');

View file

@ -14,12 +14,16 @@ import {
import type { CodeEnvRef, CodeEnvRefMap, TFile } from 'librechat-data-provider';
import type { Readable } from 'node:stream';
import type { CodeEnvIdentity } from '~/files/code/identity';
import type { CodeApiRateLimitBudget } from '~/utils';
import type { ServerRequest } from '~/types';
import {
logAxiosError,
createAxiosInstance,
createCodeApiRateLimitBudget,
codeServerHttpAgent,
codeServerHttpsAgent,
getCodeApiUploadOptions,
withCodeApiUploadRecovery,
} from '~/utils';
import { buildCodeEnvIdentityParams } from '~/files/code/identity';
import { getCodeApiAuthHeaders } from '~/auth/codeapi';
@ -89,6 +93,7 @@ export interface ProvisionService {
route?: CodeExecutionRoute;
sandboxFilename?: string;
signal?: AbortSignal;
rateLimitBudget?: CodeApiRateLimitBudget;
}) => Promise<{
referenceSet: CodeEnvReferenceSetResult;
refUpdate: CodeEnvRefUpdate;
@ -237,6 +242,7 @@ export function createProvisionService({
route,
sandboxFilename: requestedSandboxFilename,
signal,
rateLimitBudget,
}: {
req: ServerRequest;
file: TFile;
@ -244,20 +250,13 @@ export function createProvisionService({
route?: CodeExecutionRoute;
sandboxFilename?: string;
signal?: AbortSignal;
rateLimitBudget?: CodeApiRateLimitBudget;
}) {
signal?.throwIfAborted();
const { handleFileUpload: uploadCodeEnvFile } = getStrategyFunctions(FileSources.execute_code);
if (!uploadCodeEnvFile) {
throw new Error('Code environment upload strategy is unavailable');
}
const stream = await getStorageStream(file, req, signal);
signal?.throwIfAborted();
if (!stream) {
throw new Error(
`Cannot provision file "${file.filename}" to code env: storage source "${file.source}" does not support download streams`,
);
}
const kind: 'agent' | 'user' = entity_id ? 'agent' : 'user';
const id = entity_id ?? (req.user?.id as string);
@ -268,15 +267,42 @@ export function createProvisionService({
const executionProfile = route?.executionProfile ?? 'default';
const sandboxFilename =
requestedSandboxFilename ?? resolveSandboxFilename(file.filename, file.type);
const uploaded = await uploadCodeEnvFile({
const uploadOptions = getCodeApiUploadOptions(
req,
stream,
filename: sandboxFilename,
kind,
id,
...(route?.baseUrl ? { codeApiBaseUrl: route.baseUrl } : {}),
executionProfile,
route?.executionRouteKey ?? route?.baseUrl ?? executionProfile,
);
const uploaded = await withCodeApiUploadRecovery({
registry: req.app.locals.codeApiUploadRegistry,
scope: uploadOptions.scope,
concurrency: uploadOptions.concurrency,
label: `uploading "${file.filename}" to the code environment`,
budget: rateLimitBudget ?? createCodeApiRateLimitBudget(uploadOptions.retryWaitMs),
signal,
onWait: (waitMs) =>
logger.warn(
`[provisionToCodeEnv] Rate-limited uploading file ${file.file_id}; retrying in ${waitMs}ms`,
),
openSource: async () => {
const stream = await getStorageStream(file, req, signal);
signal?.throwIfAborted();
if (!stream) {
throw new Error(
`Cannot provision file "${file.filename}" to code env: storage source "${file.source}" does not support download streams`,
);
}
return stream;
},
upload: (stream) =>
uploadCodeEnvFile({
req,
stream,
filename: sandboxFilename,
kind,
id,
...(route?.baseUrl ? { codeApiBaseUrl: route.baseUrl } : {}),
executionProfile,
signal,
}),
});
signal?.throwIfAborted();

View file

@ -87,6 +87,19 @@ export const logAxiosError = ({
return logMessage;
};
/** Adds upload context without discarding the Axios response metadata used by
* Code API recovery. Transport-specific branching remains in this typed module. */
export function wrapCodeApiUploadError(error: unknown, message: string): Error {
const wrapped = new Error(logAxiosError({ message, error }), { cause: error });
if (axios.isAxiosError(error)) {
Object.assign(wrapped, {
isAxiosError: true,
response: error.response,
});
}
return wrapped;
}
/**
* Creates and configures an Axios instance with optional proxy settings.

View file

@ -1,5 +1,14 @@
import { AxiosError, AxiosHeaders } from 'axios';
import { getCodeApiRetryAfterMs, withCodeApiRateLimit, createCodeApiRateLimitBudget } from './code';
import {
getCodeApiRetryAfterMs,
getCodeApiUploadOptions,
withCodeApiRateLimit,
withCodeApiUploadRecovery,
withCodeApiUploadSlot,
createCodeApiRateLimitBudget,
createCodeApiUploadRegistry,
} from './code';
import { wrapCodeApiUploadError } from './axios';
/** Builds the error axios raises for a Code API rate-limit response. */
function rateLimited({
@ -22,6 +31,18 @@ function rateLimited({
return error;
}
describe('wrapCodeApiUploadError', () => {
it('preserves Axios rate-limit metadata on the contextual error', () => {
const cause = rateLimited({ headers: { 'retry-after': '7' } });
expect(wrapCodeApiUploadError(cause, 'Upload failed')).toMatchObject({
cause,
isAxiosError: true,
response: { status: 429 },
});
});
});
describe('getCodeApiRetryAfterMs', () => {
it('reads the standard Retry-After header', () => {
expect(getCodeApiRetryAfterMs(rateLimited({ headers: { 'retry-after': '17' } }))).toBe(17_000);
@ -88,7 +109,7 @@ describe('withCodeApiRateLimit', () => {
expect(attempts).toBe(2);
expect(waits).toEqual([1_000]);
expect(budget.remainingMs).toBe(4_000);
expect(budget.waitedMs).toBeGreaterThanOrEqual(1_000);
});
it('names the limit when the wait exceeds the remaining budget', async () => {
@ -99,9 +120,15 @@ describe('withCodeApiRateLimit', () => {
await expect(
withCodeApiRateLimit({ attempt, label: 'reading "x.png"', budget }),
).rejects.toThrow(/rate limit reached while reading "x\.png" \(retry in 300s\)/);
).rejects.toMatchObject({
name: 'CodeApiRateLimitError',
code: 'CODE_API_RATE_LIMITED',
status: 429,
statusCode: 429,
retryAfterMs: 300_000,
});
expect(attempt).toHaveBeenCalledTimes(1);
expect(budget.remainingMs).toBe(5_000);
expect(budget.waitedMs).toBe(0);
});
it('names the limit even when the response carries no usable delay', async () => {
@ -117,10 +144,15 @@ describe('withCodeApiRateLimit', () => {
label: 'reading',
budget: createCodeApiRateLimitBudget(60_000),
}),
).rejects.toThrow(/rate limit reached while reading\./);
).rejects.toMatchObject({
name: 'CodeApiRateLimitError',
code: 'CODE_API_RATE_LIMITED',
status: 429,
statusCode: 429,
});
});
it('charges at least a second so a zero delay cannot spin', async () => {
it('waits at least a second so a zero delay cannot spin', async () => {
const budget = createCodeApiRateLimitBudget(2_500);
const waits: number[] = [];
let attempts = 0;
@ -136,7 +168,6 @@ describe('withCodeApiRateLimit', () => {
withCodeApiRateLimit({ attempt, label: 'reading', budget, onWait: (ms) => waits.push(ms) }),
).resolves.toBe('ok');
expect(waits).toEqual([1_000, 1_000]);
expect(budget.remainingMs).toBe(500);
});
it('rethrows anything that is not a rate limit', async () => {
@ -147,4 +178,250 @@ describe('withCodeApiRateLimit', () => {
'Request failed',
);
});
it('stops waiting when the calling operation is cancelled', async () => {
const controller = new AbortController();
const attempt = jest.fn(async () => {
throw rateLimited({ headers: { 'retry-after': '10' } });
});
const pending = withCodeApiRateLimit({
attempt,
label: 'uploading',
budget: createCodeApiRateLimitBudget(20_000),
signal: controller.signal,
onWait: () => controller.abort(),
});
await expect(pending).rejects.toMatchObject({ name: 'AbortError' });
expect(attempt).toHaveBeenCalledTimes(1);
});
it('shares elapsed wall time across concurrent waits', async () => {
const budget = createCodeApiRateLimitBudget(2_500);
const attempts = [0, 0, 0];
const run = (index: number) =>
withCodeApiRateLimit({
budget,
label: `uploading ${index}`,
attempt: async () => {
if (attempts[index]++ === 0) {
throw rateLimited({ headers: { 'retry-after': '0' } });
}
return index;
},
});
await expect(Promise.all([run(0), run(1), run(2)])).resolves.toEqual([0, 1, 2]);
});
it('does not charge successful request execution against the wait budget', async () => {
jest.useFakeTimers();
try {
const budget = createCodeApiRateLimitBudget(1_500);
let attempts = 0;
const pending = withCodeApiRateLimit({
budget,
label: 'reading windows',
attempt: async () => {
attempts++;
if (attempts === 1) {
await new Promise((resolve) => setTimeout(resolve, 30_000));
throw rateLimited({ headers: { 'retry-after': '1' } });
}
return 'ok';
},
});
await jest.advanceTimersByTimeAsync(30_000);
await jest.advanceTimersByTimeAsync(1_000);
await expect(pending).resolves.toBe('ok');
expect(budget.waitedMs).toBe(1_000);
} finally {
jest.useRealTimers();
}
});
});
describe('withCodeApiUploadSlot', () => {
it('removes a canceled caller from a busy scope immediately', async () => {
const registry = createCodeApiUploadRegistry();
let release!: () => void;
const gate = new Promise<void>((resolve) => {
release = resolve;
});
const first = withCodeApiUploadSlot({
registry,
scope: 'route:user-a',
concurrency: 1,
task: () => gate,
});
const controller = new AbortController();
const queuedTask = jest.fn(async () => undefined);
const queued = withCodeApiUploadSlot({
registry,
scope: 'route:user-a',
concurrency: 1,
signal: controller.signal,
task: queuedTask,
});
controller.abort();
await expect(queued).rejects.toMatchObject({ name: 'AbortError' });
expect(queuedTask).not.toHaveBeenCalled();
release();
await first;
});
it('does not block a different principal and route scope', async () => {
const registry = createCodeApiUploadRegistry();
let release!: () => void;
const gate = new Promise<void>((resolve) => {
release = resolve;
});
const first = withCodeApiUploadSlot({
registry,
scope: 'route-a:user-a',
concurrency: 1,
task: () => gate,
});
const independent = jest.fn(async () => 'ok');
await expect(
withCodeApiUploadSlot({
registry,
scope: 'route-b:user-b',
concurrency: 1,
task: independent,
}),
).resolves.toBe('ok');
expect(independent).toHaveBeenCalledTimes(1);
release();
await first;
});
});
describe('withCodeApiUploadRecovery', () => {
it('disposes a failed source before opening the retry source', async () => {
jest.useFakeTimers();
try {
const first = { destroy: jest.fn() };
const second = { destroy: jest.fn() };
const openSource = jest
.fn<Promise<typeof first>, []>()
.mockResolvedValueOnce(first)
.mockResolvedValueOnce(second);
const upload = jest
.fn<Promise<string>, [typeof first]>()
.mockRejectedValueOnce(rateLimited({ headers: { 'retry-after': '0' } }))
.mockResolvedValueOnce('ok');
const pending = withCodeApiUploadRecovery({
registry: createCodeApiUploadRegistry(),
scope: 'route:user',
budget: createCodeApiRateLimitBudget(2_000),
label: 'uploading',
openSource,
upload,
});
await jest.advanceTimersByTimeAsync(1_000);
await expect(pending).resolves.toBe('ok');
expect(first.destroy).toHaveBeenCalledTimes(1);
expect(second.destroy).not.toHaveBeenCalled();
expect(upload.mock.calls.map(([source]) => source)).toEqual([first, second]);
} finally {
jest.useRealTimers();
}
});
it('disposes every stream in a failed batch source', async () => {
const first = { stream: { destroy: jest.fn() } };
const second = { stream: { destroy: jest.fn() } };
await expect(
withCodeApiUploadRecovery({
registry: createCodeApiUploadRegistry(),
scope: 'route:user',
budget: createCodeApiRateLimitBudget(0),
label: 'uploading',
openSource: async () => [first, second],
upload: async () => {
throw rateLimited({ headers: { 'retry-after': '30' } });
},
}),
).rejects.toMatchObject({ code: 'CODE_API_RATE_LIMITED' });
expect(first.stream.destroy).toHaveBeenCalledTimes(1);
expect(second.stream.destroy).toHaveBeenCalledTimes(1);
});
it('does not classify a backing-storage 429 as Code API throttling', async () => {
const storageRateLimit = rateLimited({ headers: { 'retry-after': '1' } });
const upload = jest.fn();
await expect(
withCodeApiUploadRecovery({
registry: createCodeApiUploadRegistry(),
scope: 'route:user',
budget: createCodeApiRateLimitBudget(2_000),
label: 'uploading',
openSource: async () => {
throw storageRateLimit;
},
upload,
}),
).rejects.toBe(storageRateLimit);
expect(upload).not.toHaveBeenCalled();
});
it('disposes a source acquired as cancellation arrives', async () => {
const controller = new AbortController();
const source = { destroy: jest.fn() };
const upload = jest.fn();
await expect(
withCodeApiUploadRecovery({
registry: createCodeApiUploadRegistry(),
scope: 'route:user',
signal: controller.signal,
label: 'uploading',
openSource: async () => {
controller.abort();
return source;
},
upload,
}),
).rejects.toMatchObject({ name: 'AbortError' });
expect(source.destroy).toHaveBeenCalledTimes(1);
expect(upload).not.toHaveBeenCalled();
});
});
describe('getCodeApiUploadOptions', () => {
it('uses the configured limit and isolates tenants, principals, and routes', () => {
const first = getCodeApiUploadOptions(
{
user: { id: 'user-a', tenantId: 'tenant-a' },
config: { endpoints: { agents: { codeApiUploadConcurrency: 7 } } },
} as never,
'route-a',
);
const second = getCodeApiUploadOptions(
{ user: { id: 'user-a', tenantId: 'tenant-b' } } as never,
'route-a',
);
expect(first.concurrency).toBe(7);
expect(second.concurrency).toBe(3);
expect(first.retryWaitMs).toBe(20_000);
expect(first.scope).not.toBe(second.scope);
expect(first.scope).not.toBe(
getCodeApiUploadOptions({ user: { id: 'user-b', tenantId: 'tenant-a' } } as never, 'route-a')
.scope,
);
expect(first.scope).not.toBe(
getCodeApiUploadOptions({ user: { id: 'user-a', tenantId: 'tenant-a' } } as never, 'route-b')
.scope,
);
});
});

View file

@ -1,6 +1,8 @@
import http from 'http';
import https from 'https';
import { isAxiosError } from 'axios';
import { setTimeout as delay } from 'node:timers/promises';
import type { ServerRequest } from '~/types';
/**
* Dedicated agents for code-server requests, preventing socket pool contamination.
@ -41,18 +43,223 @@ export function getCodeApiRetryAfterMs(error: unknown): number | null {
}
/** Total time one operation may spend waiting out Code API rate limits.
* Sized to cover a single limiter window without stalling a chat turn. */
* Keeps recovery bounded inside a live chat turn. */
export const MAX_CODE_API_RATE_LIMIT_WAIT_MS = 20_000;
export const CODE_API_UPLOAD_CONCURRENCY_DEFAULT = 3;
/** Remaining wait allowance, shared by every request of one operation. */
export interface CodeApiUploadRegistry {
scopes: Map<string, UploadScopeState>;
}
export function createCodeApiUploadRegistry(): CodeApiUploadRegistry {
return { scopes: new Map() };
}
export function createCodeApiUploadScope(params: {
route: string;
principalId: string;
tenantId?: string;
}): string {
return JSON.stringify([params.route, params.tenantId ?? 'legacy', params.principalId]);
}
export function getCodeApiUploadOptions(
req: ServerRequest,
route: string,
): { scope: string; concurrency: number; retryWaitMs: number } {
const user = req.user as typeof req.user & {
_id?: { toString(): string } | string;
tenantId?: unknown;
orgId?: unknown;
};
const principalId = user?.id ?? user?._id?.toString();
if (!principalId) {
throw new Error('Code API upload recovery requires an authenticated principal');
}
return {
scope: createCodeApiUploadScope({
route,
principalId,
tenantId:
user.tenantId == null && user.orgId == null
? undefined
: String(user.tenantId ?? user.orgId),
}),
concurrency:
req.config?.endpoints?.agents?.codeApiUploadConcurrency ??
CODE_API_UPLOAD_CONCURRENCY_DEFAULT,
retryWaitMs:
req.config?.endpoints?.agents?.codeApiMaxRetryWaitMs ?? MAX_CODE_API_RATE_LIMIT_WAIT_MS,
};
}
interface QueuedUpload {
run: () => void;
reject: (error: unknown) => void;
signal?: AbortSignal;
onAbort?: () => void;
}
interface UploadScopeState {
active: number;
concurrency: number;
queue: QueuedUpload[];
}
interface CodeApiRateLimitWaitToken {
endAt: number;
}
function abortReason(signal: AbortSignal): unknown {
try {
signal.throwIfAborted();
} catch (error) {
return error;
}
return new Error('The operation was aborted');
}
function drainCodeApiUploadScope(
registry: CodeApiUploadRegistry,
scope: string,
state: UploadScopeState,
): void {
while (state.active < state.concurrency && state.queue.length > 0) {
const queued = state.queue.shift();
if (!queued) {
break;
}
queued.signal?.removeEventListener('abort', queued.onAbort!);
if (queued.signal?.aborted) {
queued.reject(abortReason(queued.signal));
continue;
}
queued.run();
}
if (state.active === 0 && state.queue.length === 0) {
registry.scopes.delete(scope);
}
}
/** Limits uploads within one Code API route and principal. A throttled bucket
* holds only its own slots, and canceled callers leave the queue immediately. */
export function withCodeApiUploadSlot<T>(params: {
registry: CodeApiUploadRegistry;
task: () => Promise<T>;
scope: string;
concurrency?: number;
signal?: AbortSignal;
}): Promise<T> {
const {
registry,
task,
scope,
concurrency = CODE_API_UPLOAD_CONCURRENCY_DEFAULT,
signal,
} = params;
if (!Number.isInteger(concurrency) || concurrency < 1) {
throw new Error(`Code API upload concurrency must be a positive integer (got ${concurrency})`);
}
signal?.throwIfAborted();
let state = registry.scopes.get(scope);
if (!state) {
state = { active: 0, concurrency, queue: [] };
registry.scopes.set(scope, state);
} else {
state.concurrency = concurrency;
}
return new Promise<T>((resolve, reject) => {
const run = (): void => {
state.active++;
Promise.resolve()
.then(task)
.then(resolve, reject)
.finally(() => {
state.active--;
drainCodeApiUploadScope(registry, scope, state);
});
};
if (state.active < state.concurrency) {
run();
return;
}
const queued: QueuedUpload = { run, reject, signal };
queued.onAbort = () => {
const index = state.queue.indexOf(queued);
if (index >= 0) {
state.queue.splice(index, 1);
reject(abortReason(signal!));
drainCodeApiUploadScope(registry, scope, state);
}
};
signal?.addEventListener('abort', queued.onAbort, { once: true });
state.queue.push(queued);
});
}
/** Actual rate-limit wait time shared by every request of one operation. */
export interface CodeApiRateLimitBudget {
remainingMs: number;
limitMs: number;
waitedMs: number;
waitingSince?: number;
activeWaitEnds: Set<CodeApiRateLimitWaitToken>;
}
export function createCodeApiRateLimitBudget(
totalMs: number = MAX_CODE_API_RATE_LIMIT_WAIT_MS,
): CodeApiRateLimitBudget {
return { remainingMs: totalMs };
return { limitMs: totalMs, waitedMs: 0, activeWaitEnds: new Set() };
}
function beginCodeApiRateLimitWait(
budget: CodeApiRateLimitBudget,
waitMs: number,
): CodeApiRateLimitWaitToken | null {
const now = Date.now();
const activeElapsed = budget.waitingSince == null ? 0 : now - budget.waitingSince;
const currentWaitEnd = Math.max(
now,
...Array.from(budget.activeWaitEnds, (token) => token.endAt),
);
const projectedWaitedMs =
budget.waitedMs + activeElapsed + Math.max(currentWaitEnd, now + waitMs) - now;
if (projectedWaitedMs > budget.limitMs) {
return null;
}
const token = { endAt: now + waitMs };
if (budget.activeWaitEnds.size === 0) {
budget.waitingSince = now;
}
budget.activeWaitEnds.add(token);
return token;
}
function finishCodeApiRateLimitWait(
budget: CodeApiRateLimitBudget,
token: CodeApiRateLimitWaitToken,
): void {
if (!budget.activeWaitEnds.delete(token)) {
return;
}
if (budget.activeWaitEnds.size === 0 && budget.waitingSince != null) {
budget.waitedMs += Date.now() - budget.waitingSince;
budget.waitingSince = undefined;
}
}
function codeApiRateLimitError(label: string, cause: unknown, retryAfterMs?: number): Error {
const retrySuffix = retryAfterMs == null ? '' : ` (retry in ${Math.ceil(retryAfterMs / 1000)}s)`;
return Object.assign(
new Error(`Code API rate limit reached while ${label}${retrySuffix}.`, { cause }),
{
name: 'CodeApiRateLimitError',
code: 'CODE_API_RATE_LIMITED',
status: 429,
statusCode: 429,
...(retryAfterMs == null ? {} : { retryAfterMs }),
},
);
}
/**
@ -69,30 +276,134 @@ export async function withCodeApiRateLimit<T>(params: {
label: string;
budget?: CodeApiRateLimitBudget;
onWait?: (waitMs: number) => void;
signal?: AbortSignal;
/** Narrows retry classification when an attempt also performs non-CodeAPI work. */
isRateLimit?: (error: unknown) => boolean;
}): Promise<T> {
const { attempt, label, budget, onWait } = params;
const {
attempt,
label,
budget,
onWait,
signal,
isRateLimit = (error) => isAxiosError(error) && error.response?.status === 429,
} = params;
for (;;) {
signal?.throwIfAborted();
try {
return await attempt();
} catch (error) {
if (!isAxiosError(error) || error.response?.status !== 429) {
if (!isRateLimit(error)) {
throw error;
}
const retryAfterMs = getCodeApiRetryAfterMs(error);
if (retryAfterMs == null) {
throw new Error(`Code API rate limit reached while ${label}.`);
throw codeApiRateLimitError(label, error);
}
/* A zero delay would spin; charge at least a second so the budget
* always drains and the loop terminates. */
/* A zero delay would spin. Wait at least a second; the actual-wait
* budget still guarantees that repeated zero hints terminate. */
const waitMs = Math.max(retryAfterMs, 1000);
if (budget == null || waitMs > budget.remainingMs) {
throw new Error(
`Code API rate limit reached while ${label} (retry in ${Math.ceil(waitMs / 1000)}s).`,
);
if (budget == null) {
throw codeApiRateLimitError(label, error, waitMs);
}
const rateLimitBudget = budget;
const waitToken = beginCodeApiRateLimitWait(rateLimitBudget, waitMs);
if (waitToken == null) {
throw codeApiRateLimitError(label, error, waitMs);
}
budget.remainingMs -= waitMs;
onWait?.(waitMs);
await new Promise((resolve) => setTimeout(resolve, waitMs));
try {
await delay(waitMs, undefined, signal ? { signal } : undefined);
} finally {
finishCodeApiRateLimitWait(rateLimitBudget, waitToken);
}
}
}
}
/** Owns upload admission, retry waiting, and source reopening for every Code API
* upload producer. Callers supply storage and transport adapters only. */
export function withCodeApiUploadRecovery<S, T>(params: {
registry: CodeApiUploadRegistry;
scope: string;
concurrency?: number;
budget?: CodeApiRateLimitBudget;
signal?: AbortSignal;
label: string;
onWait?: (waitMs: number) => void;
openSource: () => Promise<S>;
upload: (source: S) => Promise<T>;
}): Promise<T> {
const { registry, scope, concurrency, budget, signal, label, onWait, openSource, upload } =
params;
return withCodeApiUploadSlot({
registry,
scope,
concurrency,
signal,
task: () => {
let uploadStarted = false;
return withCodeApiRateLimit({
budget,
signal,
label,
onWait,
/* Storage backends can also answer 429. Only the CodeAPI transport owns this
* retry policy and its operator-facing error identity. */
isRateLimit: (error) =>
uploadStarted && isAxiosError(error) && error.response?.status === 429,
attempt: async () => {
uploadStarted = false;
let source: S | undefined;
try {
source = await openSource();
signal?.throwIfAborted();
uploadStarted = true;
return await upload(source);
} catch (error) {
if (source !== undefined) {
disposeCodeApiUploadSource(source);
}
throw error;
}
},
});
},
});
}
/** Best-effort disposal for the stream shapes accepted by upload adapters: a
* single readable or a batch of `{ stream }` entries. */
function disposeCodeApiUploadSource(source: unknown): void {
try {
if (Array.isArray(source)) {
source.forEach(disposeCodeApiUploadSource);
return;
}
if (source == null || typeof source !== 'object') {
return;
}
if ('stream' in source) {
disposeCodeApiUploadSource((source as { stream?: unknown }).stream);
return;
}
const destroy = (source as { destroy?: unknown }).destroy;
if (typeof destroy === 'function') {
destroy.call(source);
}
} catch {
/* Cleanup must not replace the upload or cancellation error being propagated. */
}
}
export function isCodeApiRateLimitError(error: unknown): error is Error & {
code: 'CODE_API_RATE_LIMITED';
status: 429;
} {
return (
error != null &&
typeof error === 'object' &&
(error as { code?: unknown }).code === 'CODE_API_RATE_LIMITED' &&
(error as { status?: unknown }).status === 429
);
}

View file

@ -68,6 +68,18 @@ describe('isAbortError', () => {
expect(isAbortError(wrapped)).toBe(true);
});
it('recognizes an Axios cancellation through an upload wrapper cause', () => {
const axiosCancellation = Object.assign(new Error('canceled'), {
name: 'CanceledError',
code: 'ERR_CANCELED',
});
const wrapped = new Error('Error uploading code environment file', {
cause: axiosCancellation,
});
expect(isAbortError(wrapped)).toBe(true);
});
it('recognizes the SDK message shape that only stringifies the reason', () => {
expect(
isAbortError(new Error('MCP error -32001: AbortError: This operation was aborted')),

View file

@ -63,6 +63,7 @@ export function isAbortError(error: unknown): boolean {
if (
name === 'AbortError' ||
code === 'ABORT_ERR' ||
code === 'ERR_CANCELED' ||
message.includes('AbortError') ||
/(?:operation|request|stream) was aborted/i.test(message)
) {

View file

@ -441,6 +441,21 @@ describe('endpointSchema addParams validation', () => {
});
describe('agentsEndpointSchema', () => {
it('defaults and bounds Code API upload recovery controls', () => {
expect(agentsEndpointSchema.parse({}).codeApiUploadConcurrency).toBe(3);
expect(
agentsEndpointSchema.parse({ codeApiUploadConcurrency: 8 }).codeApiUploadConcurrency,
).toBe(8);
expect(agentsEndpointSchema.safeParse({ codeApiUploadConcurrency: 0 }).success).toBe(false);
expect(agentsEndpointSchema.safeParse({ codeApiUploadConcurrency: 101 }).success).toBe(false);
expect(agentsEndpointSchema.parse({}).codeApiMaxRetryWaitMs).toBe(20_000);
expect(
agentsEndpointSchema.parse({ codeApiMaxRetryWaitMs: 60_000 }).codeApiMaxRetryWaitMs,
).toBe(60_000);
expect(agentsEndpointSchema.safeParse({ codeApiMaxRetryWaitMs: -1 }).success).toBe(false);
expect(agentsEndpointSchema.safeParse({ codeApiMaxRetryWaitMs: 300_001 }).success).toBe(false);
});
it('accepts a non-empty stateful code environment allowlist', () => {
const result = agentsEndpointSchema.safeParse({
statefulCodeSessions: { allowedEnvironments: ['user', 'agent-user'] },

View file

@ -1201,6 +1201,10 @@ export const agentsEndpointSchema = baseEndpointSchema
.max(MAX_SUBAGENTS_CEILING)
.optional()
.default(MAX_SUBAGENTS),
/** Maximum concurrent Code API uploads per route and authenticated principal. */
codeApiUploadConcurrency: z.number().int().min(1).max(100).optional().default(3),
/** Maximum wall-clock time spent waiting on Code API rate limits per operation. */
codeApiMaxRetryWaitMs: z.number().int().min(0).max(300_000).optional().default(20_000),
allowedProviders: z.array(z.union([z.string(), eModelEndpointSchema])).optional(),
capabilities: z
.array(z.nativeEnum(AgentCapabilities))

View file

@ -632,6 +632,10 @@ export type TConfig = {
};
/** Effective subagents-per-agent cap served from `endpoints.agents.maxSubagents`. */
maxSubagents?: number;
/** Concurrent Code API uploads allowed per route and authenticated principal. */
codeApiUploadConcurrency?: number;
/** Milliseconds one operation may spend waiting on Code API rate limits. */
codeApiMaxRetryWaitMs?: number;
customParams?: {
defaultParamsEndpoint?: string;
reasoningFormat?: ReasoningParameterFormat;