From 65bb64bc681d037709aa7053525bd038f2608429 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 9 Sep 2026 15:48:11 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=9B=9F=20fix:=20Recover=20CodeAPI=20File?= =?UTF-8?q?=20Uploads=20After=20Throttling=20(#15784)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: recover CodeAPI uploads after throttling * fix: cover eager CodeAPI uploads * fix: scope CodeAPI upload recovery * test: clarify upload recovery deadlines * fix: isolate CodeAPI recovery state * fix: cancel CodeAPI recovery waits * test: provide CodeAPI registry in handler fixture * fix: propagate CodeAPI recovery controls * fix: bound CodeAPI recovery by run * fix: preserve skill upload cancellation * test: update deferred loader signal contract * fix: preserve recovery cancellation * fix: cancel recovery source reads * fix: classify Axios upload cancellation * fix: preserve lazy provisioning cancellation * fix: close resource recovery boundaries * fix: preserve provisioning failures compatibly --- api/app/clients/tools/util/handleTools.js | 1 + .../clients/tools/util/handleTools.test.js | 3 + api/server/experimental.js | 2 + api/server/index.js | 2 + .../services/Endpoints/agents/initialize.js | 5 +- api/server/services/Files/Code/crud.js | 11 +- api/server/services/Files/Code/crud.spec.js | 25 ++ api/server/services/Files/Code/process.js | 80 +++- .../services/Files/Code/process.spec.js | 176 ++++++++- api/server/services/Files/process.js | 35 +- api/server/services/Files/process.spec.js | 53 +++ api/server/services/ToolService.js | 4 + librechat.example.yaml | 4 + packages/api/src/agents/handlers.spec.ts | 175 ++++++++- packages/api/src/agents/handlers.ts | 54 ++- .../api/src/agents/openai/service.spec.ts | 1 + packages/api/src/agents/skillFiles.spec.ts | 250 ++++++++++++- packages/api/src/agents/skillFiles.ts | 178 ++++++--- .../api/src/files/provision/callback.spec.ts | 77 ++++ packages/api/src/files/provision/callback.ts | 25 +- .../api/src/files/provision/service.spec.ts | 55 ++- packages/api/src/files/provision/service.ts | 56 ++- packages/api/src/utils/axios.ts | 13 + packages/api/src/utils/code.spec.ts | 291 ++++++++++++++- packages/api/src/utils/code.ts | 341 +++++++++++++++++- packages/api/src/utils/errors.spec.ts | 12 + packages/api/src/utils/errors.ts | 1 + .../specs/config-schemas.spec.ts | 15 + packages/data-provider/src/config.ts | 4 + packages/data-provider/src/types.ts | 4 + 30 files changed, 1792 insertions(+), 161 deletions(-) diff --git a/api/app/clients/tools/util/handleTools.js b/api/app/clients/tools/util/handleTools.js index 9bcddc95c9..0d85baf700 100644 --- a/api/app/clients/tools/util/handleTools.js +++ b/api/app/clients/tools/util/handleTools.js @@ -388,6 +388,7 @@ const loadTools = async ({ }); const { files, toolContext } = await primeCodeFiles({ ...options, + signal, agentId: agent?.id, codeApiBaseUrl: codeExecutionContext.baseUrl, executionProfile: codeExecutionContext.executionProfile, diff --git a/api/app/clients/tools/util/handleTools.test.js b/api/app/clients/tools/util/handleTools.test.js index 4b9f38e70b..70684d4dd7 100644 --- a/api/app/clients/tools/util/handleTools.test.js +++ b/api/app/clients/tools/util/handleTools.test.js @@ -374,11 +374,13 @@ describe('Tool Handlers', () => { it('routes code file priming to the selected bridge worker', async () => { const bridgeWorkerId = 'principal-worker'; + const controller = new AbortController(); const toolMap = await loadTools({ user: fakeUser._id.toString(), tools: [Tools.execute_code], returnMap: true, agent: { id: 'agent-1' }, + signal: controller.signal, options: { codeExecutionContext: { baseUrl: 'https://code.example.com/v1', @@ -394,6 +396,7 @@ describe('Tool Handlers', () => { agentId: 'agent-1', codeApiBaseUrl: 'https://code.example.com/v1', bridgeWorkerId, + signal: controller.signal, }), ); }); diff --git a/api/server/experimental.js b/api/server/experimental.js index 102c6bbfed..d0173bd728 100644 --- a/api/server/experimental.js +++ b/api/server/experimental.js @@ -51,6 +51,7 @@ const { createAgentEventTerminalHandler, startCodeEnvironmentLifecycleReconciler, waitForKeyvRedisClient, + createCodeApiUploadRegistry, } = require('@librechat/api'); const { connectDb, indexSync } = require('~/db'); const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager'); @@ -356,6 +357,7 @@ if (cluster.isMaster) { * Each worker runs a full Express server instance */ const app = express(); + app.locals.codeApiUploadRegistry = createCodeApiUploadRegistry(); // The clustered entrypoint deliberately does not arm the v1 schedule engine, // but an already-fired scheduled generation can still reach HITL here. Settle // its durable run when the generic approval runtime expires it. diff --git a/api/server/index.js b/api/server/index.js index a6be1e68ae..6fc63714fe 100644 --- a/api/server/index.js +++ b/api/server/index.js @@ -58,6 +58,7 @@ const { startCodeEnvironmentLifecycleReconciler, waitForKeyvRedisClient, warnOnUnreachableDeliveryPaths, + createCodeApiUploadRegistry, } = require('@librechat/api'); const { connectDb, indexSync } = require('~/db'); const { @@ -101,6 +102,7 @@ const host = HOST || 'localhost'; const trusted_proxy = Number(TRUST_PROXY) || 1; /* trust first proxy by default */ const app = express(); +app.locals.codeApiUploadRegistry = createCodeApiUploadRegistry(); let serverReady = false; /** @type {import('@librechat/api').ScheduleEngineState} */ let scheduleEngineState = 'starting'; diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js index d3df7cc68a..8e8c183a0d 100644 --- a/api/server/services/Endpoints/agents/initialize.js +++ b/api/server/services/Endpoints/agents/initialize.js @@ -413,7 +413,7 @@ const initializeClient = async ({ runSignal: signal, foregroundRunId, ordinaryToolCancellation: ordinaryToolCancellationEnabled, - loadTools: async (toolNames, agentId, _configurable, callerCapabilityProjection) => { + loadTools: async (toolNames, agentId, _configurable, callerCapabilityProjection, runSignal) => { const ctx = agentToolContexts.get(agentId) ?? {}; logger.debug(`[ON_TOOL_EXECUTE] ctx found: ${!!ctx.userMCPAuthMap}, agent: ${ctx.agent?.id}`); logger.debug(`[ON_TOOL_EXECUTE] toolRegistry size: ${ctx.toolRegistry?.size ?? 'undefined'}`); @@ -421,7 +421,7 @@ const initializeClient = async ({ const result = await loadToolsForExecution({ req, res, - signal, + signal: runSignal ?? signal, streamId, conversationId, requestBody: runtimeRequestBody, @@ -1599,6 +1599,7 @@ const initializeClient = async ({ req, payload, skillNames, + signal, accessibleSkillIds, executionProfiles: codeExecutionProfiles, ...getSkillToolDeps(), diff --git a/api/server/services/Files/Code/crud.js b/api/server/services/Files/Code/crud.js index 3732e4f6b7..eec03bd14c 100644 --- a/api/server/services/Files/Code/crud.js +++ b/api/server/services/Files/Code/crud.js @@ -4,6 +4,7 @@ const { getCodeBaseURL } = require('@librechat/agents'); const { EModelEndpoint, getCodeEnvRefs } = require('librechat-data-provider'); const { logAxiosError, + wrapCodeApiUploadError, appendCodeEnvFile, createAxiosInstance, codeServerHttpAgent, @@ -183,6 +184,7 @@ async function deleteCodeEnvFile(req, file) { * @param {string} [params.codeApiBaseUrl] - Trusted per-agent Code API endpoint. * @param {'default'|'stateful'} [params.executionProfile] - Trusted execution profile. * @param {string} [params.bridgeWorkerId] - Trusted worker selected for this execution. + * @param {AbortSignal} [params.signal] - Effective cancellation signal. * @returns {Promise<{ storage_session_id: string; file_id: string }>} * The codeapi storage location of the uploaded file. * @throws {Error} If there's an error during the upload process. @@ -237,12 +239,7 @@ async function uploadCodeEnvFile({ file_id: result.files[0].fileId, }; } catch (error) { - throw new Error( - logAxiosError({ - message: `Error uploading code environment file: ${error.message}`, - error, - }), - ); + throw wrapCodeApiUploadError(error, `Error uploading code environment file: ${error.message}`); } } @@ -281,6 +278,7 @@ async function batchUploadCodeEnvFiles({ codeApiBaseUrl, executionProfile, bridgeWorkerId, + signal, }) { const form = new FormData(); appendCodeEnvFileIdentity(form, { kind, id, version }); @@ -308,6 +306,7 @@ async function batchUploadCodeEnvFiles({ timeout: 120000, maxContentLength: MAX_FILE_SIZE, maxBodyLength: MAX_FILE_SIZE, + signal, }; const response = await axios.post(`${baseURL}/upload/batch`, form, options); diff --git a/api/server/services/Files/Code/crud.spec.js b/api/server/services/Files/Code/crud.spec.js index e35758ee35..6f7ead2324 100644 --- a/api/server/services/Files/Code/crud.spec.js +++ b/api/server/services/Files/Code/crud.spec.js @@ -42,6 +42,14 @@ jest.mock('@librechat/api', () => { form.append('id', identity.id); if (identity.version != null) form.append('version', String(identity.version)); }), + wrapCodeApiUploadError: jest.fn((error, message) => { + const wrapped = new Error(`${message} ${error.message}`, { cause: error }); + if (error?.isAxiosError === true) { + wrapped.isAxiosError = true; + wrapped.response = error.response; + } + return wrapped; + }), buildCodeEnvDownloadQuery: jest.fn((identity) => { validateIdentity(identity, 'buildCodeEnvDownloadQuery'); const params = new URLSearchParams({ kind: identity.kind, id: identity.id }); @@ -561,10 +569,25 @@ describe('Code CRUD', () => { await expect(uploadCodeEnvFile(baseUploadParams)).rejects.toThrow(); }); + + it('preserves rate-limit metadata through the upload logging boundary', async () => { + const rateLimit = Object.assign(new Error('Too Many Requests'), { + isAxiosError: true, + response: { status: 429, headers: { 'retry-after': '7' } }, + }); + mockAxios.post.mockRejectedValue(rateLimit); + + await expect(uploadCodeEnvFile(baseUploadParams)).rejects.toMatchObject({ + isAxiosError: true, + response: { status: 429, headers: { 'retry-after': '7' } }, + cause: rateLimit, + }); + }); }); describe('batchUploadCodeEnvFiles', () => { it('routes batch uploads through the selected bridge worker', async () => { + const controller = new AbortController(); const req = { user: { id: 'user-123' } }; mockAxios.post.mockResolvedValue({ data: { @@ -584,6 +607,7 @@ describe('Code CRUD', () => { codeApiBaseUrl: 'https://stateful-code.example.com', executionProfile: 'stateful', bridgeWorkerId: 'personal-worker-1', + signal: controller.signal, }); const [url, , callConfig] = mockAxios.post.mock.calls[0]; @@ -591,6 +615,7 @@ describe('Code CRUD', () => { expect(getCodeApiAuthHeaders).toHaveBeenCalledWith(req, 'personal-worker-1'); expect(callConfig.headers['X-CodeAPI-Expected-Profile']).toBe('stateful'); expect(callConfig.headers['X-LibreChat-Code-Worker-ID']).toBe('personal-worker-1'); + expect(callConfig.signal).toBe(controller.signal); }); }); }); diff --git a/api/server/services/Files/Code/process.js b/api/server/services/Files/Code/process.js index 001a43423e..0d0eebfabf 100644 --- a/api/server/services/Files/Code/process.js +++ b/api/server/services/Files/Code/process.js @@ -11,7 +11,10 @@ const { flattenArtifactPath, createAxiosInstance, getCodeApiAuthHeaders, + isAbortError, + getCodeApiUploadOptions, withCodeApiRateLimit, + withCodeApiUploadRecovery, classifyCodeArtifact, isMissingSandboxPathError, parseSandboxImageChunk, @@ -1138,15 +1141,18 @@ function checkIfActive(dateString) { * @param {ServerRequest} [req] - Current authenticated request, used to mint Code API auth. * @param {{baseUrl?: string, executionProfile?: 'default'|'stateful', bridgeWorkerId?: string}} [route] * Trusted host-selected Code API route. + * @param {AbortSignal} [signal] - Effective run cancellation signal. * * @returns {Promise} * A promise that resolves to the `lastModified` time string of the file if successful, or null if there is an * error in initialization or fetching the info. */ -async function getSessionInfo(ref, req, route = {}) { +async function getSessionInfo(ref, req, route = {}, signal) { try { + signal?.throwIfAborted(); const baseURL = route.baseUrl ?? getCodeBaseURL(); const authHeaders = await getCodeApiAuthHeaders(req, route.bridgeWorkerId); + signal?.throwIfAborted(); /* `/sessions/.../objects/...` is gated by codeapi's `sessionAuth` * middleware (post-Phase C). The middleware reconstructs the * sessionKey from the URL query (`kind`/`id`/`version?`) plus the @@ -1174,10 +1180,15 @@ async function getSessionInfo(ref, req, route = {}) { httpAgent: codeServerHttpAgent, httpsAgent: codeServerHttpsAgent, timeout: 5000, + signal, }); + signal?.throwIfAborted(); return response.data?.lastModified; - } catch (_error) { + } catch (error) { + if (signal?.aborted && isAbortError(error)) { + throw error; + } logger.debug('[getSessionInfo] session lookup failed (treating as cache miss)'); return null; } @@ -1275,6 +1286,9 @@ const getReuploadFailureCategory = (error) => { ) { return 'resource_access_denied'; } + if (status === 429 || code === 'CODE_API_RATE_LIMITED') { + return 'rate_limited'; + } return 'reupload_failed'; }; @@ -1285,6 +1299,7 @@ const getReuploadFailureCategory = (error) => { * @param {Agent['tool_resources']} options.tool_resources * @param {string} [options.agentId] - The agent ID for file access control * @param {string} [options.agentResourceType] - Permission resource type for the authorized agent route + * @param {AbortSignal} [options.signal] - Effective run cancellation signal * @returns {Promise<{ * files: Array<{ id: string; session_id: string; name: string }>, * toolContext: string, @@ -1300,6 +1315,7 @@ const primeFiles = async (options) => { executionProfile = 'default', executionRouteKey = executionProfile, bridgeWorkerId, + signal, } = options; const codeApiRoute = { baseUrl: codeApiBaseUrl, executionProfile, bridgeWorkerId }; const file_ids = tool_resources?.[EToolResources.execute_code]?.file_ids ?? []; @@ -1342,6 +1358,9 @@ const primeFiles = async (options) => { const files = []; const sessions = new Map(); + const uploadOptions = getCodeApiUploadOptions(req, executionRouteKey); + /** All stale-file reuploads in this prime share one live-turn wait cap. */ + const uploadRateLimitBudget = createCodeApiRateLimitBudget(uploadOptions.retryWaitMs); let toolContext = ''; /* Claim order decides which record keeps the bare `/mnt/data/` path @@ -1442,22 +1461,40 @@ const primeFiles = async (options) => { const { handleFileUpload: uploadCodeEnvFile } = getStrategyFunctions( FileSources.execute_code, ); - const stream = await getDownloadStream(options.req, resolveDownloadPath(file)); /* Reupload preserves the resource identity from the existing * ref so codeapi re-buckets under the same sessionKey shape * (skill stays skill, user stays user). Without this, a * skill-cache-miss reupload would land in the user bucket * and never re-shareable cross-user. */ - const uploaded = await uploadCodeEnvFile({ - req: options.req, - stream, - filename: getDestination(), - kind: sourceRef.kind, - id: sourceRef.id, - ...(sourceRef.kind === 'skill' ? { version: sourceRef.version } : {}), - codeApiBaseUrl, - executionProfile, - bridgeWorkerId, + const uploaded = await withCodeApiUploadRecovery({ + registry: req.app?.locals?.codeApiUploadRegistry, + scope: uploadOptions.scope, + concurrency: uploadOptions.concurrency, + label: `re-uploading file ${file.file_id} to the code environment`, + budget: uploadRateLimitBudget, + signal, + onWait: (waitMs) => + logger.warn( + `[primeCodeFiles] Rate-limited reupload requestId=${getPrimingCorrelation(req).requestId} ` + + `runId=${getPrimingCorrelation(req).runId}; retrying in ${waitMs}ms`, + ), + openSource: async () => { + signal?.throwIfAborted(); + return getDownloadStream(options.req, resolveDownloadPath(file), { signal }); + }, + upload: (stream) => + uploadCodeEnvFile({ + req: options.req, + stream, + filename: getDestination(), + kind: sourceRef.kind, + id: sourceRef.id, + ...(sourceRef.kind === 'skill' ? { version: sourceRef.version } : {}), + codeApiBaseUrl, + executionProfile, + bridgeWorkerId, + signal, + }), }); /** @@ -1497,6 +1534,10 @@ const primeFiles = async (options) => { `oldSession=${session_id} newSession=${newRef.storage_session_id} newFileId=${newRef.file_id}`, ); } catch (error) { + /* Cancellation is an operation outcome, not a recoverable per-file + * miss. Swallowing it here would keep walking and could dispatch more + * uploads after the foreground run has ended. */ + signal?.throwIfAborted(); reuploadFailures += 1; const failureCategory = getReuploadFailureCategory(error); reuploadFailureCategories.add(failureCategory); @@ -1522,7 +1563,8 @@ const primeFiles = async (options) => { pushFile(); continue; } - const uploadTime = await getSessionInfo(ref, req, codeApiRoute); + const uploadTime = await getSessionInfo(ref, req, codeApiRoute, signal); + signal?.throwIfAborted(); if (!uploadTime) { logger.debug( `[primeCodeFiles] file=${file.file_id} path=reupload reason=no-uploadtime ` + @@ -1927,6 +1969,7 @@ async function previewWorkspaceEdit({ * @param {string} [params.runtime_session_hint] - Per-conversation stateful runtime-session hint. * @param {number} [params.maxBytes] - In-sandbox size cap; larger files return `{ tooLarge, bytes }`. * @param {ServerRequest} [params.req] - Current authenticated request, used to mint Code API auth. + * @param {AbortSignal} [params.signal] - Foreground run cancellation. * @param {string} [params.executionRouteKey] - Trusted deployment-local route identity. * @param {string} [params.bridgeWorkerId] - Trusted bridge worker selected for this execution. * @returns {Promise<{base64: string, bytes: number} @@ -1944,6 +1987,7 @@ async function readSandboxImage({ executionRouteKey, maxBytes, req, + signal, }) { const limit = typeof maxBytes === 'number' && maxBytes > 0 ? maxBytes : 5 * megabyte; const preparedBuffer = getPreparedCodeOutputBuffer({ @@ -1970,7 +2014,9 @@ async function readSandboxImage({ /** Every window is one `/exec` call against the Code API's per-user * execution limiter, so the read shares one wait budget: a window that * resets mid-read is worth pausing for, an exhausted budget is not. */ - const rateLimit = createCodeApiRateLimitBudget(); + const rateLimit = createCodeApiRateLimitBudget( + req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs, + ); return readWindowedSandboxImage({ filePath: file_path, baseUrl: baseURL, @@ -1987,6 +2033,7 @@ async function readSandboxImage({ files, req, rateLimit, + signal, }), }); } @@ -2009,6 +2056,7 @@ async function execSandboxImageChunk({ files, req, rateLimit, + signal, }) { /** @type {Record} */ const postData = { lang: 'bash', code }; @@ -2026,6 +2074,7 @@ async function execSandboxImageChunk({ const response = await withCodeApiRateLimit({ label: `reading "${file_path}" from the sandbox`, budget: rateLimit, + signal, onWait: (waitMs) => logger.warn( `[readSandboxImage] Rate-limited reading "${file_path}"; retrying in ${waitMs}ms`, @@ -2045,6 +2094,7 @@ async function execSandboxImageChunk({ httpAgent: codeServerHttpAgent, httpsAgent: codeServerHttpsAgent, timeout: 15000, + signal, }); }, }); diff --git a/api/server/services/Files/Code/process.spec.js b/api/server/services/Files/Code/process.spec.js index f678feefab..c073628538 100644 --- a/api/server/services/Files/Code/process.spec.js +++ b/api/server/services/Files/Code/process.spec.js @@ -90,13 +90,43 @@ jest.mock('@librechat/api', () => { : undefined, executeWorkspaceTool: (...args) => mockExecuteWorkspaceTool(...args), getCodeApiAuthHeaders: jest.fn(async () => ({})), + isAbortError: (error) => + error?.name === 'AbortError' || error?.code === 'ABORT_ERR' || error?.code === 'ERR_CANCELED', /* Windowing, sizing and rate-limit policy are real code in * `packages/api` with their own tests (`files/code/image.spec.ts`, * `utils/code.spec.ts`). These stand-ins are deliberately inert * passthroughs — they assert nothing about that behavior, so the * transport tests below cover only what this file still owns. */ - createCodeApiRateLimitBudget: jest.fn(() => ({ remainingMs: 20_000 })), - withCodeApiRateLimit: jest.fn(({ attempt }) => attempt()), + createCodeApiRateLimitBudget: jest.fn(() => ({ + limitMs: 20_000, + waitedMs: 0, + activeWaitEnds: new Set(), + })), + getCodeApiUploadOptions: jest.fn(() => ({ + scope: 'default:user-123', + concurrency: 3, + retryWaitMs: 20_000, + })), + withCodeApiRateLimit: jest.fn(async ({ attempt }) => { + try { + return await attempt(); + } catch (error) { + if (error?.response?.status !== 429) { + throw error; + } + return attempt(); + } + }), + withCodeApiUploadRecovery: jest.fn(async ({ openSource, upload }) => { + try { + return await upload(await openSource()); + } catch (error) { + if (error?.response?.status !== 429) { + throw error; + } + return upload(await openSource()); + } + }), buildSandboxImageReaderCode: jest.fn( ({ filePath, limit, offset, chunkBytes }) => `read ${filePath} ${limit} ${offset} ${chunkBytes}`, @@ -1475,6 +1505,33 @@ describe('Code Process', () => { expect(callConfig.httpsAgent.keepAlive).toBe(false); }); + it('forwards cancellation to the freshness request and preserves its error', async () => { + const controller = new AbortController(); + const canceled = Object.assign(new Error('canceled'), { + name: 'CanceledError', + code: 'ERR_CANCELED', + }); + mockAxios.mockImplementationOnce(async (config) => { + expect(config.signal).toBe(controller.signal); + controller.abort(); + throw canceled; + }); + + await expect( + getSessionInfo( + { + kind: 'user', + id: 'user-1', + storage_session_id: 'sess', + file_id: 'fid', + }, + mockReq, + {}, + controller.signal, + ), + ).rejects.toBe(canceled); + }); + it('forwards Code API auth headers when checking session object freshness', async () => { getCodeApiAuthHeaders.mockResolvedValueOnce({ Authorization: 'Bearer freshness-token' }); mockAxios.mockResolvedValue({ @@ -2585,6 +2642,112 @@ describe('Code Process', () => { return { handleFileUpload, getDownloadStream }; } + it('recovers a throttled stale-file reupload with a fresh stream', async () => { + const controller = new AbortController(); + const dbFile = { + file_id: 'rate-limited-file', + filename: 'report.csv', + filepath: '/uploads/report.csv', + source: 'local', + context: 'execute_code', + metadata: { + codeEnvRef: { + kind: 'user', + id: 'user-123', + storage_session_id: 'OLD_SESSION', + file_id: 'OLD_ID', + }, + }, + }; + const rateLimit = Object.assign(new Error('Request failed with status code 429'), { + isAxiosError: true, + response: { status: 429, headers: { 'retry-after': '0' } }, + }); + const handleFileUpload = jest + .fn() + .mockRejectedValueOnce(rateLimit) + .mockResolvedValueOnce({ storage_session_id: 'NEW_SESSION', file_id: 'NEW_ID' }); + const getDownloadStream = jest + .fn() + .mockResolvedValueOnce('first-stream') + .mockResolvedValueOnce('second-stream'); + getStrategyFunctions.mockImplementation((source) => + source === 'execute_code' ? { handleFileUpload } : { getDownloadStream }, + ); + getFiles.mockResolvedValue([dbFile]); + filterFilesByAgentAccess.mockImplementation(({ files }) => Promise.resolve(files)); + mockAxios.mockResolvedValue({ data: null }); + updateFile.mockResolvedValue({}); + + const result = await primeFiles({ + req: { user: { id: 'user-123', role: 'USER' } }, + tool_resources: { execute_code: { file_ids: [dbFile.file_id], files: [] } }, + agentId: 'agent-id', + signal: controller.signal, + }); + + const { withCodeApiUploadRecovery } = require('@librechat/api'); + expect(withCodeApiUploadRecovery).toHaveBeenCalledWith( + expect.objectContaining({ signal: controller.signal }), + ); + expect(handleFileUpload).toHaveBeenCalledTimes(2); + expect(handleFileUpload).toHaveBeenCalledWith( + expect.objectContaining({ signal: controller.signal }), + ); + expect(getDownloadStream).toHaveBeenCalledTimes(2); + expect(getDownloadStream).toHaveBeenCalledWith(expect.anything(), '/uploads/report.csv', { + signal: controller.signal, + }); + expect(handleFileUpload.mock.calls.map(([args]) => args.stream)).toEqual([ + 'first-stream', + 'second-stream', + ]); + expect(result.files).toEqual([ + expect.objectContaining({ id: 'NEW_ID', storage_session_id: 'NEW_SESSION' }), + ]); + }); + + it('propagates cancellation from a stale-file reupload', async () => { + const controller = new AbortController(); + const dbFile = { + file_id: 'canceled-file', + filename: 'report.csv', + filepath: '/uploads/report.csv', + source: 'local', + context: 'execute_code', + metadata: { + codeEnvRef: { + kind: 'user', + id: 'user-123', + storage_session_id: 'OLD_SESSION', + file_id: 'OLD_ID', + }, + }, + }; + const handleFileUpload = jest.fn(async () => { + controller.abort(); + controller.signal.throwIfAborted(); + }); + getStrategyFunctions.mockImplementation((source) => + source === 'execute_code' + ? { handleFileUpload } + : { getDownloadStream: jest.fn().mockResolvedValue('stream') }, + ); + getFiles.mockResolvedValue([dbFile]); + filterFilesByAgentAccess.mockImplementation(({ files }) => Promise.resolve(files)); + mockAxios.mockResolvedValue({ data: null }); + + await expect( + primeFiles({ + req: { user: { id: 'user-123', role: 'USER' } }, + tool_resources: { execute_code: { file_ids: [dbFile.file_id], files: [] } }, + agentId: 'agent-id', + signal: controller.signal, + }), + ).rejects.toMatchObject({ name: 'AbortError' }); + expect(handleFileUpload).toHaveBeenCalledTimes(1); + }); + it('uses the permission resource type established by the calling route', async () => { const files = [ { @@ -2711,6 +2874,7 @@ describe('Code Process', () => { expect(getDownloadStream).toHaveBeenCalledWith( { user: { id: 'user-123', role: 'USER' } }, '/uploads/trusted.txt', + { signal: undefined }, ); expect(handleFileUpload).toHaveBeenCalledTimes(1); }); @@ -3022,6 +3186,7 @@ describe('Code Process', () => { ], ['Azure BlobNotFound', { code: 'BlobNotFound', statusCode: 404 }, 'missing_backing_object'], ['storage access denied', { code: 'AccessDenied', status: 403 }, 'resource_access_denied'], + ['Code API throttling', { code: 'CODE_API_RATE_LIMITED', status: 429 }, 'rate_limited'], ])( 'fails with a typed recovery error for %s', async (_errorShape, downloadError, expectedCategory) => { @@ -3550,6 +3715,7 @@ describe('Code Process', () => { }); it('forwards the sandbox session, runtime hint and profile header', async () => { + const controller = new AbortController(); mockAxios.mockResolvedValue({ data: { stdout: '{}' } }); await readSandboxImage({ @@ -3559,6 +3725,7 @@ describe('Code Process', () => { files: [{ id: 'file-1', name: 'seed.csv' }], codeApiBaseUrl: 'https://code-stateful.example.com', executionProfile: 'stateful', + signal: controller.signal, }); expect(mockAxios).toHaveBeenCalledTimes(1); @@ -3571,6 +3738,11 @@ describe('Code Process', () => { files: [{ id: 'file-1', name: 'seed.csv' }], }); expect(call.headers['X-CodeAPI-Expected-Profile']).toBe('stateful'); + expect(call.signal).toBe(controller.signal); + const { withCodeApiRateLimit } = require('@librechat/api'); + expect(withCodeApiRateLimit).toHaveBeenCalledWith( + expect.objectContaining({ signal: controller.signal }), + ); }); it('omits the profile header and optional session fields when unset', async () => { diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js index c888746bea..3e216c1aad 100644 --- a/api/server/services/Files/process.js +++ b/api/server/services/Files/process.js @@ -45,6 +45,9 @@ const { sweepExpiredFiles: sweepExpiredFilesWithDeps, startExpiredFileSweep: startExpiredFileSweepWithDeps, resolveToolRoleGrants, + createCodeApiRateLimitBudget, + getCodeApiUploadOptions, + withCodeApiUploadRecovery, } = require('@librechat/api'); const { convertImage, @@ -1221,20 +1224,34 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { /* Upload from the persisted representation. Images have already been converted here, * so both the bytes and extension advertised to the sandbox describe the same file. */ const downloadPath = storageResult.storageKey ?? storageResult.filepath; - const stream = getDownloadStream - ? await getDownloadStream(req, downloadPath) - : fs.createReadStream(file.path); const codeKind = messageAttachment === true ? 'user' : 'agent'; const codeId = messageAttachment === true ? req.user.id : agent_id; const sandboxFilename = resolveSandboxFilename(sanitizeFilename(file.originalname), storedType); + const uploadOptions = getCodeApiUploadOptions(req, 'default'); let uploaded; try { - uploaded = await uploadCodeEnvFile({ - req, - stream, - filename: sandboxFilename, - kind: codeKind, - id: codeId, + uploaded = await withCodeApiUploadRecovery({ + registry: req.app?.locals?.codeApiUploadRegistry, + scope: uploadOptions.scope, + concurrency: uploadOptions.concurrency, + label: `uploading "${sandboxFilename}" to the code environment`, + budget: createCodeApiRateLimitBudget(uploadOptions.retryWaitMs), + onWait: (waitMs) => + logger.warn( + `[processAgentFileUpload] Rate-limited Code API upload; retrying in ${waitMs}ms`, + ), + openSource: () => + getDownloadStream + ? getDownloadStream(req, downloadPath) + : Promise.resolve(fs.createReadStream(file.path)), + upload: (stream) => + uploadCodeEnvFile({ + req, + stream, + filename: sandboxFilename, + kind: codeKind, + id: codeId, + }), }); } catch (error) { const { deleteFile } = getStrategyFunctions(source); diff --git a/api/server/services/Files/process.spec.js b/api/server/services/Files/process.spec.js index 76c0e2044e..8ac28b207b 100644 --- a/api/server/services/Files/process.spec.js +++ b/api/server/services/Files/process.spec.js @@ -37,6 +37,26 @@ jest.mock('@librechat/api', () => { const actualDataProvider = jest.requireActual('librechat-data-provider'); const RetentionMode = actualDataProvider.RetentionMode ?? { ALL: 'all', TEMPORARY: 'temporary' }; const getRetentionExpiry = jest.fn(() => ({})); + const createCodeApiRateLimitBudget = jest.fn(() => ({ + limitMs: 20_000, + waitedMs: 0, + activeWaitEnds: new Set(), + })); + const getCodeApiUploadOptions = jest.fn(() => ({ + scope: 'default:user-1', + concurrency: 3, + retryWaitMs: 20_000, + })); + const withCodeApiUploadRecovery = jest.fn(async ({ openSource, upload }) => { + try { + return await upload(await openSource()); + } catch (error) { + if (error?.response?.status !== 429) { + throw error; + } + return upload(await openSource()); + } + }); const UPLOAD_EXTRACTED_TEXT_PLANS = { configuredOCR: 'configured_ocr', configuredRAG: 'configured_rag', @@ -131,6 +151,9 @@ jest.mock('@librechat/api', () => { }), getStorageMetadata: jest.fn(() => ({})), getRetentionExpiry, + createCodeApiRateLimitBudget, + getCodeApiUploadOptions, + withCodeApiUploadRecovery, getAgentFileRetentionExpiry: jest.fn(({ req, messageAttachment, toolResource }) => { const interfaceConfig = req?.config?.interfaceConfig; if ( @@ -259,6 +282,9 @@ const { extractInspectableFileText, assertExtractedTextInspectable, contentFilterBlockResponse, + createCodeApiRateLimitBudget, + getCodeApiUploadOptions, + withCodeApiUploadRecovery, } = require('@librechat/api'); const PDF_MIME = 'application/pdf'; @@ -1348,6 +1374,33 @@ describe('processAgentFileUpload', () => { expect(codeEnvUpload).toHaveBeenCalled(); }); + it('retries a throttled eager upload with a fresh persisted stream', async () => { + const rateLimited = Object.assign(new Error('rate limited'), { + isAxiosError: true, + response: { status: 429, headers: { 'retry-after': '1' } }, + }); + const codeEnvUpload = setupCodeEnvUpload({ storage_session_id: 'sess-y', file_id: 'fid-y' }); + codeEnvUpload + .mockRejectedValueOnce(rateLimited) + .mockResolvedValueOnce({ storage_session_id: 'sess-retry', file_id: 'fid-retry' }); + + await processAgentFileUpload({ + req: agentsZipReq(), + res: mockRes, + metadata: { + agent_id: 'agent-abc', + tool_resource: EToolResources.execute_code, + file_id: 'file-throttled', + }, + }).catch(() => {}); + + expect(getCodeApiUploadOptions).toHaveBeenCalledTimes(1); + expect(withCodeApiUploadRecovery).toHaveBeenCalledTimes(1); + expect(createCodeApiRateLimitBudget).toHaveBeenCalledTimes(1); + expect(codeEnvUpload).toHaveBeenCalledTimes(2); + expect(codeEnvUpload.mock.calls[0][0].stream).not.toBe(codeEnvUpload.mock.calls[1][0].stream); + }); + it('defers an inferred file-search destination until tool execution', async () => { const { uploadVectors } = require('~/server/services/Files/VectorDB/crud'); setupStoredFileUpload(); diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.js index 35253aa595..011d79f475 100644 --- a/api/server/services/ToolService.js +++ b/api/server/services/ToolService.js @@ -773,6 +773,7 @@ const isBuiltInTool = (toolName) => * @param {string} [params.agentResourceType] - Permission resource type for the authorized agent route * @param {string|null} [params.streamId] - Stream ID for resumable mode * @param {number} [params.jobCreatedAt] - The generation epoch that owns emitted tool events + * @param {AbortSignal} [params.signal] - Effective run cancellation signal * @returns {Promise<{ * toolDefinitions?: import('@librechat/api').LCTool[]; * toolRegistry?: Map; @@ -792,6 +793,7 @@ async function loadToolDefinitionsWrapper({ tool_resources, codeExecutionContext, accessibleMcpServerNames, + signal, }) { if (!agent.tools || agent.tools.length === 0) { return { toolDefinitions: [] }; @@ -1448,6 +1450,7 @@ async function loadToolDefinitionsWrapper({ tool_resources, agentId: agent.id, agentResourceType, + signal, codeApiBaseUrl: resolvedCodeExecutionContext.baseUrl, executionProfile: resolvedCodeExecutionContext.executionProfile, executionRouteKey: resolvedCodeExecutionContext.executionRouteKey, @@ -1581,6 +1584,7 @@ async function loadAgentTools({ tool_resources, codeExecutionContext: providedCodeExecutionContext, accessibleMcpServerNames, + signal, }); } catch (error) { if ( diff --git a/librechat.example.yaml b/librechat.example.yaml index d91e8e2176..61bc012676 100644 --- a/librechat.example.yaml +++ b/librechat.example.yaml @@ -583,6 +583,10 @@ endpoints: # # (optional) Maximum explicit subagents per agent, for both the flat list and # # graph definitions. Defaults to 10; hard cap 50. # maxSubagents: 20 + # # (optional) Maximum concurrent Code API uploads per route and user. Defaults to 3. + # codeApiUploadConcurrency: 3 + # # (optional) Maximum total time spent waiting on Code API rate limits. Defaults to 20000 ms. + # codeApiMaxRetryWaitMs: 20000 # # (optional) Cap the number of active accessible skills shown in the model-visible catalog. # # Useful for large organizations where many department-specific skills may be available. # skills: diff --git a/packages/api/src/agents/handlers.spec.ts b/packages/api/src/agents/handlers.spec.ts index a1351a02a6..f7b8f317a9 100644 --- a/packages/api/src/agents/handlers.spec.ts +++ b/packages/api/src/agents/handlers.spec.ts @@ -20,6 +20,7 @@ import { import { markSandboxReady } from './prewarm'; import { ContentFilterError } from '../middleware/contentFilter'; import { WorkspaceToolHttpError } from '../code/workspace'; +import { createCodeApiUploadRegistry } from '~/utils'; function createMockTool( name: string, @@ -415,6 +416,35 @@ describe('createToolExecuteHandler', () => { expect(results[0].status).toBe('error'); }); + it('forwards the effective batch signal into deferred tool loading', async () => { + const controller = new AbortController(); + const tool = { + name: 'loaded_tool', + invoke: jest.fn(async () => ({ content: 'done' })), + }; + const loadTools: ToolExecuteOptions['loadTools'] = jest.fn(async () => ({ + loadedTools: [tool] as never[], + })); + const handler = createToolExecuteHandler({ loadTools }); + + await new Promise((resolve, reject) => { + handler.handle('on_tool_execute', { + toolCalls: [{ id: 'call-load', name: tool.name, args: {} }], + signal: controller.signal, + resolve, + reject, + } as ToolExecuteBatchRequest); + }); + + expect(loadTools).toHaveBeenCalledWith( + [tool.name], + undefined, + undefined, + undefined, + controller.signal, + ); + }); + it('uses the host-owned run signal when an SDK event omits its signal', async () => { const controller = new AbortController(); const tool = abortingTool(); @@ -803,7 +833,13 @@ describe('createToolExecuteHandler', () => { ); expect(loadTools).toHaveBeenCalledTimes(1); - expect(loadTools).toHaveBeenCalledWith(['allowed_tool'], undefined, configurable, undefined); + expect(loadTools).toHaveBeenCalledWith( + ['allowed_tool'], + undefined, + configurable, + undefined, + undefined, + ); expect(JSON.stringify(jest.mocked(loadTools).mock.calls)).not.toContain(protectedName); expect(results[0]).toEqual( expect.objectContaining({ @@ -1333,6 +1369,7 @@ describe('createToolExecuteHandler', () => { undefined, undefined, callerCapabilityProjection, + undefined, ); expect(capturedConfigs[0].toolDefs).toEqual([ { name: 'active_programmatic_tool', allowed_callers: ['code_execution'] }, @@ -1829,12 +1866,20 @@ describe('createToolExecuteHandler', () => { configurable: { accessibleSkillIds: skillsInScope(), codeEnvAvailable: true, - req: { user: { id: 'user-1' } }, + req: { + user: { id: 'user-1', tenantId: 'tenant-1' }, + app: { locals: { codeApiUploadRegistry: createCodeApiUploadRegistry() } }, + config: { + endpoints: { + agents: { codeApiUploadConcurrency: 1, codeApiMaxRetryWaitMs: 1_000 }, + }, + }, + }, }, })); - const getSkillByName: ToolExecuteOptions['getSkillByName'] = jest.fn(async () => ({ - _id: `${skillName}-id` as unknown as never, - name: skillName, + const getSkillByName: ToolExecuteOptions['getSkillByName'] = jest.fn(async (name) => ({ + _id: `${name ?? skillName}-id` as unknown as never, + name: name ?? skillName, body: 'skill body', fileCount: 1, version: 1, @@ -2173,6 +2218,7 @@ describe('createToolExecuteHandler', () => { }); it('omits the unavailability note when file priming succeeds', async () => { + const controller = new AbortController(); const batchUploadCodeEnvFiles = jest.fn(async () => ({ storage_session_id: 'session-ok', files: [ @@ -2182,14 +2228,24 @@ describe('createToolExecuteHandler', () => { })); const handler = createPrimingSkillHandler('note-ok-skill', batchUploadCodeEnvFiles); - const [result] = await invokeHandler(handler, [ - { - id: 'call_prime_ok', - name: Constants.SKILL_TOOL, - args: { skillName: 'note-ok-skill' }, - }, - ]); + const [result] = await new Promise((resolve, reject) => { + handler.handle('on_tool_execute', { + toolCalls: [ + { + id: 'call_prime_ok', + name: Constants.SKILL_TOOL, + args: { skillName: 'note-ok-skill' }, + }, + ], + signal: controller.signal, + resolve, + reject, + } as ToolExecuteBatchRequest); + }); + expect(batchUploadCodeEnvFiles).toHaveBeenCalledWith( + expect.objectContaining({ signal: controller.signal }), + ); expect(result.status).toBe('success'); expect(result.content).not.toContain('could not be loaded'); expect(result.artifact).toEqual( @@ -2206,6 +2262,74 @@ describe('createToolExecuteHandler', () => { ); }); + it('shares one retry-wait budget across skill calls in a tool batch', async () => { + const attempts = new Map(); + const batchUploadCodeEnvFiles = jest.fn(async ({ id }: { id: string }) => { + const attempt = (attempts.get(id) ?? 0) + 1; + attempts.set(id, attempt); + if (attempt === 1) { + const error = Object.assign(new Error('Request failed with status code 429'), { + isAxiosError: true, + response: { status: 429, headers: { 'retry-after': '0' } }, + }); + throw error; + } + const name = id.replace(/-id$/, ''); + return { + storage_session_id: `session-${name}`, + files: [{ fileId: `file-${name}`, filename: `skills/${name}/references/style.md` }], + }; + }); + const handler = createPrimingSkillHandler('fallback-skill', batchUploadCodeEnvFiles); + + const results = await invokeHandler(handler, [ + { + id: 'call_first_skill', + name: Constants.SKILL_TOOL, + args: { skillName: 'first-skill' }, + }, + { + id: 'call_second_skill', + name: Constants.SKILL_TOOL, + args: { skillName: 'second-skill' }, + }, + ]); + + expect(batchUploadCodeEnvFiles).toHaveBeenCalledTimes(3); + expect(results.filter((result) => result.artifact != null)).toHaveLength(1); + }); + + it('returns cancellation instead of a successful skill when upload recovery is aborted', async () => { + const controller = new AbortController(); + const batchUploadCodeEnvFiles = jest.fn( + ({ signal }: { signal?: AbortSignal }) => + new Promise((_resolve, reject) => { + signal?.addEventListener('abort', () => reject(signal.reason), { once: true }); + }), + ); + const handler = createPrimingSkillHandler('cancelled-skill', batchUploadCodeEnvFiles); + const resultPromise = new Promise((resolve, reject) => { + handler.handle('on_tool_execute', { + toolCalls: [ + { + id: 'call_cancelled_skill', + name: Constants.SKILL_TOOL, + args: { skillName: 'cancelled-skill' }, + }, + ], + signal: controller.signal, + resolve, + reject, + } as ToolExecuteBatchRequest); + }); + setTimeout(() => controller.abort(), 10); + + const [result] = await resultPromise; + + expect(result.status).toBe('error'); + expect(result.content).not.toContain('Skill "cancelled-skill" loaded'); + }); + it("read_file pins lookup to the primed skill's _id when manually invoked this turn (no shadowing on collision)", async () => { /* Same-name collision corner: the resolver primed a specific doc (its `_id` is in `skillPrimedIdsByName`). If read_file used @@ -5248,6 +5372,7 @@ describe('createToolExecuteHandler', () => { listWorkspaceFiles?: ToolExecuteOptions['listWorkspaceFiles']; readSandboxFile?: ToolExecuteOptions['readSandboxFile']; readSandboxImage?: ToolExecuteOptions['readSandboxImage']; + runSignal?: AbortSignal; getSkillByName?: ToolExecuteOptions['getSkillByName']; getAuthorSkillByName?: ToolExecuteOptions['getAuthorSkillByName']; }) { @@ -5265,6 +5390,7 @@ describe('createToolExecuteHandler', () => { })); return createToolExecuteHandler({ loadTools, + runSignal: params.runSignal, getSkillByName: params.getSkillByName, getAuthorSkillByName: params.getAuthorSkillByName, readWorkspaceFile: params.readWorkspaceFile, @@ -6900,6 +7026,7 @@ describe('createToolExecuteHandler', () => { * bytes, which was the matplotlib-shape mojibake regression. */ it('returns a sandbox image as an image_url artifact the model can see', async () => { + const controller = new AbortController(); const readSandboxFile = jest.fn(); const readSandboxImage = jest.fn(async () => ({ base64: PNG_B64, bytes: pngBytes })); const handler = makeReadFileHandler({ @@ -6909,14 +7036,21 @@ describe('createToolExecuteHandler', () => { readSandboxImage, }); - const [result] = await invokeHandler(handler, [ - { - id: 'call_png', - name: Constants.READ_FILE, - args: { path: '/mnt/data/simple_graph.png' }, - codeSessionContext: { session_id: 'sess-Z', files: [] }, - } as unknown as ToolCallRequest, - ]); + const [result] = await new Promise((resolve, reject) => { + handler.handle('on_tool_execute', { + toolCalls: [ + { + id: 'call_png', + name: Constants.READ_FILE, + args: { path: '/mnt/data/simple_graph.png' }, + codeSessionContext: { session_id: 'sess-Z', files: [] }, + } as unknown as ToolCallRequest, + ], + signal: controller.signal, + resolve, + reject, + } as ToolExecuteBatchRequest); + }); expect(readSandboxFile).not.toHaveBeenCalled(); expect(readSandboxImage).toHaveBeenCalledWith( @@ -6924,6 +7058,7 @@ describe('createToolExecuteHandler', () => { file_path: '/mnt/data/simple_graph.png', session_id: 'sess-Z', maxBytes: expect.any(Number), + signal: controller.signal, }), ); expect(result.status).toBe('success'); diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts index aaeaa8ee71..b922af2ffe 100644 --- a/packages/api/src/agents/handlers.ts +++ b/packages/api/src/agents/handlers.ts @@ -78,6 +78,14 @@ import { SEARCH_WORKSPACE_TOOL_NAME, isCodeSessionToolName, } from './tools'; +import { + createCodeApiRateLimitBudget, + isAbortError, + logAxiosError, + truncateMiddle, + runOutsideTracing, + getSafeErrorMetadata, +} from '~/utils'; import { ContentFilterError, contentFilterModelBoundBlockResponse, @@ -87,13 +95,6 @@ import { BACKGROUND_TASK_ABORT_GRACE_MS, BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS, } from './backgroundCompletion'; -import { - isAbortError, - logAxiosError, - truncateMiddle, - runOutsideTracing, - getSafeErrorMetadata, -} from '~/utils'; import { WorkspaceToolHttpError, WORKSPACE_EDIT_MAX_COUNT, @@ -237,6 +238,8 @@ export interface ToolExecuteOptions { configurable?: Record, /** SDK-owned live caller capability projection for this agent context. */ callerCapabilityProjection?: CallerCapabilityProjectionSnapshot, + /** Effective cancellation signal for this tool-execute batch. */ + signal?: AbortSignal, ) => Promise<{ loadedTools: StructuredToolInterface[]; /** Additional configurable properties to merge (e.g., userMCPAuthMap) */ @@ -466,6 +469,7 @@ export interface ToolExecuteOptions { codeApiBaseUrl?: string; executionProfile?: CodeExecutionContext['executionProfile']; bridgeWorkerId?: string; + signal?: AbortSignal; }) => Promise<{ storage_session_id: string; files: Array<{ fileId: string; filename: string }>; @@ -626,6 +630,7 @@ export interface ToolExecuteOptions { /** In-sandbox size cap; files larger than this return `tooLarge` without transferring bytes. */ maxBytes?: number; req?: ServerRequest; + signal?: AbortSignal; }) => Promise< | { base64: string; bytes: number } /** `size`: over `maxBytes`. `round_trips`: within the byte cap, but more @@ -2119,6 +2124,7 @@ async function handleSandboxImageRead( req?: ServerRequest, codeExecutionContext?: CodeExecutionContext, onSuccess?: () => void, + signal?: AbortSignal, ): Promise { const filtered = filteredBinaryFileResult(tc, req, filePath); if (filtered != null) { @@ -2153,6 +2159,7 @@ async function handleSandboxImageRead( session_id: ctx?.session_id, files: ctx?.files, maxBytes: MAX_SANDBOX_INLINE_IMAGE_BYTES, + ...(signal ? { signal } : {}), ...codeExecutionRequestParams(codeExecutionContext), ...(req ? { req } : {}), }); @@ -2235,10 +2242,20 @@ async function handleSandboxFileFallback( req?: ServerRequest, codeExecutionContext?: CodeExecutionContext, onSuccess?: () => void, + signal?: AbortSignal, ): Promise { const ext = lowercaseExtension(filePath); if (SANDBOX_IMAGE_EXTENSIONS.has(ext)) { - return handleSandboxImageRead(tc, filePath, ext, options, req, codeExecutionContext, onSuccess); + return handleSandboxImageRead( + tc, + filePath, + ext, + options, + req, + codeExecutionContext, + onSuccess, + signal, + ); } const filteredName = filteredFileNameResult(tc, req, filePath); if (filteredName != null) { @@ -4366,6 +4383,7 @@ async function handleReadFileCall( req, codeExecutionContext, onSandboxReadSuccess, + signal, ); } return { @@ -4402,6 +4420,7 @@ async function handleReadFileCall( req, codeExecutionContext, onSandboxReadSuccess, + signal, ); } return { @@ -4429,6 +4448,7 @@ async function handleReadFileCall( req, codeExecutionContext, onSandboxReadSuccess, + signal, ); } return { @@ -4498,6 +4518,7 @@ async function handleReadFileCall( req, codeExecutionContext, onSandboxReadSuccess, + signal, ); } return { @@ -4544,6 +4565,7 @@ async function handleReadFileCall( req, codeExecutionContext, onSandboxReadSuccess, + signal, ); } return { @@ -4869,6 +4891,8 @@ async function handleSkillToolCall( options: ToolExecuteOptions, agentId?: string, req?: ServerRequest, + signal?: AbortSignal, + rateLimitBudget?: import('~/utils').CodeApiRateLimitBudget, ): Promise { const { getSkillByName, @@ -4984,7 +5008,9 @@ async function handleSkillToolCall( ) { let primeResult: PrimeSkillFilesResult | null = null; try { + signal?.throwIfAborted(); const skillFiles = await listSkillFiles(skill._id); + signal?.throwIfAborted(); primeResult = await primeSkillFiles({ skill, skillFiles, @@ -4995,6 +5021,8 @@ async function handleSkillToolCall( checkIfActive, updateSkillFileCodeEnvIds, codeExecutionContext, + signal, + rateLimitBudget, }); if (primeResult) { /* `session_id` at the top of the artifact is the (representative) @@ -5019,6 +5047,9 @@ async function handleSkillToolCall( }; } } catch (error) { + if (isAbortError(error)) { + throw error; + } if (isContentFilterError(error)) { return error instanceof ContentFilterError ? errorResult(tc, modelBoundContentFilterErrorMessage(error.body)) @@ -5303,6 +5334,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand agentId, sourceConfigurable, callerCapabilityProjection, + runSignal, ); const toolMap = new Map(loadedTools.map((t) => [t.name, t])); const loadedConfigurable = toolConfigurable as Record | undefined; @@ -6186,6 +6218,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand }; }; + const batchReq = mergedConfigurable?.req as ServerRequest | undefined; + const batchCodeApiRateLimitBudget = createCodeApiRateLimitBudget( + batchReq?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs, + ); const results: ToolExecuteResult[] = await Promise.all( toolCalls.map(async (tc: ToolCallRequest) => { const preloadedNameBlock = preloadedNameBlocks.get(tc); @@ -6478,6 +6514,8 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand options, agentId, req, + runSignal, + batchCodeApiRateLimitBudget, ); } else if (tc.name === Constants.READ_FILE) { handlerResult = await handleReadFileCall( diff --git a/packages/api/src/agents/openai/service.spec.ts b/packages/api/src/agents/openai/service.spec.ts index 9655ff1823..b06b32ba67 100644 --- a/packages/api/src/agents/openai/service.spec.ts +++ b/packages/api/src/agents/openai/service.spec.ts @@ -255,6 +255,7 @@ describe('createAgentChatCompletion - MCP permission user propagation', () => { 'agent_test', expect.objectContaining({ requestBody: runArgs.requestBody }), undefined, + undefined, ); }); diff --git a/packages/api/src/agents/skillFiles.spec.ts b/packages/api/src/agents/skillFiles.spec.ts index f7a666ed02..5c6dd875c4 100644 --- a/packages/api/src/agents/skillFiles.spec.ts +++ b/packages/api/src/agents/skillFiles.spec.ts @@ -14,6 +14,7 @@ jest.mock('./run', () => ({ import { Readable } from 'stream'; import { Types } from 'mongoose'; +import { createCodeApiUploadRegistry } from '~/utils'; import { primeInvokedSkills, primeInvokedSkillsForProfiles, primeSkillFiles } from './skillFiles'; import type { PrimeInvokedSkillsDeps, @@ -23,6 +24,7 @@ import type { const SKILL_ID = new Types.ObjectId(); const SKILL_VERSION = 7; +const uploadRegistry = createCodeApiUploadRegistry(); function makeDeps(overrides: Partial = {}): PrimeInvokedSkillsDeps { const listSkillFiles = jest.fn().mockResolvedValue([]); @@ -31,7 +33,10 @@ function makeDeps(overrides: Partial = {}): PrimeInvoked files: [], }); return { - req: { user: { id: 'user-1' } } as PrimeInvokedSkillsDeps['req'], + req: { + user: { id: 'user-1', tenantId: 'tenant-1' }, + app: { locals: { codeApiUploadRegistry: uploadRegistry } }, + } as unknown as PrimeInvokedSkillsDeps['req'], payload: [{ role: 'assistant', content: [] }], accessibleSkillIds: [SKILL_ID], codeEnvAvailable: true, @@ -90,6 +95,66 @@ describe('primeInvokedSkills — execute_code capability gate', () => { expect(deps.listSkillFiles).toHaveBeenCalledWith(SKILL_ID); }); + it('shares one retry-wait budget across historical skill uploads', async () => { + const skillIds = [new Types.ObjectId(), new Types.ObjectId()]; + const skillNames = ['first-skill', 'second-skill']; + mockExtract.mockReturnValue(new Set(skillNames)); + const getSkillByName = jest.fn(async (name: string) => { + const index = skillNames.indexOf(name); + return { + _id: skillIds[index], + name, + body: `${name} body`, + version: 1, + fileCount: 1, + }; + }); + const listSkillFiles = jest.fn(async () => [ + { + relativePath: 'references/style.md', + filename: 'style.md', + filepath: '/storage/style.md', + source: 's3', + bytes: 5, + }, + ]); + const attempts = new Map(); + const batchUploadCodeEnvFiles = jest.fn(async ({ id }: { id: string }) => { + const attempt = (attempts.get(id) ?? 0) + 1; + attempts.set(id, attempt); + if (attempt === 1) { + const error = new Error('Request failed with status code 429') as Error & { + isAxiosError: boolean; + response: { status: number; headers: Record }; + }; + error.isAxiosError = true; + error.response = { status: 429, headers: { 'retry-after': '0' } }; + throw error; + } + const name = skillNames[skillIds.findIndex((skillId) => skillId.toString() === id)]; + return { + storage_session_id: `session-${name}`, + files: [{ fileId: `file-${name}`, filename: `skills/${name}/references/style.md` }], + }; + }); + const deps = makeDeps({ + getSkillByName, + listSkillFiles, + getStrategyFunctions: jest.fn().mockReturnValue({ + getDownloadStream: jest.fn().mockResolvedValue(Readable.from(Buffer.from('style'))), + }), + batchUploadCodeEnvFiles, + }); + deps.req.config = { + endpoints: { agents: { codeApiUploadConcurrency: 1, codeApiMaxRetryWaitMs: 1_000 } }, + } as never; + + const result = await primeInvokedSkills(deps); + + expect(batchUploadCodeEnvFiles).toHaveBeenCalledTimes(3); + expect(result.initialSessions?.get('execute_code')?.files).toHaveLength(1); + }); + it('calls batchUploadCodeEnvFiles without an apiKey when files are returned', async () => { const fileRecords = [ { @@ -527,6 +592,7 @@ describe('primeInvokedSkills — execute_code capability gate', () => { }, deps.req, undefined, + undefined, ); const codeSession = result.initialSessions?.get('execute_code'); expect(codeSession?.files).toEqual([ @@ -698,6 +764,82 @@ describe('primeInvokedSkillsForProfiles', () => { ).toEqual(['stateful:first', 'stateful:second']); }); + it('shares one retry-wait budget across execution profiles', async () => { + const file = { + relativePath: 'references/style.md', + filename: 'style.md', + filepath: '/storage/style.md', + source: 's3', + bytes: 5, + }; + const attempts = new Map(); + const batchUploadCodeEnvFiles = jest.fn( + async ({ executionProfile }: { executionProfile?: string }) => { + const profile = executionProfile ?? 'default'; + const attempt = (attempts.get(profile) ?? 0) + 1; + attempts.set(profile, attempt); + if (attempt === 1) { + if (profile === 'stateful') { + await new Promise((resolve) => setTimeout(resolve, 500)); + } + const error = Object.assign(new Error('Request failed with status code 429'), { + isAxiosError: true, + response: { status: 429, headers: { 'retry-after': '0' } }, + }); + throw error; + } + return { + storage_session_id: `${profile}-session`, + files: [ + { + fileId: `${profile}-file`, + filename: 'skills/brand-guidelines/references/style.md', + }, + ], + }; + }, + ); + const { codeEnvAvailable: _codeEnvAvailable, ...baseDeps } = makeDeps({ + listSkillFiles: jest.fn().mockResolvedValue([file]), + getStrategyFunctions: jest.fn().mockReturnValue({ + getDownloadStream: jest.fn().mockResolvedValue(Readable.from(Buffer.from('style'))), + }), + batchUploadCodeEnvFiles, + }); + baseDeps.req.config = { + endpoints: { agents: { codeApiMaxRetryWaitMs: 1_000 } }, + } as never; + + const result = await primeInvokedSkillsForProfiles({ + ...baseDeps, + executionProfiles: [ + { + codeExecutionContext: { + baseUrl: 'https://default.example.com/v1', + codeSessionKey: 'execute_code', + executionProfile: 'default', + statefulSessions: false, + }, + codeSessionKeys: ['execute_code'], + }, + { + codeExecutionContext: { + baseUrl: 'https://stateful.example.com/v1', + codeSessionKey: 'execute_code:stateful', + executionProfile: 'stateful', + executionRouteKey: 'stateful:one', + statefulSessions: true, + }, + codeSessionKeys: ['execute_code:stateful'], + }, + ], + }); + + expect(batchUploadCodeEnvFiles).toHaveBeenCalledTimes(3); + expect(result.initialSessions?.has('execute_code')).toBe(true); + expect(result.initialSessions?.has('execute_code:stateful')).toBe(false); + }); + it('keeps a successful profile Skill body and identity after another profile lookup fails', async () => { const { codeEnvAvailable: _codeEnvAvailable, @@ -768,7 +910,10 @@ function makeSkillFilesDeps(overrides: Partial = {}): Pri version: SKILL_VERSION, }, skillFiles: [], - req: { user: { id: 'user-1' } } as PrimeSkillFilesParams['req'], + req: { + user: { id: 'user-1', tenantId: 'tenant-1' }, + app: { locals: { codeApiUploadRegistry: uploadRegistry } }, + } as unknown as PrimeSkillFilesParams['req'], getStrategyFunctions: jest.fn().mockReturnValue({ getDownloadStream: jest.fn().mockResolvedValue(Readable.from(Buffer.from(''))), }), @@ -861,10 +1006,15 @@ describe('primeSkillFiles — resource identity propagation', () => { const result = await primeSkillFiles(deps); expect(batchUploadCodeEnvFiles).not.toHaveBeenCalled(); - expect(deps.getSessionInfo).toHaveBeenCalledWith(cachedRef, deps.req, { - baseUrl: 'https://stateful-code.example.com', - executionProfile: 'stateful', - }); + expect(deps.getSessionInfo).toHaveBeenCalledWith( + cachedRef, + deps.req, + { + baseUrl: 'https://stateful-code.example.com', + executionProfile: 'stateful', + }, + undefined, + ); expect(result?.files).toEqual([ { id: 'file-cached', @@ -969,7 +1119,8 @@ describe('primeSkillFiles — resource identity propagation', () => { const batchUploadCodeEnvFiles = jest.fn(); const deps = makeSkillFilesDeps({ req: { - user: { id: 'user-1' }, + user: { id: 'user-1', tenantId: 'tenant-1' }, + app: { locals: { codeApiUploadRegistry: uploadRegistry } }, config: { filters: { skills: { @@ -980,7 +1131,7 @@ describe('primeSkillFiles — resource identity propagation', () => { }, }, }, - } as PrimeSkillFilesParams['req'], + } as unknown as PrimeSkillFilesParams['req'], skillFiles: [ { relativePath: 'references/style.md', @@ -1019,7 +1170,8 @@ describe('primeSkillFiles — resource identity propagation', () => { const batchUploadCodeEnvFiles = jest.fn(); const deps = makeSkillFilesDeps({ req: { - user: { id: 'user-1' }, + user: { id: 'user-1', tenantId: 'tenant-1' }, + app: { locals: { codeApiUploadRegistry: uploadRegistry } }, config: { filters: { files: { @@ -1029,7 +1181,7 @@ describe('primeSkillFiles — resource identity propagation', () => { }, }, }, - } as PrimeSkillFilesParams['req'], + } as unknown as PrimeSkillFilesParams['req'], skillFiles: [ { relativePath: 'references/style.md', @@ -1077,7 +1229,7 @@ describe('primeSkillFiles — resource identity propagation', () => { }, }, }, - } as PrimeSkillFilesParams['req'], + } as unknown as PrimeSkillFilesParams['req'], skillFiles: [ { relativePath: 'references/sk-private-name.md', @@ -1123,7 +1275,7 @@ describe('primeSkillFiles — resource identity propagation', () => { }, }, }, - } as PrimeSkillFilesParams['req'], + } as unknown as PrimeSkillFilesParams['req'], skillFiles: [ { relativePath: 'references/style.md', @@ -1374,7 +1526,7 @@ describe('primeSkillFiles — resource identity propagation', () => { }, }, }, - } as PrimeSkillFilesParams['req'], + } as unknown as PrimeSkillFilesParams['req'], skillFiles: [ { relativePath: 'references/unavailable.txt', @@ -1505,7 +1657,8 @@ describe('primeSkillFiles — resource identity propagation', () => { }); const deps = makeSkillFilesDeps({ req: { - user: { id: 'user-1' }, + user: { id: 'user-1', tenantId: 'tenant-1' }, + app: { locals: { codeApiUploadRegistry: uploadRegistry } }, config: { filters: { files: { @@ -1517,7 +1670,7 @@ describe('primeSkillFiles — resource identity propagation', () => { }, }, }, - } as PrimeSkillFilesParams['req'], + } as unknown as PrimeSkillFilesParams['req'], skillFiles: [ { relativePath: 'references/archive.bin', @@ -1679,7 +1832,72 @@ describe('primeSkillFiles — upload rate-limit resilience', () => { expect(result).toBeNull(); }); - it('bounds concurrent batch uploads to 3 process-wide slots', async () => { + it('does not upload a partial skill bundle when stream acquisition is canceled', async () => { + const controller = new AbortController(); + const batchUploadCodeEnvFiles = jest.fn(); + const acquiredStream = Readable.from(Buffer.from('style')); + const destroy = jest.spyOn(acquiredStream, 'destroy'); + const getDownloadStream = jest.fn(async () => { + controller.abort(); + return acquiredStream; + }); + + await expect( + primeSkillFiles( + makeSkillFilesDeps({ + signal: controller.signal, + skillFiles: [styleFileRecord()], + batchUploadCodeEnvFiles, + getStrategyFunctions: jest.fn().mockReturnValue({ getDownloadStream }), + }), + ), + ).rejects.toMatchObject({ name: 'AbortError' }); + expect(getDownloadStream).toHaveBeenCalledWith(expect.anything(), expect.any(String), { + signal: controller.signal, + }); + expect(destroy).toHaveBeenCalled(); + expect(batchUploadCodeEnvFiles).not.toHaveBeenCalled(); + }); + + it('does not return a cache hit when cancellation arrives during its liveness check', async () => { + const controller = new AbortController(); + const batchUploadCodeEnvFiles = jest.fn(); + const getSessionInfo = jest.fn(async () => { + controller.abort(); + return '2026-05-06T00:00:00Z'; + }); + const cachedFile = { + ...styleFileRecord(), + codeEnvRef: { + kind: 'skill' as const, + id: SKILL_ID.toString(), + storage_session_id: 'session-cached', + file_id: 'file-cached', + version: SKILL_VERSION, + }, + }; + + await expect( + primeSkillFiles( + makeSkillFilesDeps({ + signal: controller.signal, + skillFiles: [cachedFile], + batchUploadCodeEnvFiles, + getSessionInfo, + checkIfActive: jest.fn().mockReturnValue(true), + }), + ), + ).rejects.toMatchObject({ name: 'AbortError' }); + expect(getSessionInfo).toHaveBeenCalledWith( + expect.any(Object), + expect.any(Object), + undefined, + controller.signal, + ); + expect(batchUploadCodeEnvFiles).not.toHaveBeenCalled(); + }); + + it('bounds concurrent batch uploads to 3 application-scoped slots', async () => { const gates = Array.from({ length: 5 }, () => deferred>()); let uploadIndex = 0; const batchUploadCodeEnvFiles = jest diff --git a/packages/api/src/agents/skillFiles.ts b/packages/api/src/agents/skillFiles.ts index 8afb75cf64..26ac74e51e 100644 --- a/packages/api/src/agents/skillFiles.ts +++ b/packages/api/src/agents/skillFiles.ts @@ -18,7 +18,14 @@ import { inspectContent, UninspectableFileError, } from '~/protection'; -import { createConcurrencyLimiter, getCodeApiRetryAfterMs, getSafeErrorMetadata } from '~/utils'; +import { + createCodeApiRateLimitBudget, + getCodeApiUploadOptions, + getSafeErrorMetadata, + isAbortError, + type CodeApiRateLimitBudget, + withCodeApiUploadRecovery, +} from '~/utils'; import { seedCodeFilesIntoSessions, type CodeExecutionProfileRoute } from './codeFilesSession'; import { ContentFilterError, isContentFilterError } from '~/middleware/contentFilter'; import { getCodeExecutionRouteKey, type CodeExecutionContext } from './execution'; @@ -55,8 +62,16 @@ export interface PrimeSkillFilesParams { }; skillFiles: SkillFileRecord[]; req: ServerRequest; + /** Optional operation-wide wait allowance shared by sibling skill primes. */ + rateLimitBudget?: CodeApiRateLimitBudget; + /** Effective foreground cancellation signal for acquisition, waits, and transport. */ + signal?: AbortSignal; getStrategyFunctions: (source: string) => { - getDownloadStream?: (req: ServerRequest, filepath: string) => Promise; + getDownloadStream?: ( + req: ServerRequest, + filepath: string, + options?: { signal?: AbortSignal }, + ) => Promise; [key: string]: unknown; }; batchUploadCodeEnvFiles: (params: { @@ -76,6 +91,7 @@ export interface PrimeSkillFilesParams { codeApiBaseUrl?: string; executionProfile?: CodeExecutionContext['executionProfile']; bridgeWorkerId?: string; + signal?: AbortSignal; }) => Promise<{ storage_session_id: string; files: Array<{ fileId: string; filename: string }>; @@ -89,6 +105,7 @@ export interface PrimeSkillFilesParams { executionProfile?: CodeExecutionContext['executionProfile']; bridgeWorkerId?: string; }, + signal?: AbortSignal, ) => Promise; /** Trusted Code API route selected for the executing agent. */ codeExecutionContext?: Pick< @@ -127,16 +144,21 @@ export interface PrimeSkillFilesResult { }>; } -/** Cap on concurrent skill batch uploads per process. Bounds burst pressure - * on codeapi's per-user upload limiter (default 30 requests / 5 min). */ -const SKILL_UPLOAD_CONCURRENCY = 3; - -/** Retry a 429'd upload only when the server's Retry-After fits under this - * cap; a longer wait would stall a live chat turn worse than degrading. */ -const MAX_RETRY_AFTER_MS = 15_000; - -const uploadSlots = createConcurrencyLimiter(SKILL_UPLOAD_CONCURRENCY); const inflightPrimes = new Map>(); +const signalIds = new WeakMap(); +let nextSignalId = 1; + +function getSignalScope(signal?: AbortSignal): string { + if (!signal) { + return 'unscoped'; + } + let id = signalIds.get(signal); + if (id == null) { + id = nextSignalId++; + signalIds.set(signal, id); + } + return String(id); +} type SkillUploadFiles = Array<{ stream: NodeJS.ReadableStream; filename: string }>; type SkillCodeEnvRef = Extract; @@ -148,19 +170,16 @@ function isCurrentSkillRef( return ref?.kind === 'skill' && ref.version === skillVersion; } -/** Single retry on 429, honoring Retry-After up to MAX_RETRY_AFTER_MS. - * Runs inside an upload slot so the wait also brakes queued uploads. */ -async function retryOn429(attempt: () => Promise, label: string): Promise { +function destroySkillUploadStream(stream: NodeJS.ReadableStream): void { try { - return await attempt(); - } catch (error) { - const retryAfterMs = getCodeApiRetryAfterMs(error); - if (retryAfterMs == null || retryAfterMs > MAX_RETRY_AFTER_MS) { - throw error; + if ( + 'destroy' in stream && + typeof (stream as NodeJS.ReadableStream & { destroy?: () => void }).destroy === 'function' + ) { + (stream as NodeJS.ReadableStream & { destroy: () => void }).destroy(); } - logger.warn(`[primeSkillFiles] Rate-limited priming ${label}; retrying in ${retryAfterMs}ms`); - await new Promise((resolve) => setTimeout(resolve, retryAfterMs)); - return attempt(); + } catch { + /* Preserve the run cancellation even if a provider stream rejects cleanup. */ } } @@ -171,7 +190,8 @@ async function collectSkillUploadFiles( params: PrimeSkillFilesParams, inspectedBuffers: ReadonlyMap, ): Promise { - const { skill, skillFiles, req, getStrategyFunctions } = params; + const { skill, skillFiles, req, getStrategyFunctions, signal } = params; + signal?.throwIfAborted(); const filesToUpload: SkillUploadFiles = []; // SKILL.md from the skill body @@ -184,6 +204,7 @@ async function collectSkillUploadFiles( // Bundled files from storage (parallel stream acquisition) const streamResults = await Promise.allSettled( skillFiles.map(async (file) => { + signal?.throwIfAborted(); const inspected = inspectedBuffers.get(file); if (inspected != null) { return { @@ -198,10 +219,28 @@ async function collectSkillUploadFiles( ); return null; } - const stream = await strategy.getDownloadStream(req, resolveDownloadPath(file)); + const stream = await strategy.getDownloadStream(req, resolveDownloadPath(file), { signal }); + if (signal?.aborted) { + destroySkillUploadStream(stream); + signal.throwIfAborted(); + } return { stream, filename: `${SKILL_FILE_PREFIX}${skill.name}/${file.relativePath}` }; }), ); + /* Do not let allSettled turn foreground cancellation into a skipped bundle + * member. A partial skill upload can look successful while leaving required + * files unavailable to the sandbox. */ + if (signal?.aborted) { + for (const file of filesToUpload) { + destroySkillUploadStream(file.stream); + } + for (const result of streamResults) { + if (result.status === 'fulfilled' && result.value) { + destroySkillUploadStream(result.value.stream); + } + } + signal.throwIfAborted(); + } for (const result of streamResults) { if (result.status === 'fulfilled' && result.value) { filesToUpload.push(result.value); @@ -308,10 +347,14 @@ function throwIfStoredSkillFileMustBeInspectable(req: ServerRequest): void { } } -async function bufferSkillFileStream(stream: NodeJS.ReadableStream): Promise { +async function bufferSkillFileStream( + stream: NodeJS.ReadableStream, + signal?: AbortSignal, +): Promise { const chunks: Buffer[] = []; let bytes = 0; for await (const chunk of stream as AsyncIterable) { + signal?.throwIfAborted(); const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); bytes += buffer.length; if (bytes > MAX_INSPECTABLE_SKILL_FILE_BYTES) { @@ -319,6 +362,7 @@ async function bufferSkillFileStream(stream: NodeJS.ReadableStream): Promise MAX_INSPECTABLE_SKILL_FILE_BYTES) { throwIfStoredSkillFileMustBeInspectable(req); continue; @@ -403,8 +454,10 @@ async function executePrimeSkillFiles( logger.warn('[primeSkillFiles] No download stream for stored skill file'); continue; } - const sourceStream = await strategy.getDownloadStream(req, resolveDownloadPath(file)); - const buffer = await bufferSkillFileStream(sourceStream); + const sourceStream = await strategy.getDownloadStream(req, resolveDownloadPath(file), { + signal, + }); + const buffer = await bufferSkillFileStream(sourceStream, signal); if (buffer == null) { throwIfStoredSkillFileMustBeInspectable(req); continue; @@ -412,6 +465,9 @@ async function executePrimeSkillFiles( assertStoredSkillFileAllowed(file, buffer, req); inspectedBuffers.set(file, buffer); } catch (error) { + if (isAbortError(error)) { + throw error; + } if (isContentFilterError(error)) { throw error; } @@ -445,10 +501,11 @@ async function executePrimeSkillFiles( try { const checkResults = await Promise.all( Array.from(refsBySession.values()).map(async (ref) => { - const lastModified = await getSessionInfo(ref, req, codeExecutionContext); + const lastModified = await getSessionInfo(ref, req, codeExecutionContext, signal); return !!(lastModified && checkIfActive(lastModified)); }), ); + signal?.throwIfAborted(); const allActive = checkResults.every(Boolean); if (allActive) { @@ -480,7 +537,10 @@ async function executePrimeSkillFiles( return { storage_session_id: files[0].storage_session_id, files }; } } - } catch { + } catch (error) { + if (isAbortError(error)) { + throw error; + } // Session check failed — fall through to re-upload } } @@ -488,12 +548,20 @@ async function executePrimeSkillFiles( const entityId = skill._id.toString(); try { - /* Streams open inside the slot (not while queued) and inside the retry - * closure (a failed attempt consumes them). The slot bounds concurrent - * uploads process-wide across both prime call sites. */ - const uploaded = await uploadSlots(() => - retryOn429(async () => { - const filesToUpload = await collectSkillUploadFiles(params, inspectedBuffers); + const uploadOptions = getCodeApiUploadOptions(req, executionRouteKey); + const uploaded = await withCodeApiUploadRecovery({ + registry: req.app.locals.codeApiUploadRegistry, + scope: uploadOptions.scope, + concurrency: uploadOptions.concurrency, + label: `priming skill "${skill.name}"`, + budget: rateLimitBudget ?? createCodeApiRateLimitBudget(uploadOptions.retryWaitMs), + signal, + onWait: (waitMs) => + logger.warn( + `[primeSkillFiles] Rate-limited priming skill "${skill.name}"; retrying in ${waitMs}ms`, + ), + openSource: () => collectSkillUploadFiles(params, inspectedBuffers), + upload: async (filesToUpload) => { if (filesToUpload.length === 0) { return null; } @@ -503,7 +571,7 @@ async function executePrimeSkillFiles( /* Resource identity for codeapi's sessionKey: skill files share * cross-user-within-tenant under `:skill::v:`. * Bumping `skill.version` on edit naturally invalidates the prior - * cache entry under the new sessionKey. */ + * cache entry under the new version's sessionKey. */ kind: 'skill', id: entityId, version: skill.version, @@ -518,10 +586,11 @@ async function executePrimeSkillFiles( codeApiBaseUrl: codeExecutionContext?.baseUrl, executionProfile: codeExecutionContext?.executionProfile, bridgeWorkerId: codeExecutionContext?.bridgeWorkerId, + signal, }); return { filesToUpload, result }; - }, `skill "${skill.name}"`), - ); + }, + }); if (uploaded == null) { return null; } @@ -608,6 +677,9 @@ async function executePrimeSkillFiles( return { storage_session_id: result.storage_session_id, files }; } catch (error) { + if (isAbortError(error)) { + throw error; + } logger.error('[primeSkillFiles] Batch upload failed', getSafeErrorMetadata(error)); return null; } @@ -615,6 +687,8 @@ async function executePrimeSkillFiles( export interface PrimeInvokedSkillsDeps { req: ServerRequest; + /** Effective cancellation signal for the historical priming operation. */ + signal?: AbortSignal; /** Raw message payload (before formatAgentMessages). Used to extract invoked skill names. */ payload?: Array>; /** Explicit durable names used by a validated event-actor preflight. */ @@ -642,6 +716,7 @@ export interface PrimeInvokedSkillsDeps { checkIfActive?: PrimeSkillFilesParams['checkIfActive']; updateSkillFileCodeEnvIds?: PrimeSkillFilesParams['updateSkillFileCodeEnvIds']; codeExecutionContext?: PrimeSkillFilesParams['codeExecutionContext']; + rateLimitBudget?: CodeApiRateLimitBudget; } export interface PrimeInvokedSkillsResult { @@ -781,13 +856,19 @@ export async function primeInvokedSkills( ref, deps.req, deps.codeExecutionContext, + deps.signal, ); + deps.signal?.throwIfAborted(); return !!(lastModified && deps.checkIfActive?.(lastModified)); - } catch { + } catch (error) { + if (isAbortError(error)) { + throw error; + } return false; } }), ); + deps.signal?.throwIfAborted(); const allActive = checkResults.every(Boolean); if (allActive) { @@ -844,6 +925,9 @@ export async function primeInvokedSkills( kind: 'skill'; version: number; }> = []; + const uploadOptions = getCodeApiUploadOptions(deps.req, executionRouteKey); + const rateLimitBudget = + deps.rateLimitBudget ?? createCodeApiRateLimitBudget(uploadOptions.retryWaitMs); const primeResults = await Promise.allSettled( fileListResults.map(async ({ skill, files }) => { const result = await primeSkillFiles({ @@ -856,6 +940,8 @@ export async function primeInvokedSkills( checkIfActive: deps.checkIfActive, updateSkillFileCodeEnvIds: deps.updateSkillFileCodeEnvIds, codeExecutionContext: deps.codeExecutionContext, + rateLimitBudget, + signal: deps.signal, }); return { skill, result }; }), @@ -876,7 +962,7 @@ export async function primeInvokedSkills( }); } } else if (r.status === 'rejected') { - if (isContentFilterError(r.reason)) { + if (isContentFilterError(r.reason) || isAbortError(r.reason)) { throw r.reason; } logger.warn( @@ -921,6 +1007,10 @@ export async function primeInvokedSkillsForProfiles( return primeInvokedSkills({ ...deps, codeEnvAvailable: false }); } + const rateLimitBudget = createCodeApiRateLimitBudget( + deps.req.config?.endpoints?.agents?.codeApiMaxRetryWaitMs, + ); + const profileResults = await Promise.all( deps.executionProfiles.map(async (profile) => ({ profile, @@ -929,6 +1019,8 @@ export async function primeInvokedSkillsForProfiles( codeEnvAvailable: true, codeExecutionContext: profile.codeExecutionContext, updateSkillFileCodeEnvIds: deps.updateSkillFileCodeEnvIds, + rateLimitBudget, + signal: deps.signal, }), })), ); diff --git a/packages/api/src/files/provision/callback.spec.ts b/packages/api/src/files/provision/callback.spec.ts index cac57da4ab..a17dc5bb9f 100644 --- a/packages/api/src/files/provision/callback.spec.ts +++ b/packages/api/src/files/provision/callback.spec.ts @@ -353,6 +353,83 @@ describe('createProvisionFilesCallback', () => { expect(provisionToCodeEnv).not.toHaveBeenCalled(); }); + it('preserves cancellation raised during code provisioning and skips search work', async () => { + const controller = new AbortController(); + const codeImpl = jest.fn(async () => { + controller.abort(); + controller.signal.throwIfAborted(); + }); + const vectorImpl = jest.fn(); + const { provisionFiles } = buildHarness({ + contexts: [ + ['agent-a', { provisionState: state([makeFile()], [makeFile({ file_id: 'search-1' })]) }], + ], + codeImpl, + vectorImpl, + }); + + await expect( + provisionFiles([Constants.EXECUTE_CODE, 'file_search'], 'agent-a', controller.signal), + ).rejects.toMatchObject({ name: 'AbortError' }); + expect(vectorImpl).not.toHaveBeenCalled(); + }); + + it('preserves cancellation raised during vector provisioning', async () => { + const controller = new AbortController(); + const vectorImpl = jest.fn(async () => { + controller.abort(); + controller.signal.throwIfAborted(); + }); + const { provisionFiles } = buildHarness({ + contexts: [['agent-a', { provisionState: state([], [makeFile()]) }]], + vectorImpl, + }); + + await expect( + provisionFiles(['file_search'], 'agent-a', controller.signal), + ).rejects.toMatchObject({ name: 'AbortError' }); + }); + + it('preserves a homogeneous Code API rate-limit failure from lazy provisioning', async () => { + const rateLimit = Object.assign(new Error('rate limited'), { + name: 'CodeApiRateLimitError', + code: 'CODE_API_RATE_LIMITED', + status: 429, + statusCode: 429, + retryAfterMs: 30_000, + }); + const { provisionFiles } = buildHarness({ + contexts: [['agent-a', { provisionState: state([makeFile()], []) }]], + codeImpl: jest.fn().mockRejectedValue(rateLimit), + }); + + await expect(provisionFiles([Constants.EXECUTE_CODE], 'agent-a')).rejects.toBe(rateLimit); + }); + + it('retains every failure when lazy code provisioning has mixed causes', async () => { + const first = new Error('storage unavailable'); + const second = new Error('code api unavailable'); + const { provisionFiles } = buildHarness({ + contexts: [ + [ + 'agent-a', + { + provisionState: state( + [makeFile({ file_id: 'first' }), makeFile({ file_id: 'second' })], + [], + ), + }, + ], + ], + codeImpl: jest.fn().mockRejectedValueOnce(first).mockRejectedValueOnce(second), + }); + + await expect(provisionFiles([Constants.EXECUTE_CODE], 'agent-a')).rejects.toMatchObject({ + cause: first, + errors: [first, second], + }); + }); + it('shares embedding work across agents queueing the same search file', async () => { const shared = makeFile(); const { provisionFiles, provisionToVectorDB, addEmbeddedEntity } = buildHarness({ diff --git a/packages/api/src/files/provision/callback.ts b/packages/api/src/files/provision/callback.ts index d76e7090a3..714cbb4ac4 100644 --- a/packages/api/src/files/provision/callback.ts +++ b/packages/api/src/files/provision/callback.ts @@ -7,6 +7,7 @@ import type { CodeEnvRefUpdate, CodeExecutionRoute, ProvisionService } from './s import type { ProvisionState } from '~/agents/resources'; import type { ServerRequest } from '~/types'; import { claimCodeDestination, createCodeDestinationSet } from '~/files/code/destinations'; +import { createCodeApiRateLimitBudget, isCodeApiRateLimitError } from '~/utils'; import { isCodeFileToolName } from '~/agents/tools'; /** Deferred database write produced by a successful provisioning call. */ @@ -238,6 +239,7 @@ export function createProvisionFilesCallback({ /** Files whose provisioning rejected this turn; kept queued so a transient * outage can retry next turn instead of being silently dropped. */ const failedCodeFiles: TFile[] = []; + const codeFailureReasons: unknown[] = []; const failedVectorFiles: TFile[] = []; /* The graph seeded each tool call's code-session context from the sessions that * existed at run start, which predate this upload. Returning the refs lets the @@ -248,6 +250,11 @@ export function createProvisionFilesCallback({ : []; if (needsCode && provisionState.codeEnvFiles.length > 0) { const queuedCodeFiles = provisionState.codeEnvFiles; + /** Every file in this tool-load batch shares one wait allowance. This + * prevents a large recovery set from multiplying the live-turn delay. */ + const codeApiRateLimitBudget = createCodeApiRateLimitBudget( + req.config?.endpoints?.agents?.codeApiMaxRetryWaitMs, + ); const destinations = createCodeDestinationSet(); const existingCodeFiles = ( ctx.tool_resources as Record @@ -287,6 +294,7 @@ export function createProvisionFilesCallback({ route: ctx.codeExecutionContext, sandboxFilename, signal, + rateLimitBudget: codeApiRateLimitBudget, }); signal?.throwIfAborted(); /* primeCodeFiles re-reads the database and skips files without a stored @@ -334,10 +342,15 @@ export function createProvisionFilesCallback({ } }), ); + /* allSettled keeps independent file failures retryable, but cancellation belongs + * to the whole tool run. Preserve it before translating rejections into queued + * files or starting search provisioning. */ + signal?.throwIfAborted(); results.forEach((result, index) => { if (result.status === 'rejected') { logger.error('[provisionFiles] Code env provisioning failed', result.reason); failedCodeFiles.push(queuedCodeFiles[index]); + codeFailureReasons.push(result.reason); } }); provisionState.codeEnvFiles = failedCodeFiles; @@ -403,6 +416,7 @@ export function createProvisionFilesCallback({ throw new Error(`Vector store did not embed "${file.filename}" (${file.file_id})`); }), ); + signal?.throwIfAborted(); results.forEach((result, index) => { if (result.status === 'rejected') { logger.error('[provisionFiles] Vector DB provisioning failed', result.reason); @@ -419,8 +433,15 @@ export function createProvisionFilesCallback({ /* Provisioning failed outright, so the sandbox or vector store does not have the * attachment; running the tool anyway would answer from missing input. */ if (failedCodeFiles.length > 0) { - throw new Error( - `Failed to provision ${failedCodeFiles.length} file(s) to the code environment; aborting tool execution rather than running without them`, + if (codeFailureReasons.length > 0 && codeFailureReasons.every(isCodeApiRateLimitError)) { + throw codeFailureReasons[0]; + } + throw Object.assign( + new Error( + `Failed to provision ${failedCodeFiles.length} file(s) to the code environment; aborting tool execution rather than running without them`, + codeFailureReasons[0] !== undefined ? { cause: codeFailureReasons[0] } : undefined, + ), + { errors: codeFailureReasons }, ); } if (failedVectorFiles.length > 0) { diff --git a/packages/api/src/files/provision/service.spec.ts b/packages/api/src/files/provision/service.spec.ts index 17fd593532..03b413302c 100644 --- a/packages/api/src/files/provision/service.spec.ts +++ b/packages/api/src/files/provision/service.spec.ts @@ -1,4 +1,5 @@ import { Readable } from 'node:stream'; +import { AxiosError, AxiosHeaders } from 'axios'; import type { TFile } from 'librechat-data-provider'; import type { ServerRequest } from '~/types'; @@ -20,9 +21,13 @@ jest.mock('@librechat/agents', () => ({ getCodeBaseURL: () => 'http://code.test/v1', })); +import { createCodeApiUploadRegistry } from '~/utils'; import { createProvisionService } from './service'; -const req = { user: { id: 'u1' } } as unknown as ServerRequest; +const req = { + user: { id: 'u1' }, + app: { locals: { codeApiUploadRegistry: createCodeApiUploadRegistry() } }, +} as unknown as ServerRequest; const makeFile = (overrides: Partial = {}): TFile => ({ @@ -73,6 +78,48 @@ describe('createProvisionService', () => { }); describe('provisionToCodeEnv', () => { + it('waits out Code API throttling and reopens the upload stream', async () => { + const rateLimit = new AxiosError('Request failed', 'ERR_BAD_REQUEST'); + rateLimit.response = { + status: 429, + statusText: 'Too Many Requests', + headers: new AxiosHeaders({ 'retry-after': '0' }), + config: { headers: new AxiosHeaders() }, + data: {}, + }; + const streams = [Readable.from('first'), Readable.from('second')]; + const getDownloadStream = jest + .fn() + .mockResolvedValueOnce(streams[0]) + .mockResolvedValueOnce(streams[1]); + const uploadCodeEnvFile = jest + .fn() + .mockRejectedValueOnce(rateLimit) + .mockResolvedValueOnce({ storage_session_id: 's1', file_id: 'remote-1' }); + const { service } = buildService({ + getStrategyFunctions: (source: string) => + source === 'execute_code' + ? { handleFileUpload: uploadCodeEnvFile } + : { getDownloadStream }, + }); + + await expect( + service.provisionToCodeEnv({ + req, + file: makeFile(), + rateLimitBudget: { + limitMs: 2_000, + waitedMs: 0, + activeWaitEnds: new Set(), + }, + }), + ).resolves.toMatchObject({ referenceSet: { codeEnvRef: { file_id: 'remote-1' } } }); + + expect(uploadCodeEnvFile).toHaveBeenCalledTimes(2); + expect(getDownloadStream).toHaveBeenCalledTimes(2); + expect(uploadCodeEnvFile.mock.calls.map(([args]) => args.stream)).toEqual(streams); + }); + it('renames a converted image to match its stored MIME type', async () => { const { service, uploadCodeEnvFile } = buildService(); @@ -112,9 +159,14 @@ describe('createProvisionService', () => { it('cancels storage acquisition and destroys a stream returned after cancellation', async () => { const controller = new AbortController(); let resolveDownload!: (stream: Readable) => void; + let markDownloadStarted!: () => void; + const downloadStarted = new Promise((resolve) => { + markDownloadStarted = resolve; + }); const getDownloadStream = jest.fn( () => new Promise((resolve) => { + markDownloadStarted(); resolveDownload = resolve; }), ); @@ -128,6 +180,7 @@ describe('createProvisionService', () => { signal: controller.signal, }); + await downloadStarted; controller.abort(); const stream = new Readable({ read() {} }); const destroy = jest.spyOn(stream, 'destroy'); diff --git a/packages/api/src/files/provision/service.ts b/packages/api/src/files/provision/service.ts index 4a6f680b23..a557a768c1 100644 --- a/packages/api/src/files/provision/service.ts +++ b/packages/api/src/files/provision/service.ts @@ -14,12 +14,16 @@ import { import type { CodeEnvRef, CodeEnvRefMap, TFile } from 'librechat-data-provider'; import type { Readable } from 'node:stream'; import type { CodeEnvIdentity } from '~/files/code/identity'; +import type { CodeApiRateLimitBudget } from '~/utils'; import type { ServerRequest } from '~/types'; import { logAxiosError, createAxiosInstance, + createCodeApiRateLimitBudget, codeServerHttpAgent, codeServerHttpsAgent, + getCodeApiUploadOptions, + withCodeApiUploadRecovery, } from '~/utils'; import { buildCodeEnvIdentityParams } from '~/files/code/identity'; import { getCodeApiAuthHeaders } from '~/auth/codeapi'; @@ -89,6 +93,7 @@ export interface ProvisionService { route?: CodeExecutionRoute; sandboxFilename?: string; signal?: AbortSignal; + rateLimitBudget?: CodeApiRateLimitBudget; }) => Promise<{ referenceSet: CodeEnvReferenceSetResult; refUpdate: CodeEnvRefUpdate; @@ -237,6 +242,7 @@ export function createProvisionService({ route, sandboxFilename: requestedSandboxFilename, signal, + rateLimitBudget, }: { req: ServerRequest; file: TFile; @@ -244,20 +250,13 @@ export function createProvisionService({ route?: CodeExecutionRoute; sandboxFilename?: string; signal?: AbortSignal; + rateLimitBudget?: CodeApiRateLimitBudget; }) { signal?.throwIfAborted(); const { handleFileUpload: uploadCodeEnvFile } = getStrategyFunctions(FileSources.execute_code); if (!uploadCodeEnvFile) { throw new Error('Code environment upload strategy is unavailable'); } - const stream = await getStorageStream(file, req, signal); - signal?.throwIfAborted(); - if (!stream) { - throw new Error( - `Cannot provision file "${file.filename}" to code env: storage source "${file.source}" does not support download streams`, - ); - } - const kind: 'agent' | 'user' = entity_id ? 'agent' : 'user'; const id = entity_id ?? (req.user?.id as string); @@ -268,15 +267,42 @@ export function createProvisionService({ const executionProfile = route?.executionProfile ?? 'default'; const sandboxFilename = requestedSandboxFilename ?? resolveSandboxFilename(file.filename, file.type); - const uploaded = await uploadCodeEnvFile({ + const uploadOptions = getCodeApiUploadOptions( req, - stream, - filename: sandboxFilename, - kind, - id, - ...(route?.baseUrl ? { codeApiBaseUrl: route.baseUrl } : {}), - executionProfile, + route?.executionRouteKey ?? route?.baseUrl ?? executionProfile, + ); + const uploaded = await withCodeApiUploadRecovery({ + registry: req.app.locals.codeApiUploadRegistry, + scope: uploadOptions.scope, + concurrency: uploadOptions.concurrency, + label: `uploading "${file.filename}" to the code environment`, + budget: rateLimitBudget ?? createCodeApiRateLimitBudget(uploadOptions.retryWaitMs), signal, + onWait: (waitMs) => + logger.warn( + `[provisionToCodeEnv] Rate-limited uploading file ${file.file_id}; retrying in ${waitMs}ms`, + ), + openSource: async () => { + const stream = await getStorageStream(file, req, signal); + signal?.throwIfAborted(); + if (!stream) { + throw new Error( + `Cannot provision file "${file.filename}" to code env: storage source "${file.source}" does not support download streams`, + ); + } + return stream; + }, + upload: (stream) => + uploadCodeEnvFile({ + req, + stream, + filename: sandboxFilename, + kind, + id, + ...(route?.baseUrl ? { codeApiBaseUrl: route.baseUrl } : {}), + executionProfile, + signal, + }), }); signal?.throwIfAborted(); diff --git a/packages/api/src/utils/axios.ts b/packages/api/src/utils/axios.ts index 918e841d9e..d5e79d1a49 100644 --- a/packages/api/src/utils/axios.ts +++ b/packages/api/src/utils/axios.ts @@ -87,6 +87,19 @@ export const logAxiosError = ({ return logMessage; }; +/** Adds upload context without discarding the Axios response metadata used by + * Code API recovery. Transport-specific branching remains in this typed module. */ +export function wrapCodeApiUploadError(error: unknown, message: string): Error { + const wrapped = new Error(logAxiosError({ message, error }), { cause: error }); + if (axios.isAxiosError(error)) { + Object.assign(wrapped, { + isAxiosError: true, + response: error.response, + }); + } + return wrapped; +} + /** * Creates and configures an Axios instance with optional proxy settings. diff --git a/packages/api/src/utils/code.spec.ts b/packages/api/src/utils/code.spec.ts index be4d25bfa3..f471114224 100644 --- a/packages/api/src/utils/code.spec.ts +++ b/packages/api/src/utils/code.spec.ts @@ -1,5 +1,14 @@ import { AxiosError, AxiosHeaders } from 'axios'; -import { getCodeApiRetryAfterMs, withCodeApiRateLimit, createCodeApiRateLimitBudget } from './code'; +import { + getCodeApiRetryAfterMs, + getCodeApiUploadOptions, + withCodeApiRateLimit, + withCodeApiUploadRecovery, + withCodeApiUploadSlot, + createCodeApiRateLimitBudget, + createCodeApiUploadRegistry, +} from './code'; +import { wrapCodeApiUploadError } from './axios'; /** Builds the error axios raises for a Code API rate-limit response. */ function rateLimited({ @@ -22,6 +31,18 @@ function rateLimited({ return error; } +describe('wrapCodeApiUploadError', () => { + it('preserves Axios rate-limit metadata on the contextual error', () => { + const cause = rateLimited({ headers: { 'retry-after': '7' } }); + + expect(wrapCodeApiUploadError(cause, 'Upload failed')).toMatchObject({ + cause, + isAxiosError: true, + response: { status: 429 }, + }); + }); +}); + describe('getCodeApiRetryAfterMs', () => { it('reads the standard Retry-After header', () => { expect(getCodeApiRetryAfterMs(rateLimited({ headers: { 'retry-after': '17' } }))).toBe(17_000); @@ -88,7 +109,7 @@ describe('withCodeApiRateLimit', () => { expect(attempts).toBe(2); expect(waits).toEqual([1_000]); - expect(budget.remainingMs).toBe(4_000); + expect(budget.waitedMs).toBeGreaterThanOrEqual(1_000); }); it('names the limit when the wait exceeds the remaining budget', async () => { @@ -99,9 +120,15 @@ describe('withCodeApiRateLimit', () => { await expect( withCodeApiRateLimit({ attempt, label: 'reading "x.png"', budget }), - ).rejects.toThrow(/rate limit reached while reading "x\.png" \(retry in 300s\)/); + ).rejects.toMatchObject({ + name: 'CodeApiRateLimitError', + code: 'CODE_API_RATE_LIMITED', + status: 429, + statusCode: 429, + retryAfterMs: 300_000, + }); expect(attempt).toHaveBeenCalledTimes(1); - expect(budget.remainingMs).toBe(5_000); + expect(budget.waitedMs).toBe(0); }); it('names the limit even when the response carries no usable delay', async () => { @@ -117,10 +144,15 @@ describe('withCodeApiRateLimit', () => { label: 'reading', budget: createCodeApiRateLimitBudget(60_000), }), - ).rejects.toThrow(/rate limit reached while reading\./); + ).rejects.toMatchObject({ + name: 'CodeApiRateLimitError', + code: 'CODE_API_RATE_LIMITED', + status: 429, + statusCode: 429, + }); }); - it('charges at least a second so a zero delay cannot spin', async () => { + it('waits at least a second so a zero delay cannot spin', async () => { const budget = createCodeApiRateLimitBudget(2_500); const waits: number[] = []; let attempts = 0; @@ -136,7 +168,6 @@ describe('withCodeApiRateLimit', () => { withCodeApiRateLimit({ attempt, label: 'reading', budget, onWait: (ms) => waits.push(ms) }), ).resolves.toBe('ok'); expect(waits).toEqual([1_000, 1_000]); - expect(budget.remainingMs).toBe(500); }); it('rethrows anything that is not a rate limit', async () => { @@ -147,4 +178,250 @@ describe('withCodeApiRateLimit', () => { 'Request failed', ); }); + + it('stops waiting when the calling operation is cancelled', async () => { + const controller = new AbortController(); + const attempt = jest.fn(async () => { + throw rateLimited({ headers: { 'retry-after': '10' } }); + }); + + const pending = withCodeApiRateLimit({ + attempt, + label: 'uploading', + budget: createCodeApiRateLimitBudget(20_000), + signal: controller.signal, + onWait: () => controller.abort(), + }); + + await expect(pending).rejects.toMatchObject({ name: 'AbortError' }); + expect(attempt).toHaveBeenCalledTimes(1); + }); + + it('shares elapsed wall time across concurrent waits', async () => { + const budget = createCodeApiRateLimitBudget(2_500); + const attempts = [0, 0, 0]; + const run = (index: number) => + withCodeApiRateLimit({ + budget, + label: `uploading ${index}`, + attempt: async () => { + if (attempts[index]++ === 0) { + throw rateLimited({ headers: { 'retry-after': '0' } }); + } + return index; + }, + }); + + await expect(Promise.all([run(0), run(1), run(2)])).resolves.toEqual([0, 1, 2]); + }); + + it('does not charge successful request execution against the wait budget', async () => { + jest.useFakeTimers(); + try { + const budget = createCodeApiRateLimitBudget(1_500); + let attempts = 0; + const pending = withCodeApiRateLimit({ + budget, + label: 'reading windows', + attempt: async () => { + attempts++; + if (attempts === 1) { + await new Promise((resolve) => setTimeout(resolve, 30_000)); + throw rateLimited({ headers: { 'retry-after': '1' } }); + } + return 'ok'; + }, + }); + + await jest.advanceTimersByTimeAsync(30_000); + await jest.advanceTimersByTimeAsync(1_000); + await expect(pending).resolves.toBe('ok'); + expect(budget.waitedMs).toBe(1_000); + } finally { + jest.useRealTimers(); + } + }); +}); + +describe('withCodeApiUploadSlot', () => { + it('removes a canceled caller from a busy scope immediately', async () => { + const registry = createCodeApiUploadRegistry(); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const first = withCodeApiUploadSlot({ + registry, + scope: 'route:user-a', + concurrency: 1, + task: () => gate, + }); + const controller = new AbortController(); + const queuedTask = jest.fn(async () => undefined); + const queued = withCodeApiUploadSlot({ + registry, + scope: 'route:user-a', + concurrency: 1, + signal: controller.signal, + task: queuedTask, + }); + + controller.abort(); + await expect(queued).rejects.toMatchObject({ name: 'AbortError' }); + expect(queuedTask).not.toHaveBeenCalled(); + release(); + await first; + }); + + it('does not block a different principal and route scope', async () => { + const registry = createCodeApiUploadRegistry(); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const first = withCodeApiUploadSlot({ + registry, + scope: 'route-a:user-a', + concurrency: 1, + task: () => gate, + }); + const independent = jest.fn(async () => 'ok'); + + await expect( + withCodeApiUploadSlot({ + registry, + scope: 'route-b:user-b', + concurrency: 1, + task: independent, + }), + ).resolves.toBe('ok'); + expect(independent).toHaveBeenCalledTimes(1); + release(); + await first; + }); +}); + +describe('withCodeApiUploadRecovery', () => { + it('disposes a failed source before opening the retry source', async () => { + jest.useFakeTimers(); + try { + const first = { destroy: jest.fn() }; + const second = { destroy: jest.fn() }; + const openSource = jest + .fn, []>() + .mockResolvedValueOnce(first) + .mockResolvedValueOnce(second); + const upload = jest + .fn, [typeof first]>() + .mockRejectedValueOnce(rateLimited({ headers: { 'retry-after': '0' } })) + .mockResolvedValueOnce('ok'); + + const pending = withCodeApiUploadRecovery({ + registry: createCodeApiUploadRegistry(), + scope: 'route:user', + budget: createCodeApiRateLimitBudget(2_000), + label: 'uploading', + openSource, + upload, + }); + await jest.advanceTimersByTimeAsync(1_000); + + await expect(pending).resolves.toBe('ok'); + expect(first.destroy).toHaveBeenCalledTimes(1); + expect(second.destroy).not.toHaveBeenCalled(); + expect(upload.mock.calls.map(([source]) => source)).toEqual([first, second]); + } finally { + jest.useRealTimers(); + } + }); + + it('disposes every stream in a failed batch source', async () => { + const first = { stream: { destroy: jest.fn() } }; + const second = { stream: { destroy: jest.fn() } }; + + await expect( + withCodeApiUploadRecovery({ + registry: createCodeApiUploadRegistry(), + scope: 'route:user', + budget: createCodeApiRateLimitBudget(0), + label: 'uploading', + openSource: async () => [first, second], + upload: async () => { + throw rateLimited({ headers: { 'retry-after': '30' } }); + }, + }), + ).rejects.toMatchObject({ code: 'CODE_API_RATE_LIMITED' }); + expect(first.stream.destroy).toHaveBeenCalledTimes(1); + expect(second.stream.destroy).toHaveBeenCalledTimes(1); + }); + + it('does not classify a backing-storage 429 as Code API throttling', async () => { + const storageRateLimit = rateLimited({ headers: { 'retry-after': '1' } }); + const upload = jest.fn(); + + await expect( + withCodeApiUploadRecovery({ + registry: createCodeApiUploadRegistry(), + scope: 'route:user', + budget: createCodeApiRateLimitBudget(2_000), + label: 'uploading', + openSource: async () => { + throw storageRateLimit; + }, + upload, + }), + ).rejects.toBe(storageRateLimit); + expect(upload).not.toHaveBeenCalled(); + }); + + it('disposes a source acquired as cancellation arrives', async () => { + const controller = new AbortController(); + const source = { destroy: jest.fn() }; + const upload = jest.fn(); + + await expect( + withCodeApiUploadRecovery({ + registry: createCodeApiUploadRegistry(), + scope: 'route:user', + signal: controller.signal, + label: 'uploading', + openSource: async () => { + controller.abort(); + return source; + }, + upload, + }), + ).rejects.toMatchObject({ name: 'AbortError' }); + expect(source.destroy).toHaveBeenCalledTimes(1); + expect(upload).not.toHaveBeenCalled(); + }); +}); + +describe('getCodeApiUploadOptions', () => { + it('uses the configured limit and isolates tenants, principals, and routes', () => { + const first = getCodeApiUploadOptions( + { + user: { id: 'user-a', tenantId: 'tenant-a' }, + config: { endpoints: { agents: { codeApiUploadConcurrency: 7 } } }, + } as never, + 'route-a', + ); + const second = getCodeApiUploadOptions( + { user: { id: 'user-a', tenantId: 'tenant-b' } } as never, + 'route-a', + ); + + expect(first.concurrency).toBe(7); + expect(second.concurrency).toBe(3); + expect(first.retryWaitMs).toBe(20_000); + expect(first.scope).not.toBe(second.scope); + expect(first.scope).not.toBe( + getCodeApiUploadOptions({ user: { id: 'user-b', tenantId: 'tenant-a' } } as never, 'route-a') + .scope, + ); + expect(first.scope).not.toBe( + getCodeApiUploadOptions({ user: { id: 'user-a', tenantId: 'tenant-a' } } as never, 'route-b') + .scope, + ); + }); }); diff --git a/packages/api/src/utils/code.ts b/packages/api/src/utils/code.ts index 42931b5bca..344c1d07ac 100644 --- a/packages/api/src/utils/code.ts +++ b/packages/api/src/utils/code.ts @@ -1,6 +1,8 @@ import http from 'http'; import https from 'https'; import { isAxiosError } from 'axios'; +import { setTimeout as delay } from 'node:timers/promises'; +import type { ServerRequest } from '~/types'; /** * Dedicated agents for code-server requests, preventing socket pool contamination. @@ -41,18 +43,223 @@ export function getCodeApiRetryAfterMs(error: unknown): number | null { } /** Total time one operation may spend waiting out Code API rate limits. - * Sized to cover a single limiter window without stalling a chat turn. */ + * Keeps recovery bounded inside a live chat turn. */ export const MAX_CODE_API_RATE_LIMIT_WAIT_MS = 20_000; +export const CODE_API_UPLOAD_CONCURRENCY_DEFAULT = 3; -/** Remaining wait allowance, shared by every request of one operation. */ +export interface CodeApiUploadRegistry { + scopes: Map; +} + +export function createCodeApiUploadRegistry(): CodeApiUploadRegistry { + return { scopes: new Map() }; +} + +export function createCodeApiUploadScope(params: { + route: string; + principalId: string; + tenantId?: string; +}): string { + return JSON.stringify([params.route, params.tenantId ?? 'legacy', params.principalId]); +} + +export function getCodeApiUploadOptions( + req: ServerRequest, + route: string, +): { scope: string; concurrency: number; retryWaitMs: number } { + const user = req.user as typeof req.user & { + _id?: { toString(): string } | string; + tenantId?: unknown; + orgId?: unknown; + }; + const principalId = user?.id ?? user?._id?.toString(); + if (!principalId) { + throw new Error('Code API upload recovery requires an authenticated principal'); + } + return { + scope: createCodeApiUploadScope({ + route, + principalId, + tenantId: + user.tenantId == null && user.orgId == null + ? undefined + : String(user.tenantId ?? user.orgId), + }), + concurrency: + req.config?.endpoints?.agents?.codeApiUploadConcurrency ?? + CODE_API_UPLOAD_CONCURRENCY_DEFAULT, + retryWaitMs: + req.config?.endpoints?.agents?.codeApiMaxRetryWaitMs ?? MAX_CODE_API_RATE_LIMIT_WAIT_MS, + }; +} + +interface QueuedUpload { + run: () => void; + reject: (error: unknown) => void; + signal?: AbortSignal; + onAbort?: () => void; +} + +interface UploadScopeState { + active: number; + concurrency: number; + queue: QueuedUpload[]; +} + +interface CodeApiRateLimitWaitToken { + endAt: number; +} + +function abortReason(signal: AbortSignal): unknown { + try { + signal.throwIfAborted(); + } catch (error) { + return error; + } + return new Error('The operation was aborted'); +} + +function drainCodeApiUploadScope( + registry: CodeApiUploadRegistry, + scope: string, + state: UploadScopeState, +): void { + while (state.active < state.concurrency && state.queue.length > 0) { + const queued = state.queue.shift(); + if (!queued) { + break; + } + queued.signal?.removeEventListener('abort', queued.onAbort!); + if (queued.signal?.aborted) { + queued.reject(abortReason(queued.signal)); + continue; + } + queued.run(); + } + if (state.active === 0 && state.queue.length === 0) { + registry.scopes.delete(scope); + } +} + +/** Limits uploads within one Code API route and principal. A throttled bucket + * holds only its own slots, and canceled callers leave the queue immediately. */ +export function withCodeApiUploadSlot(params: { + registry: CodeApiUploadRegistry; + task: () => Promise; + scope: string; + concurrency?: number; + signal?: AbortSignal; +}): Promise { + const { + registry, + task, + scope, + concurrency = CODE_API_UPLOAD_CONCURRENCY_DEFAULT, + signal, + } = params; + if (!Number.isInteger(concurrency) || concurrency < 1) { + throw new Error(`Code API upload concurrency must be a positive integer (got ${concurrency})`); + } + signal?.throwIfAborted(); + let state = registry.scopes.get(scope); + if (!state) { + state = { active: 0, concurrency, queue: [] }; + registry.scopes.set(scope, state); + } else { + state.concurrency = concurrency; + } + + return new Promise((resolve, reject) => { + const run = (): void => { + state.active++; + Promise.resolve() + .then(task) + .then(resolve, reject) + .finally(() => { + state.active--; + drainCodeApiUploadScope(registry, scope, state); + }); + }; + if (state.active < state.concurrency) { + run(); + return; + } + const queued: QueuedUpload = { run, reject, signal }; + queued.onAbort = () => { + const index = state.queue.indexOf(queued); + if (index >= 0) { + state.queue.splice(index, 1); + reject(abortReason(signal!)); + drainCodeApiUploadScope(registry, scope, state); + } + }; + signal?.addEventListener('abort', queued.onAbort, { once: true }); + state.queue.push(queued); + }); +} + +/** Actual rate-limit wait time shared by every request of one operation. */ export interface CodeApiRateLimitBudget { - remainingMs: number; + limitMs: number; + waitedMs: number; + waitingSince?: number; + activeWaitEnds: Set; } export function createCodeApiRateLimitBudget( totalMs: number = MAX_CODE_API_RATE_LIMIT_WAIT_MS, ): CodeApiRateLimitBudget { - return { remainingMs: totalMs }; + return { limitMs: totalMs, waitedMs: 0, activeWaitEnds: new Set() }; +} + +function beginCodeApiRateLimitWait( + budget: CodeApiRateLimitBudget, + waitMs: number, +): CodeApiRateLimitWaitToken | null { + const now = Date.now(); + const activeElapsed = budget.waitingSince == null ? 0 : now - budget.waitingSince; + const currentWaitEnd = Math.max( + now, + ...Array.from(budget.activeWaitEnds, (token) => token.endAt), + ); + const projectedWaitedMs = + budget.waitedMs + activeElapsed + Math.max(currentWaitEnd, now + waitMs) - now; + if (projectedWaitedMs > budget.limitMs) { + return null; + } + const token = { endAt: now + waitMs }; + if (budget.activeWaitEnds.size === 0) { + budget.waitingSince = now; + } + budget.activeWaitEnds.add(token); + return token; +} + +function finishCodeApiRateLimitWait( + budget: CodeApiRateLimitBudget, + token: CodeApiRateLimitWaitToken, +): void { + if (!budget.activeWaitEnds.delete(token)) { + return; + } + if (budget.activeWaitEnds.size === 0 && budget.waitingSince != null) { + budget.waitedMs += Date.now() - budget.waitingSince; + budget.waitingSince = undefined; + } +} + +function codeApiRateLimitError(label: string, cause: unknown, retryAfterMs?: number): Error { + const retrySuffix = retryAfterMs == null ? '' : ` (retry in ${Math.ceil(retryAfterMs / 1000)}s)`; + return Object.assign( + new Error(`Code API rate limit reached while ${label}${retrySuffix}.`, { cause }), + { + name: 'CodeApiRateLimitError', + code: 'CODE_API_RATE_LIMITED', + status: 429, + statusCode: 429, + ...(retryAfterMs == null ? {} : { retryAfterMs }), + }, + ); } /** @@ -69,30 +276,134 @@ export async function withCodeApiRateLimit(params: { label: string; budget?: CodeApiRateLimitBudget; onWait?: (waitMs: number) => void; + signal?: AbortSignal; + /** Narrows retry classification when an attempt also performs non-CodeAPI work. */ + isRateLimit?: (error: unknown) => boolean; }): Promise { - const { attempt, label, budget, onWait } = params; + const { + attempt, + label, + budget, + onWait, + signal, + isRateLimit = (error) => isAxiosError(error) && error.response?.status === 429, + } = params; for (;;) { + signal?.throwIfAborted(); try { return await attempt(); } catch (error) { - if (!isAxiosError(error) || error.response?.status !== 429) { + if (!isRateLimit(error)) { throw error; } const retryAfterMs = getCodeApiRetryAfterMs(error); if (retryAfterMs == null) { - throw new Error(`Code API rate limit reached while ${label}.`); + throw codeApiRateLimitError(label, error); } - /* A zero delay would spin; charge at least a second so the budget - * always drains and the loop terminates. */ + /* A zero delay would spin. Wait at least a second; the actual-wait + * budget still guarantees that repeated zero hints terminate. */ const waitMs = Math.max(retryAfterMs, 1000); - if (budget == null || waitMs > budget.remainingMs) { - throw new Error( - `Code API rate limit reached while ${label} (retry in ${Math.ceil(waitMs / 1000)}s).`, - ); + if (budget == null) { + throw codeApiRateLimitError(label, error, waitMs); + } + const rateLimitBudget = budget; + const waitToken = beginCodeApiRateLimitWait(rateLimitBudget, waitMs); + if (waitToken == null) { + throw codeApiRateLimitError(label, error, waitMs); } - budget.remainingMs -= waitMs; onWait?.(waitMs); - await new Promise((resolve) => setTimeout(resolve, waitMs)); + try { + await delay(waitMs, undefined, signal ? { signal } : undefined); + } finally { + finishCodeApiRateLimitWait(rateLimitBudget, waitToken); + } } } } + +/** Owns upload admission, retry waiting, and source reopening for every Code API + * upload producer. Callers supply storage and transport adapters only. */ +export function withCodeApiUploadRecovery(params: { + registry: CodeApiUploadRegistry; + scope: string; + concurrency?: number; + budget?: CodeApiRateLimitBudget; + signal?: AbortSignal; + label: string; + onWait?: (waitMs: number) => void; + openSource: () => Promise; + upload: (source: S) => Promise; +}): Promise { + const { registry, scope, concurrency, budget, signal, label, onWait, openSource, upload } = + params; + return withCodeApiUploadSlot({ + registry, + scope, + concurrency, + signal, + task: () => { + let uploadStarted = false; + return withCodeApiRateLimit({ + budget, + signal, + label, + onWait, + /* Storage backends can also answer 429. Only the CodeAPI transport owns this + * retry policy and its operator-facing error identity. */ + isRateLimit: (error) => + uploadStarted && isAxiosError(error) && error.response?.status === 429, + attempt: async () => { + uploadStarted = false; + let source: S | undefined; + try { + source = await openSource(); + signal?.throwIfAborted(); + uploadStarted = true; + return await upload(source); + } catch (error) { + if (source !== undefined) { + disposeCodeApiUploadSource(source); + } + throw error; + } + }, + }); + }, + }); +} + +/** Best-effort disposal for the stream shapes accepted by upload adapters: a + * single readable or a batch of `{ stream }` entries. */ +function disposeCodeApiUploadSource(source: unknown): void { + try { + if (Array.isArray(source)) { + source.forEach(disposeCodeApiUploadSource); + return; + } + if (source == null || typeof source !== 'object') { + return; + } + if ('stream' in source) { + disposeCodeApiUploadSource((source as { stream?: unknown }).stream); + return; + } + const destroy = (source as { destroy?: unknown }).destroy; + if (typeof destroy === 'function') { + destroy.call(source); + } + } catch { + /* Cleanup must not replace the upload or cancellation error being propagated. */ + } +} + +export function isCodeApiRateLimitError(error: unknown): error is Error & { + code: 'CODE_API_RATE_LIMITED'; + status: 429; +} { + return ( + error != null && + typeof error === 'object' && + (error as { code?: unknown }).code === 'CODE_API_RATE_LIMITED' && + (error as { status?: unknown }).status === 429 + ); +} diff --git a/packages/api/src/utils/errors.spec.ts b/packages/api/src/utils/errors.spec.ts index 63acde6738..ebe8e16b3a 100644 --- a/packages/api/src/utils/errors.spec.ts +++ b/packages/api/src/utils/errors.spec.ts @@ -68,6 +68,18 @@ describe('isAbortError', () => { expect(isAbortError(wrapped)).toBe(true); }); + it('recognizes an Axios cancellation through an upload wrapper cause', () => { + const axiosCancellation = Object.assign(new Error('canceled'), { + name: 'CanceledError', + code: 'ERR_CANCELED', + }); + const wrapped = new Error('Error uploading code environment file', { + cause: axiosCancellation, + }); + + expect(isAbortError(wrapped)).toBe(true); + }); + it('recognizes the SDK message shape that only stringifies the reason', () => { expect( isAbortError(new Error('MCP error -32001: AbortError: This operation was aborted')), diff --git a/packages/api/src/utils/errors.ts b/packages/api/src/utils/errors.ts index f43e0354e1..897cf58f32 100644 --- a/packages/api/src/utils/errors.ts +++ b/packages/api/src/utils/errors.ts @@ -63,6 +63,7 @@ export function isAbortError(error: unknown): boolean { if ( name === 'AbortError' || code === 'ABORT_ERR' || + code === 'ERR_CANCELED' || message.includes('AbortError') || /(?:operation|request|stream) was aborted/i.test(message) ) { diff --git a/packages/data-provider/specs/config-schemas.spec.ts b/packages/data-provider/specs/config-schemas.spec.ts index 0671696509..4e4b3ded1f 100644 --- a/packages/data-provider/specs/config-schemas.spec.ts +++ b/packages/data-provider/specs/config-schemas.spec.ts @@ -441,6 +441,21 @@ describe('endpointSchema addParams validation', () => { }); describe('agentsEndpointSchema', () => { + it('defaults and bounds Code API upload recovery controls', () => { + expect(agentsEndpointSchema.parse({}).codeApiUploadConcurrency).toBe(3); + expect( + agentsEndpointSchema.parse({ codeApiUploadConcurrency: 8 }).codeApiUploadConcurrency, + ).toBe(8); + expect(agentsEndpointSchema.safeParse({ codeApiUploadConcurrency: 0 }).success).toBe(false); + expect(agentsEndpointSchema.safeParse({ codeApiUploadConcurrency: 101 }).success).toBe(false); + expect(agentsEndpointSchema.parse({}).codeApiMaxRetryWaitMs).toBe(20_000); + expect( + agentsEndpointSchema.parse({ codeApiMaxRetryWaitMs: 60_000 }).codeApiMaxRetryWaitMs, + ).toBe(60_000); + expect(agentsEndpointSchema.safeParse({ codeApiMaxRetryWaitMs: -1 }).success).toBe(false); + expect(agentsEndpointSchema.safeParse({ codeApiMaxRetryWaitMs: 300_001 }).success).toBe(false); + }); + it('accepts a non-empty stateful code environment allowlist', () => { const result = agentsEndpointSchema.safeParse({ statefulCodeSessions: { allowedEnvironments: ['user', 'agent-user'] }, diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index acfa2561e4..c860446f08 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -1201,6 +1201,10 @@ export const agentsEndpointSchema = baseEndpointSchema .max(MAX_SUBAGENTS_CEILING) .optional() .default(MAX_SUBAGENTS), + /** Maximum concurrent Code API uploads per route and authenticated principal. */ + codeApiUploadConcurrency: z.number().int().min(1).max(100).optional().default(3), + /** Maximum wall-clock time spent waiting on Code API rate limits per operation. */ + codeApiMaxRetryWaitMs: z.number().int().min(0).max(300_000).optional().default(20_000), allowedProviders: z.array(z.union([z.string(), eModelEndpointSchema])).optional(), capabilities: z .array(z.nativeEnum(AgentCapabilities)) diff --git a/packages/data-provider/src/types.ts b/packages/data-provider/src/types.ts index 798b12237a..00e17728eb 100644 --- a/packages/data-provider/src/types.ts +++ b/packages/data-provider/src/types.ts @@ -632,6 +632,10 @@ export type TConfig = { }; /** Effective subagents-per-agent cap served from `endpoints.agents.maxSubagents`. */ maxSubagents?: number; + /** Concurrent Code API uploads allowed per route and authenticated principal. */ + codeApiUploadConcurrency?: number; + /** Milliseconds one operation may spend waiting on Code API rate limits. */ + codeApiMaxRetryWaitMs?: number; customParams?: { defaultParamsEndpoint?: string; reasoningFormat?: ReasoningParameterFormat;