mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-20 23:13:35 +00:00
🛟 fix: Recover CodeAPI File Uploads After Throttling (#15784)
* fix: recover CodeAPI uploads after throttling * fix: cover eager CodeAPI uploads * fix: scope CodeAPI upload recovery * test: clarify upload recovery deadlines * fix: isolate CodeAPI recovery state * fix: cancel CodeAPI recovery waits * test: provide CodeAPI registry in handler fixture * fix: propagate CodeAPI recovery controls * fix: bound CodeAPI recovery by run * fix: preserve skill upload cancellation * test: update deferred loader signal contract * fix: preserve recovery cancellation * fix: cancel recovery source reads * fix: classify Axios upload cancellation * fix: preserve lazy provisioning cancellation * fix: close resource recovery boundaries * fix: preserve provisioning failures compatibly
This commit is contained in:
parent
f0c9dee669
commit
65bb64bc68
30 changed files with 1792 additions and 161 deletions
|
|
@ -388,6 +388,7 @@ const loadTools = async ({
|
|||
});
|
||||
const { files, toolContext } = await primeCodeFiles({
|
||||
...options,
|
||||
signal,
|
||||
agentId: agent?.id,
|
||||
codeApiBaseUrl: codeExecutionContext.baseUrl,
|
||||
executionProfile: codeExecutionContext.executionProfile,
|
||||
|
|
|
|||
|
|
@ -374,11 +374,13 @@ describe('Tool Handlers', () => {
|
|||
|
||||
it('routes code file priming to the selected bridge worker', async () => {
|
||||
const bridgeWorkerId = 'principal-worker';
|
||||
const controller = new AbortController();
|
||||
const toolMap = await loadTools({
|
||||
user: fakeUser._id.toString(),
|
||||
tools: [Tools.execute_code],
|
||||
returnMap: true,
|
||||
agent: { id: 'agent-1' },
|
||||
signal: controller.signal,
|
||||
options: {
|
||||
codeExecutionContext: {
|
||||
baseUrl: 'https://code.example.com/v1',
|
||||
|
|
@ -394,6 +396,7 @@ describe('Tool Handlers', () => {
|
|||
agentId: 'agent-1',
|
||||
codeApiBaseUrl: 'https://code.example.com/v1',
|
||||
bridgeWorkerId,
|
||||
signal: controller.signal,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ const {
|
|||
createAgentEventTerminalHandler,
|
||||
startCodeEnvironmentLifecycleReconciler,
|
||||
waitForKeyvRedisClient,
|
||||
createCodeApiUploadRegistry,
|
||||
} = require('@librechat/api');
|
||||
const { connectDb, indexSync } = require('~/db');
|
||||
const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager');
|
||||
|
|
@ -356,6 +357,7 @@ if (cluster.isMaster) {
|
|||
* Each worker runs a full Express server instance
|
||||
*/
|
||||
const app = express();
|
||||
app.locals.codeApiUploadRegistry = createCodeApiUploadRegistry();
|
||||
// The clustered entrypoint deliberately does not arm the v1 schedule engine,
|
||||
// but an already-fired scheduled generation can still reach HITL here. Settle
|
||||
// its durable run when the generic approval runtime expires it.
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ const {
|
|||
startCodeEnvironmentLifecycleReconciler,
|
||||
waitForKeyvRedisClient,
|
||||
warnOnUnreachableDeliveryPaths,
|
||||
createCodeApiUploadRegistry,
|
||||
} = require('@librechat/api');
|
||||
const { connectDb, indexSync } = require('~/db');
|
||||
const {
|
||||
|
|
@ -101,6 +102,7 @@ const host = HOST || 'localhost';
|
|||
const trusted_proxy = Number(TRUST_PROXY) || 1; /* trust first proxy by default */
|
||||
|
||||
const app = express();
|
||||
app.locals.codeApiUploadRegistry = createCodeApiUploadRegistry();
|
||||
let serverReady = false;
|
||||
/** @type {import('@librechat/api').ScheduleEngineState} */
|
||||
let scheduleEngineState = 'starting';
|
||||
|
|
|
|||
|
|
@ -413,7 +413,7 @@ const initializeClient = async ({
|
|||
runSignal: signal,
|
||||
foregroundRunId,
|
||||
ordinaryToolCancellation: ordinaryToolCancellationEnabled,
|
||||
loadTools: async (toolNames, agentId, _configurable, callerCapabilityProjection) => {
|
||||
loadTools: async (toolNames, agentId, _configurable, callerCapabilityProjection, runSignal) => {
|
||||
const ctx = agentToolContexts.get(agentId) ?? {};
|
||||
logger.debug(`[ON_TOOL_EXECUTE] ctx found: ${!!ctx.userMCPAuthMap}, agent: ${ctx.agent?.id}`);
|
||||
logger.debug(`[ON_TOOL_EXECUTE] toolRegistry size: ${ctx.toolRegistry?.size ?? 'undefined'}`);
|
||||
|
|
@ -421,7 +421,7 @@ const initializeClient = async ({
|
|||
const result = await loadToolsForExecution({
|
||||
req,
|
||||
res,
|
||||
signal,
|
||||
signal: runSignal ?? signal,
|
||||
streamId,
|
||||
conversationId,
|
||||
requestBody: runtimeRequestBody,
|
||||
|
|
@ -1599,6 +1599,7 @@ const initializeClient = async ({
|
|||
req,
|
||||
payload,
|
||||
skillNames,
|
||||
signal,
|
||||
accessibleSkillIds,
|
||||
executionProfiles: codeExecutionProfiles,
|
||||
...getSkillToolDeps(),
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ const { getCodeBaseURL } = require('@librechat/agents');
|
|||
const { EModelEndpoint, getCodeEnvRefs } = require('librechat-data-provider');
|
||||
const {
|
||||
logAxiosError,
|
||||
wrapCodeApiUploadError,
|
||||
appendCodeEnvFile,
|
||||
createAxiosInstance,
|
||||
codeServerHttpAgent,
|
||||
|
|
@ -183,6 +184,7 @@ async function deleteCodeEnvFile(req, file) {
|
|||
* @param {string} [params.codeApiBaseUrl] - Trusted per-agent Code API endpoint.
|
||||
* @param {'default'|'stateful'} [params.executionProfile] - Trusted execution profile.
|
||||
* @param {string} [params.bridgeWorkerId] - Trusted worker selected for this execution.
|
||||
* @param {AbortSignal} [params.signal] - Effective cancellation signal.
|
||||
* @returns {Promise<{ storage_session_id: string; file_id: string }>}
|
||||
* The codeapi storage location of the uploaded file.
|
||||
* @throws {Error} If there's an error during the upload process.
|
||||
|
|
@ -237,12 +239,7 @@ async function uploadCodeEnvFile({
|
|||
file_id: result.files[0].fileId,
|
||||
};
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
logAxiosError({
|
||||
message: `Error uploading code environment file: ${error.message}`,
|
||||
error,
|
||||
}),
|
||||
);
|
||||
throw wrapCodeApiUploadError(error, `Error uploading code environment file: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -281,6 +278,7 @@ async function batchUploadCodeEnvFiles({
|
|||
codeApiBaseUrl,
|
||||
executionProfile,
|
||||
bridgeWorkerId,
|
||||
signal,
|
||||
}) {
|
||||
const form = new FormData();
|
||||
appendCodeEnvFileIdentity(form, { kind, id, version });
|
||||
|
|
@ -308,6 +306,7 @@ async function batchUploadCodeEnvFiles({
|
|||
timeout: 120000,
|
||||
maxContentLength: MAX_FILE_SIZE,
|
||||
maxBodyLength: MAX_FILE_SIZE,
|
||||
signal,
|
||||
};
|
||||
|
||||
const response = await axios.post(`${baseURL}/upload/batch`, form, options);
|
||||
|
|
|
|||
|
|
@ -42,6 +42,14 @@ jest.mock('@librechat/api', () => {
|
|||
form.append('id', identity.id);
|
||||
if (identity.version != null) form.append('version', String(identity.version));
|
||||
}),
|
||||
wrapCodeApiUploadError: jest.fn((error, message) => {
|
||||
const wrapped = new Error(`${message} ${error.message}`, { cause: error });
|
||||
if (error?.isAxiosError === true) {
|
||||
wrapped.isAxiosError = true;
|
||||
wrapped.response = error.response;
|
||||
}
|
||||
return wrapped;
|
||||
}),
|
||||
buildCodeEnvDownloadQuery: jest.fn((identity) => {
|
||||
validateIdentity(identity, 'buildCodeEnvDownloadQuery');
|
||||
const params = new URLSearchParams({ kind: identity.kind, id: identity.id });
|
||||
|
|
@ -561,10 +569,25 @@ describe('Code CRUD', () => {
|
|||
|
||||
await expect(uploadCodeEnvFile(baseUploadParams)).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('preserves rate-limit metadata through the upload logging boundary', async () => {
|
||||
const rateLimit = Object.assign(new Error('Too Many Requests'), {
|
||||
isAxiosError: true,
|
||||
response: { status: 429, headers: { 'retry-after': '7' } },
|
||||
});
|
||||
mockAxios.post.mockRejectedValue(rateLimit);
|
||||
|
||||
await expect(uploadCodeEnvFile(baseUploadParams)).rejects.toMatchObject({
|
||||
isAxiosError: true,
|
||||
response: { status: 429, headers: { 'retry-after': '7' } },
|
||||
cause: rateLimit,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('batchUploadCodeEnvFiles', () => {
|
||||
it('routes batch uploads through the selected bridge worker', async () => {
|
||||
const controller = new AbortController();
|
||||
const req = { user: { id: 'user-123' } };
|
||||
mockAxios.post.mockResolvedValue({
|
||||
data: {
|
||||
|
|
@ -584,6 +607,7 @@ describe('Code CRUD', () => {
|
|||
codeApiBaseUrl: 'https://stateful-code.example.com',
|
||||
executionProfile: 'stateful',
|
||||
bridgeWorkerId: 'personal-worker-1',
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
const [url, , callConfig] = mockAxios.post.mock.calls[0];
|
||||
|
|
@ -591,6 +615,7 @@ describe('Code CRUD', () => {
|
|||
expect(getCodeApiAuthHeaders).toHaveBeenCalledWith(req, 'personal-worker-1');
|
||||
expect(callConfig.headers['X-CodeAPI-Expected-Profile']).toBe('stateful');
|
||||
expect(callConfig.headers['X-LibreChat-Code-Worker-ID']).toBe('personal-worker-1');
|
||||
expect(callConfig.signal).toBe(controller.signal);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -11,7 +11,10 @@ const {
|
|||
flattenArtifactPath,
|
||||
createAxiosInstance,
|
||||
getCodeApiAuthHeaders,
|
||||
isAbortError,
|
||||
getCodeApiUploadOptions,
|
||||
withCodeApiRateLimit,
|
||||
withCodeApiUploadRecovery,
|
||||
classifyCodeArtifact,
|
||||
isMissingSandboxPathError,
|
||||
parseSandboxImageChunk,
|
||||
|
|
@ -1138,15 +1141,18 @@ function checkIfActive(dateString) {
|
|||
* @param {ServerRequest} [req] - Current authenticated request, used to mint Code API auth.
|
||||
* @param {{baseUrl?: string, executionProfile?: 'default'|'stateful', bridgeWorkerId?: string}} [route]
|
||||
* Trusted host-selected Code API route.
|
||||
* @param {AbortSignal} [signal] - Effective run cancellation signal.
|
||||
*
|
||||
* @returns {Promise<string|null>}
|
||||
* A promise that resolves to the `lastModified` time string of the file if successful, or null if there is an
|
||||
* error in initialization or fetching the info.
|
||||
*/
|
||||
async function getSessionInfo(ref, req, route = {}) {
|
||||
async function getSessionInfo(ref, req, route = {}, signal) {
|
||||
try {
|
||||
signal?.throwIfAborted();
|
||||
const baseURL = route.baseUrl ?? getCodeBaseURL();
|
||||
const authHeaders = await getCodeApiAuthHeaders(req, route.bridgeWorkerId);
|
||||
signal?.throwIfAborted();
|
||||
/* `/sessions/.../objects/...` is gated by codeapi's `sessionAuth`
|
||||
* middleware (post-Phase C). The middleware reconstructs the
|
||||
* sessionKey from the URL query (`kind`/`id`/`version?`) plus the
|
||||
|
|
@ -1174,10 +1180,15 @@ async function getSessionInfo(ref, req, route = {}) {
|
|||
httpAgent: codeServerHttpAgent,
|
||||
httpsAgent: codeServerHttpsAgent,
|
||||
timeout: 5000,
|
||||
signal,
|
||||
});
|
||||
signal?.throwIfAborted();
|
||||
|
||||
return response.data?.lastModified;
|
||||
} catch (_error) {
|
||||
} catch (error) {
|
||||
if (signal?.aborted && isAbortError(error)) {
|
||||
throw error;
|
||||
}
|
||||
logger.debug('[getSessionInfo] session lookup failed (treating as cache miss)');
|
||||
return null;
|
||||
}
|
||||
|
|
@ -1275,6 +1286,9 @@ const getReuploadFailureCategory = (error) => {
|
|||
) {
|
||||
return 'resource_access_denied';
|
||||
}
|
||||
if (status === 429 || code === 'CODE_API_RATE_LIMITED') {
|
||||
return 'rate_limited';
|
||||
}
|
||||
return 'reupload_failed';
|
||||
};
|
||||
|
||||
|
|
@ -1285,6 +1299,7 @@ const getReuploadFailureCategory = (error) => {
|
|||
* @param {Agent['tool_resources']} options.tool_resources
|
||||
* @param {string} [options.agentId] - The agent ID for file access control
|
||||
* @param {string} [options.agentResourceType] - Permission resource type for the authorized agent route
|
||||
* @param {AbortSignal} [options.signal] - Effective run cancellation signal
|
||||
* @returns {Promise<{
|
||||
* files: Array<{ id: string; session_id: string; name: string }>,
|
||||
* toolContext: string,
|
||||
|
|
@ -1300,6 +1315,7 @@ const primeFiles = async (options) => {
|
|||
executionProfile = 'default',
|
||||
executionRouteKey = executionProfile,
|
||||
bridgeWorkerId,
|
||||
signal,
|
||||
} = options;
|
||||
const codeApiRoute = { baseUrl: codeApiBaseUrl, executionProfile, bridgeWorkerId };
|
||||
const file_ids = tool_resources?.[EToolResources.execute_code]?.file_ids ?? [];
|
||||
|
|
@ -1342,6 +1358,9 @@ const primeFiles = async (options) => {
|
|||
|
||||
const files = [];
|
||||
const sessions = new Map();
|
||||
const uploadOptions = getCodeApiUploadOptions(req, executionRouteKey);
|
||||
/** All stale-file reuploads in this prime share one live-turn wait cap. */
|
||||
const uploadRateLimitBudget = createCodeApiRateLimitBudget(uploadOptions.retryWaitMs);
|
||||
let toolContext = '';
|
||||
|
||||
/* Claim order decides which record keeps the bare `/mnt/data/<name>` path
|
||||
|
|
@ -1442,22 +1461,40 @@ const primeFiles = async (options) => {
|
|||
const { handleFileUpload: uploadCodeEnvFile } = getStrategyFunctions(
|
||||
FileSources.execute_code,
|
||||
);
|
||||
const stream = await getDownloadStream(options.req, resolveDownloadPath(file));
|
||||
/* Reupload preserves the resource identity from the existing
|
||||
* ref so codeapi re-buckets under the same sessionKey shape
|
||||
* (skill stays skill, user stays user). Without this, a
|
||||
* skill-cache-miss reupload would land in the user bucket
|
||||
* and never re-shareable cross-user. */
|
||||
const uploaded = await uploadCodeEnvFile({
|
||||
req: options.req,
|
||||
stream,
|
||||
filename: getDestination(),
|
||||
kind: sourceRef.kind,
|
||||
id: sourceRef.id,
|
||||
...(sourceRef.kind === 'skill' ? { version: sourceRef.version } : {}),
|
||||
codeApiBaseUrl,
|
||||
executionProfile,
|
||||
bridgeWorkerId,
|
||||
const uploaded = await withCodeApiUploadRecovery({
|
||||
registry: req.app?.locals?.codeApiUploadRegistry,
|
||||
scope: uploadOptions.scope,
|
||||
concurrency: uploadOptions.concurrency,
|
||||
label: `re-uploading file ${file.file_id} to the code environment`,
|
||||
budget: uploadRateLimitBudget,
|
||||
signal,
|
||||
onWait: (waitMs) =>
|
||||
logger.warn(
|
||||
`[primeCodeFiles] Rate-limited reupload requestId=${getPrimingCorrelation(req).requestId} ` +
|
||||
`runId=${getPrimingCorrelation(req).runId}; retrying in ${waitMs}ms`,
|
||||
),
|
||||
openSource: async () => {
|
||||
signal?.throwIfAborted();
|
||||
return getDownloadStream(options.req, resolveDownloadPath(file), { signal });
|
||||
},
|
||||
upload: (stream) =>
|
||||
uploadCodeEnvFile({
|
||||
req: options.req,
|
||||
stream,
|
||||
filename: getDestination(),
|
||||
kind: sourceRef.kind,
|
||||
id: sourceRef.id,
|
||||
...(sourceRef.kind === 'skill' ? { version: sourceRef.version } : {}),
|
||||
codeApiBaseUrl,
|
||||
executionProfile,
|
||||
bridgeWorkerId,
|
||||
signal,
|
||||
}),
|
||||
});
|
||||
|
||||
/**
|
||||
|
|
@ -1497,6 +1534,10 @@ const primeFiles = async (options) => {
|
|||
`oldSession=${session_id} newSession=${newRef.storage_session_id} newFileId=${newRef.file_id}`,
|
||||
);
|
||||
} catch (error) {
|
||||
/* Cancellation is an operation outcome, not a recoverable per-file
|
||||
* miss. Swallowing it here would keep walking and could dispatch more
|
||||
* uploads after the foreground run has ended. */
|
||||
signal?.throwIfAborted();
|
||||
reuploadFailures += 1;
|
||||
const failureCategory = getReuploadFailureCategory(error);
|
||||
reuploadFailureCategories.add(failureCategory);
|
||||
|
|
@ -1522,7 +1563,8 @@ const primeFiles = async (options) => {
|
|||
pushFile();
|
||||
continue;
|
||||
}
|
||||
const uploadTime = await getSessionInfo(ref, req, codeApiRoute);
|
||||
const uploadTime = await getSessionInfo(ref, req, codeApiRoute, signal);
|
||||
signal?.throwIfAborted();
|
||||
if (!uploadTime) {
|
||||
logger.debug(
|
||||
`[primeCodeFiles] file=${file.file_id} path=reupload reason=no-uploadtime ` +
|
||||
|
|
@ -1927,6 +1969,7 @@ async function previewWorkspaceEdit({
|
|||
* @param {string} [params.runtime_session_hint] - Per-conversation stateful runtime-session hint.
|
||||
* @param {number} [params.maxBytes] - In-sandbox size cap; larger files return `{ tooLarge, bytes }`.
|
||||
* @param {ServerRequest} [params.req] - Current authenticated request, used to mint Code API auth.
|
||||
* @param {AbortSignal} [params.signal] - Foreground run cancellation.
|
||||
* @param {string} [params.executionRouteKey] - Trusted deployment-local route identity.
|
||||
* @param {string} [params.bridgeWorkerId] - Trusted bridge worker selected for this execution.
|
||||
* @returns {Promise<{base64: string, bytes: number}
|
||||
|
|
@ -1944,6 +1987,7 @@ async function readSandboxImage({
|
|||
executionRouteKey,
|
||||
maxBytes,
|
||||
req,
|
||||
signal,
|
||||
}) {
|
||||
const limit = typeof maxBytes === 'number' && maxBytes > 0 ? maxBytes : 5 * megabyte;
|
||||
const preparedBuffer = getPreparedCodeOutputBuffer({
|
||||
|
|
@ -1970,7 +2014,9 @@ async function readSandboxImage({
|
|||
/** Every window is one `/exec` call against the Code API's per-user
|
||||
* execution limiter, so the read shares one wait budget: a window that
|
||||
* resets mid-read is worth pausing for, an exhausted budget is not. */
|
||||
const rateLimit = createCodeApiRateLimitBudget();
|
||||
const rateLimit = createCodeApiRateLimitBudget(
|
||||
req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
|
||||
);
|
||||
return readWindowedSandboxImage({
|
||||
filePath: file_path,
|
||||
baseUrl: baseURL,
|
||||
|
|
@ -1987,6 +2033,7 @@ async function readSandboxImage({
|
|||
files,
|
||||
req,
|
||||
rateLimit,
|
||||
signal,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
|
@ -2009,6 +2056,7 @@ async function execSandboxImageChunk({
|
|||
files,
|
||||
req,
|
||||
rateLimit,
|
||||
signal,
|
||||
}) {
|
||||
/** @type {Record<string, unknown>} */
|
||||
const postData = { lang: 'bash', code };
|
||||
|
|
@ -2026,6 +2074,7 @@ async function execSandboxImageChunk({
|
|||
const response = await withCodeApiRateLimit({
|
||||
label: `reading "${file_path}" from the sandbox`,
|
||||
budget: rateLimit,
|
||||
signal,
|
||||
onWait: (waitMs) =>
|
||||
logger.warn(
|
||||
`[readSandboxImage] Rate-limited reading "${file_path}"; retrying in ${waitMs}ms`,
|
||||
|
|
@ -2045,6 +2094,7 @@ async function execSandboxImageChunk({
|
|||
httpAgent: codeServerHttpAgent,
|
||||
httpsAgent: codeServerHttpsAgent,
|
||||
timeout: 15000,
|
||||
signal,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
|
|
|||
|
|
@ -90,13 +90,43 @@ jest.mock('@librechat/api', () => {
|
|||
: undefined,
|
||||
executeWorkspaceTool: (...args) => mockExecuteWorkspaceTool(...args),
|
||||
getCodeApiAuthHeaders: jest.fn(async () => ({})),
|
||||
isAbortError: (error) =>
|
||||
error?.name === 'AbortError' || error?.code === 'ABORT_ERR' || error?.code === 'ERR_CANCELED',
|
||||
/* Windowing, sizing and rate-limit policy are real code in
|
||||
* `packages/api` with their own tests (`files/code/image.spec.ts`,
|
||||
* `utils/code.spec.ts`). These stand-ins are deliberately inert
|
||||
* passthroughs — they assert nothing about that behavior, so the
|
||||
* transport tests below cover only what this file still owns. */
|
||||
createCodeApiRateLimitBudget: jest.fn(() => ({ remainingMs: 20_000 })),
|
||||
withCodeApiRateLimit: jest.fn(({ attempt }) => attempt()),
|
||||
createCodeApiRateLimitBudget: jest.fn(() => ({
|
||||
limitMs: 20_000,
|
||||
waitedMs: 0,
|
||||
activeWaitEnds: new Set(),
|
||||
})),
|
||||
getCodeApiUploadOptions: jest.fn(() => ({
|
||||
scope: 'default:user-123',
|
||||
concurrency: 3,
|
||||
retryWaitMs: 20_000,
|
||||
})),
|
||||
withCodeApiRateLimit: jest.fn(async ({ attempt }) => {
|
||||
try {
|
||||
return await attempt();
|
||||
} catch (error) {
|
||||
if (error?.response?.status !== 429) {
|
||||
throw error;
|
||||
}
|
||||
return attempt();
|
||||
}
|
||||
}),
|
||||
withCodeApiUploadRecovery: jest.fn(async ({ openSource, upload }) => {
|
||||
try {
|
||||
return await upload(await openSource());
|
||||
} catch (error) {
|
||||
if (error?.response?.status !== 429) {
|
||||
throw error;
|
||||
}
|
||||
return upload(await openSource());
|
||||
}
|
||||
}),
|
||||
buildSandboxImageReaderCode: jest.fn(
|
||||
({ filePath, limit, offset, chunkBytes }) =>
|
||||
`read ${filePath} ${limit} ${offset} ${chunkBytes}`,
|
||||
|
|
@ -1475,6 +1505,33 @@ describe('Code Process', () => {
|
|||
expect(callConfig.httpsAgent.keepAlive).toBe(false);
|
||||
});
|
||||
|
||||
it('forwards cancellation to the freshness request and preserves its error', async () => {
|
||||
const controller = new AbortController();
|
||||
const canceled = Object.assign(new Error('canceled'), {
|
||||
name: 'CanceledError',
|
||||
code: 'ERR_CANCELED',
|
||||
});
|
||||
mockAxios.mockImplementationOnce(async (config) => {
|
||||
expect(config.signal).toBe(controller.signal);
|
||||
controller.abort();
|
||||
throw canceled;
|
||||
});
|
||||
|
||||
await expect(
|
||||
getSessionInfo(
|
||||
{
|
||||
kind: 'user',
|
||||
id: 'user-1',
|
||||
storage_session_id: 'sess',
|
||||
file_id: 'fid',
|
||||
},
|
||||
mockReq,
|
||||
{},
|
||||
controller.signal,
|
||||
),
|
||||
).rejects.toBe(canceled);
|
||||
});
|
||||
|
||||
it('forwards Code API auth headers when checking session object freshness', async () => {
|
||||
getCodeApiAuthHeaders.mockResolvedValueOnce({ Authorization: 'Bearer freshness-token' });
|
||||
mockAxios.mockResolvedValue({
|
||||
|
|
@ -2585,6 +2642,112 @@ describe('Code Process', () => {
|
|||
return { handleFileUpload, getDownloadStream };
|
||||
}
|
||||
|
||||
it('recovers a throttled stale-file reupload with a fresh stream', async () => {
|
||||
const controller = new AbortController();
|
||||
const dbFile = {
|
||||
file_id: 'rate-limited-file',
|
||||
filename: 'report.csv',
|
||||
filepath: '/uploads/report.csv',
|
||||
source: 'local',
|
||||
context: 'execute_code',
|
||||
metadata: {
|
||||
codeEnvRef: {
|
||||
kind: 'user',
|
||||
id: 'user-123',
|
||||
storage_session_id: 'OLD_SESSION',
|
||||
file_id: 'OLD_ID',
|
||||
},
|
||||
},
|
||||
};
|
||||
const rateLimit = Object.assign(new Error('Request failed with status code 429'), {
|
||||
isAxiosError: true,
|
||||
response: { status: 429, headers: { 'retry-after': '0' } },
|
||||
});
|
||||
const handleFileUpload = jest
|
||||
.fn()
|
||||
.mockRejectedValueOnce(rateLimit)
|
||||
.mockResolvedValueOnce({ storage_session_id: 'NEW_SESSION', file_id: 'NEW_ID' });
|
||||
const getDownloadStream = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce('first-stream')
|
||||
.mockResolvedValueOnce('second-stream');
|
||||
getStrategyFunctions.mockImplementation((source) =>
|
||||
source === 'execute_code' ? { handleFileUpload } : { getDownloadStream },
|
||||
);
|
||||
getFiles.mockResolvedValue([dbFile]);
|
||||
filterFilesByAgentAccess.mockImplementation(({ files }) => Promise.resolve(files));
|
||||
mockAxios.mockResolvedValue({ data: null });
|
||||
updateFile.mockResolvedValue({});
|
||||
|
||||
const result = await primeFiles({
|
||||
req: { user: { id: 'user-123', role: 'USER' } },
|
||||
tool_resources: { execute_code: { file_ids: [dbFile.file_id], files: [] } },
|
||||
agentId: 'agent-id',
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
const { withCodeApiUploadRecovery } = require('@librechat/api');
|
||||
expect(withCodeApiUploadRecovery).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ signal: controller.signal }),
|
||||
);
|
||||
expect(handleFileUpload).toHaveBeenCalledTimes(2);
|
||||
expect(handleFileUpload).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ signal: controller.signal }),
|
||||
);
|
||||
expect(getDownloadStream).toHaveBeenCalledTimes(2);
|
||||
expect(getDownloadStream).toHaveBeenCalledWith(expect.anything(), '/uploads/report.csv', {
|
||||
signal: controller.signal,
|
||||
});
|
||||
expect(handleFileUpload.mock.calls.map(([args]) => args.stream)).toEqual([
|
||||
'first-stream',
|
||||
'second-stream',
|
||||
]);
|
||||
expect(result.files).toEqual([
|
||||
expect.objectContaining({ id: 'NEW_ID', storage_session_id: 'NEW_SESSION' }),
|
||||
]);
|
||||
});
|
||||
|
||||
it('propagates cancellation from a stale-file reupload', async () => {
|
||||
const controller = new AbortController();
|
||||
const dbFile = {
|
||||
file_id: 'canceled-file',
|
||||
filename: 'report.csv',
|
||||
filepath: '/uploads/report.csv',
|
||||
source: 'local',
|
||||
context: 'execute_code',
|
||||
metadata: {
|
||||
codeEnvRef: {
|
||||
kind: 'user',
|
||||
id: 'user-123',
|
||||
storage_session_id: 'OLD_SESSION',
|
||||
file_id: 'OLD_ID',
|
||||
},
|
||||
},
|
||||
};
|
||||
const handleFileUpload = jest.fn(async () => {
|
||||
controller.abort();
|
||||
controller.signal.throwIfAborted();
|
||||
});
|
||||
getStrategyFunctions.mockImplementation((source) =>
|
||||
source === 'execute_code'
|
||||
? { handleFileUpload }
|
||||
: { getDownloadStream: jest.fn().mockResolvedValue('stream') },
|
||||
);
|
||||
getFiles.mockResolvedValue([dbFile]);
|
||||
filterFilesByAgentAccess.mockImplementation(({ files }) => Promise.resolve(files));
|
||||
mockAxios.mockResolvedValue({ data: null });
|
||||
|
||||
await expect(
|
||||
primeFiles({
|
||||
req: { user: { id: 'user-123', role: 'USER' } },
|
||||
tool_resources: { execute_code: { file_ids: [dbFile.file_id], files: [] } },
|
||||
agentId: 'agent-id',
|
||||
signal: controller.signal,
|
||||
}),
|
||||
).rejects.toMatchObject({ name: 'AbortError' });
|
||||
expect(handleFileUpload).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('uses the permission resource type established by the calling route', async () => {
|
||||
const files = [
|
||||
{
|
||||
|
|
@ -2711,6 +2874,7 @@ describe('Code Process', () => {
|
|||
expect(getDownloadStream).toHaveBeenCalledWith(
|
||||
{ user: { id: 'user-123', role: 'USER' } },
|
||||
'/uploads/trusted.txt',
|
||||
{ signal: undefined },
|
||||
);
|
||||
expect(handleFileUpload).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
|
@ -3022,6 +3186,7 @@ describe('Code Process', () => {
|
|||
],
|
||||
['Azure BlobNotFound', { code: 'BlobNotFound', statusCode: 404 }, 'missing_backing_object'],
|
||||
['storage access denied', { code: 'AccessDenied', status: 403 }, 'resource_access_denied'],
|
||||
['Code API throttling', { code: 'CODE_API_RATE_LIMITED', status: 429 }, 'rate_limited'],
|
||||
])(
|
||||
'fails with a typed recovery error for %s',
|
||||
async (_errorShape, downloadError, expectedCategory) => {
|
||||
|
|
@ -3550,6 +3715,7 @@ describe('Code Process', () => {
|
|||
});
|
||||
|
||||
it('forwards the sandbox session, runtime hint and profile header', async () => {
|
||||
const controller = new AbortController();
|
||||
mockAxios.mockResolvedValue({ data: { stdout: '{}' } });
|
||||
|
||||
await readSandboxImage({
|
||||
|
|
@ -3559,6 +3725,7 @@ describe('Code Process', () => {
|
|||
files: [{ id: 'file-1', name: 'seed.csv' }],
|
||||
codeApiBaseUrl: 'https://code-stateful.example.com',
|
||||
executionProfile: 'stateful',
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
expect(mockAxios).toHaveBeenCalledTimes(1);
|
||||
|
|
@ -3571,6 +3738,11 @@ describe('Code Process', () => {
|
|||
files: [{ id: 'file-1', name: 'seed.csv' }],
|
||||
});
|
||||
expect(call.headers['X-CodeAPI-Expected-Profile']).toBe('stateful');
|
||||
expect(call.signal).toBe(controller.signal);
|
||||
const { withCodeApiRateLimit } = require('@librechat/api');
|
||||
expect(withCodeApiRateLimit).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ signal: controller.signal }),
|
||||
);
|
||||
});
|
||||
|
||||
it('omits the profile header and optional session fields when unset', async () => {
|
||||
|
|
|
|||
|
|
@ -45,6 +45,9 @@ const {
|
|||
sweepExpiredFiles: sweepExpiredFilesWithDeps,
|
||||
startExpiredFileSweep: startExpiredFileSweepWithDeps,
|
||||
resolveToolRoleGrants,
|
||||
createCodeApiRateLimitBudget,
|
||||
getCodeApiUploadOptions,
|
||||
withCodeApiUploadRecovery,
|
||||
} = require('@librechat/api');
|
||||
const {
|
||||
convertImage,
|
||||
|
|
@ -1221,20 +1224,34 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => {
|
|||
/* Upload from the persisted representation. Images have already been converted here,
|
||||
* so both the bytes and extension advertised to the sandbox describe the same file. */
|
||||
const downloadPath = storageResult.storageKey ?? storageResult.filepath;
|
||||
const stream = getDownloadStream
|
||||
? await getDownloadStream(req, downloadPath)
|
||||
: fs.createReadStream(file.path);
|
||||
const codeKind = messageAttachment === true ? 'user' : 'agent';
|
||||
const codeId = messageAttachment === true ? req.user.id : agent_id;
|
||||
const sandboxFilename = resolveSandboxFilename(sanitizeFilename(file.originalname), storedType);
|
||||
const uploadOptions = getCodeApiUploadOptions(req, 'default');
|
||||
let uploaded;
|
||||
try {
|
||||
uploaded = await uploadCodeEnvFile({
|
||||
req,
|
||||
stream,
|
||||
filename: sandboxFilename,
|
||||
kind: codeKind,
|
||||
id: codeId,
|
||||
uploaded = await withCodeApiUploadRecovery({
|
||||
registry: req.app?.locals?.codeApiUploadRegistry,
|
||||
scope: uploadOptions.scope,
|
||||
concurrency: uploadOptions.concurrency,
|
||||
label: `uploading "${sandboxFilename}" to the code environment`,
|
||||
budget: createCodeApiRateLimitBudget(uploadOptions.retryWaitMs),
|
||||
onWait: (waitMs) =>
|
||||
logger.warn(
|
||||
`[processAgentFileUpload] Rate-limited Code API upload; retrying in ${waitMs}ms`,
|
||||
),
|
||||
openSource: () =>
|
||||
getDownloadStream
|
||||
? getDownloadStream(req, downloadPath)
|
||||
: Promise.resolve(fs.createReadStream(file.path)),
|
||||
upload: (stream) =>
|
||||
uploadCodeEnvFile({
|
||||
req,
|
||||
stream,
|
||||
filename: sandboxFilename,
|
||||
kind: codeKind,
|
||||
id: codeId,
|
||||
}),
|
||||
});
|
||||
} catch (error) {
|
||||
const { deleteFile } = getStrategyFunctions(source);
|
||||
|
|
|
|||
|
|
@ -37,6 +37,26 @@ jest.mock('@librechat/api', () => {
|
|||
const actualDataProvider = jest.requireActual('librechat-data-provider');
|
||||
const RetentionMode = actualDataProvider.RetentionMode ?? { ALL: 'all', TEMPORARY: 'temporary' };
|
||||
const getRetentionExpiry = jest.fn(() => ({}));
|
||||
const createCodeApiRateLimitBudget = jest.fn(() => ({
|
||||
limitMs: 20_000,
|
||||
waitedMs: 0,
|
||||
activeWaitEnds: new Set(),
|
||||
}));
|
||||
const getCodeApiUploadOptions = jest.fn(() => ({
|
||||
scope: 'default:user-1',
|
||||
concurrency: 3,
|
||||
retryWaitMs: 20_000,
|
||||
}));
|
||||
const withCodeApiUploadRecovery = jest.fn(async ({ openSource, upload }) => {
|
||||
try {
|
||||
return await upload(await openSource());
|
||||
} catch (error) {
|
||||
if (error?.response?.status !== 429) {
|
||||
throw error;
|
||||
}
|
||||
return upload(await openSource());
|
||||
}
|
||||
});
|
||||
const UPLOAD_EXTRACTED_TEXT_PLANS = {
|
||||
configuredOCR: 'configured_ocr',
|
||||
configuredRAG: 'configured_rag',
|
||||
|
|
@ -131,6 +151,9 @@ jest.mock('@librechat/api', () => {
|
|||
}),
|
||||
getStorageMetadata: jest.fn(() => ({})),
|
||||
getRetentionExpiry,
|
||||
createCodeApiRateLimitBudget,
|
||||
getCodeApiUploadOptions,
|
||||
withCodeApiUploadRecovery,
|
||||
getAgentFileRetentionExpiry: jest.fn(({ req, messageAttachment, toolResource }) => {
|
||||
const interfaceConfig = req?.config?.interfaceConfig;
|
||||
if (
|
||||
|
|
@ -259,6 +282,9 @@ const {
|
|||
extractInspectableFileText,
|
||||
assertExtractedTextInspectable,
|
||||
contentFilterBlockResponse,
|
||||
createCodeApiRateLimitBudget,
|
||||
getCodeApiUploadOptions,
|
||||
withCodeApiUploadRecovery,
|
||||
} = require('@librechat/api');
|
||||
|
||||
const PDF_MIME = 'application/pdf';
|
||||
|
|
@ -1348,6 +1374,33 @@ describe('processAgentFileUpload', () => {
|
|||
expect(codeEnvUpload).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('retries a throttled eager upload with a fresh persisted stream', async () => {
|
||||
const rateLimited = Object.assign(new Error('rate limited'), {
|
||||
isAxiosError: true,
|
||||
response: { status: 429, headers: { 'retry-after': '1' } },
|
||||
});
|
||||
const codeEnvUpload = setupCodeEnvUpload({ storage_session_id: 'sess-y', file_id: 'fid-y' });
|
||||
codeEnvUpload
|
||||
.mockRejectedValueOnce(rateLimited)
|
||||
.mockResolvedValueOnce({ storage_session_id: 'sess-retry', file_id: 'fid-retry' });
|
||||
|
||||
await processAgentFileUpload({
|
||||
req: agentsZipReq(),
|
||||
res: mockRes,
|
||||
metadata: {
|
||||
agent_id: 'agent-abc',
|
||||
tool_resource: EToolResources.execute_code,
|
||||
file_id: 'file-throttled',
|
||||
},
|
||||
}).catch(() => {});
|
||||
|
||||
expect(getCodeApiUploadOptions).toHaveBeenCalledTimes(1);
|
||||
expect(withCodeApiUploadRecovery).toHaveBeenCalledTimes(1);
|
||||
expect(createCodeApiRateLimitBudget).toHaveBeenCalledTimes(1);
|
||||
expect(codeEnvUpload).toHaveBeenCalledTimes(2);
|
||||
expect(codeEnvUpload.mock.calls[0][0].stream).not.toBe(codeEnvUpload.mock.calls[1][0].stream);
|
||||
});
|
||||
|
||||
it('defers an inferred file-search destination until tool execution', async () => {
|
||||
const { uploadVectors } = require('~/server/services/Files/VectorDB/crud');
|
||||
setupStoredFileUpload();
|
||||
|
|
|
|||
|
|
@ -773,6 +773,7 @@ const isBuiltInTool = (toolName) =>
|
|||
* @param {string} [params.agentResourceType] - Permission resource type for the authorized agent route
|
||||
* @param {string|null} [params.streamId] - Stream ID for resumable mode
|
||||
* @param {number} [params.jobCreatedAt] - The generation epoch that owns emitted tool events
|
||||
* @param {AbortSignal} [params.signal] - Effective run cancellation signal
|
||||
* @returns {Promise<{
|
||||
* toolDefinitions?: import('@librechat/api').LCTool[];
|
||||
* toolRegistry?: Map<string, import('@librechat/api').LCTool>;
|
||||
|
|
@ -792,6 +793,7 @@ async function loadToolDefinitionsWrapper({
|
|||
tool_resources,
|
||||
codeExecutionContext,
|
||||
accessibleMcpServerNames,
|
||||
signal,
|
||||
}) {
|
||||
if (!agent.tools || agent.tools.length === 0) {
|
||||
return { toolDefinitions: [] };
|
||||
|
|
@ -1448,6 +1450,7 @@ async function loadToolDefinitionsWrapper({
|
|||
tool_resources,
|
||||
agentId: agent.id,
|
||||
agentResourceType,
|
||||
signal,
|
||||
codeApiBaseUrl: resolvedCodeExecutionContext.baseUrl,
|
||||
executionProfile: resolvedCodeExecutionContext.executionProfile,
|
||||
executionRouteKey: resolvedCodeExecutionContext.executionRouteKey,
|
||||
|
|
@ -1581,6 +1584,7 @@ async function loadAgentTools({
|
|||
tool_resources,
|
||||
codeExecutionContext: providedCodeExecutionContext,
|
||||
accessibleMcpServerNames,
|
||||
signal,
|
||||
});
|
||||
} catch (error) {
|
||||
if (
|
||||
|
|
|
|||
|
|
@ -583,6 +583,10 @@ endpoints:
|
|||
# # (optional) Maximum explicit subagents per agent, for both the flat list and
|
||||
# # graph definitions. Defaults to 10; hard cap 50.
|
||||
# maxSubagents: 20
|
||||
# # (optional) Maximum concurrent Code API uploads per route and user. Defaults to 3.
|
||||
# codeApiUploadConcurrency: 3
|
||||
# # (optional) Maximum total time spent waiting on Code API rate limits. Defaults to 20000 ms.
|
||||
# codeApiMaxRetryWaitMs: 20000
|
||||
# # (optional) Cap the number of active accessible skills shown in the model-visible catalog.
|
||||
# # Useful for large organizations where many department-specific skills may be available.
|
||||
# skills:
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import {
|
|||
import { markSandboxReady } from './prewarm';
|
||||
import { ContentFilterError } from '../middleware/contentFilter';
|
||||
import { WorkspaceToolHttpError } from '../code/workspace';
|
||||
import { createCodeApiUploadRegistry } from '~/utils';
|
||||
|
||||
function createMockTool(
|
||||
name: string,
|
||||
|
|
@ -415,6 +416,35 @@ describe('createToolExecuteHandler', () => {
|
|||
expect(results[0].status).toBe('error');
|
||||
});
|
||||
|
||||
it('forwards the effective batch signal into deferred tool loading', async () => {
|
||||
const controller = new AbortController();
|
||||
const tool = {
|
||||
name: 'loaded_tool',
|
||||
invoke: jest.fn(async () => ({ content: 'done' })),
|
||||
};
|
||||
const loadTools: ToolExecuteOptions['loadTools'] = jest.fn(async () => ({
|
||||
loadedTools: [tool] as never[],
|
||||
}));
|
||||
const handler = createToolExecuteHandler({ loadTools });
|
||||
|
||||
await new Promise<ToolExecuteResult[]>((resolve, reject) => {
|
||||
handler.handle('on_tool_execute', {
|
||||
toolCalls: [{ id: 'call-load', name: tool.name, args: {} }],
|
||||
signal: controller.signal,
|
||||
resolve,
|
||||
reject,
|
||||
} as ToolExecuteBatchRequest);
|
||||
});
|
||||
|
||||
expect(loadTools).toHaveBeenCalledWith(
|
||||
[tool.name],
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
controller.signal,
|
||||
);
|
||||
});
|
||||
|
||||
it('uses the host-owned run signal when an SDK event omits its signal', async () => {
|
||||
const controller = new AbortController();
|
||||
const tool = abortingTool();
|
||||
|
|
@ -803,7 +833,13 @@ describe('createToolExecuteHandler', () => {
|
|||
);
|
||||
|
||||
expect(loadTools).toHaveBeenCalledTimes(1);
|
||||
expect(loadTools).toHaveBeenCalledWith(['allowed_tool'], undefined, configurable, undefined);
|
||||
expect(loadTools).toHaveBeenCalledWith(
|
||||
['allowed_tool'],
|
||||
undefined,
|
||||
configurable,
|
||||
undefined,
|
||||
undefined,
|
||||
);
|
||||
expect(JSON.stringify(jest.mocked(loadTools).mock.calls)).not.toContain(protectedName);
|
||||
expect(results[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
|
|
@ -1333,6 +1369,7 @@ describe('createToolExecuteHandler', () => {
|
|||
undefined,
|
||||
undefined,
|
||||
callerCapabilityProjection,
|
||||
undefined,
|
||||
);
|
||||
expect(capturedConfigs[0].toolDefs).toEqual([
|
||||
{ name: 'active_programmatic_tool', allowed_callers: ['code_execution'] },
|
||||
|
|
@ -1829,12 +1866,20 @@ describe('createToolExecuteHandler', () => {
|
|||
configurable: {
|
||||
accessibleSkillIds: skillsInScope(),
|
||||
codeEnvAvailable: true,
|
||||
req: { user: { id: 'user-1' } },
|
||||
req: {
|
||||
user: { id: 'user-1', tenantId: 'tenant-1' },
|
||||
app: { locals: { codeApiUploadRegistry: createCodeApiUploadRegistry() } },
|
||||
config: {
|
||||
endpoints: {
|
||||
agents: { codeApiUploadConcurrency: 1, codeApiMaxRetryWaitMs: 1_000 },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}));
|
||||
const getSkillByName: ToolExecuteOptions['getSkillByName'] = jest.fn(async () => ({
|
||||
_id: `${skillName}-id` as unknown as never,
|
||||
name: skillName,
|
||||
const getSkillByName: ToolExecuteOptions['getSkillByName'] = jest.fn(async (name) => ({
|
||||
_id: `${name ?? skillName}-id` as unknown as never,
|
||||
name: name ?? skillName,
|
||||
body: 'skill body',
|
||||
fileCount: 1,
|
||||
version: 1,
|
||||
|
|
@ -2173,6 +2218,7 @@ describe('createToolExecuteHandler', () => {
|
|||
});
|
||||
|
||||
it('omits the unavailability note when file priming succeeds', async () => {
|
||||
const controller = new AbortController();
|
||||
const batchUploadCodeEnvFiles = jest.fn(async () => ({
|
||||
storage_session_id: 'session-ok',
|
||||
files: [
|
||||
|
|
@ -2182,14 +2228,24 @@ describe('createToolExecuteHandler', () => {
|
|||
}));
|
||||
const handler = createPrimingSkillHandler('note-ok-skill', batchUploadCodeEnvFiles);
|
||||
|
||||
const [result] = await invokeHandler(handler, [
|
||||
{
|
||||
id: 'call_prime_ok',
|
||||
name: Constants.SKILL_TOOL,
|
||||
args: { skillName: 'note-ok-skill' },
|
||||
},
|
||||
]);
|
||||
const [result] = await new Promise<ToolExecuteResult[]>((resolve, reject) => {
|
||||
handler.handle('on_tool_execute', {
|
||||
toolCalls: [
|
||||
{
|
||||
id: 'call_prime_ok',
|
||||
name: Constants.SKILL_TOOL,
|
||||
args: { skillName: 'note-ok-skill' },
|
||||
},
|
||||
],
|
||||
signal: controller.signal,
|
||||
resolve,
|
||||
reject,
|
||||
} as ToolExecuteBatchRequest);
|
||||
});
|
||||
|
||||
expect(batchUploadCodeEnvFiles).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ signal: controller.signal }),
|
||||
);
|
||||
expect(result.status).toBe('success');
|
||||
expect(result.content).not.toContain('could not be loaded');
|
||||
expect(result.artifact).toEqual(
|
||||
|
|
@ -2206,6 +2262,74 @@ describe('createToolExecuteHandler', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('shares one retry-wait budget across skill calls in a tool batch', async () => {
|
||||
const attempts = new Map<string, number>();
|
||||
const batchUploadCodeEnvFiles = jest.fn(async ({ id }: { id: string }) => {
|
||||
const attempt = (attempts.get(id) ?? 0) + 1;
|
||||
attempts.set(id, attempt);
|
||||
if (attempt === 1) {
|
||||
const error = Object.assign(new Error('Request failed with status code 429'), {
|
||||
isAxiosError: true,
|
||||
response: { status: 429, headers: { 'retry-after': '0' } },
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
const name = id.replace(/-id$/, '');
|
||||
return {
|
||||
storage_session_id: `session-${name}`,
|
||||
files: [{ fileId: `file-${name}`, filename: `skills/${name}/references/style.md` }],
|
||||
};
|
||||
});
|
||||
const handler = createPrimingSkillHandler('fallback-skill', batchUploadCodeEnvFiles);
|
||||
|
||||
const results = await invokeHandler(handler, [
|
||||
{
|
||||
id: 'call_first_skill',
|
||||
name: Constants.SKILL_TOOL,
|
||||
args: { skillName: 'first-skill' },
|
||||
},
|
||||
{
|
||||
id: 'call_second_skill',
|
||||
name: Constants.SKILL_TOOL,
|
||||
args: { skillName: 'second-skill' },
|
||||
},
|
||||
]);
|
||||
|
||||
expect(batchUploadCodeEnvFiles).toHaveBeenCalledTimes(3);
|
||||
expect(results.filter((result) => result.artifact != null)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('returns cancellation instead of a successful skill when upload recovery is aborted', async () => {
|
||||
const controller = new AbortController();
|
||||
const batchUploadCodeEnvFiles = jest.fn(
|
||||
({ signal }: { signal?: AbortSignal }) =>
|
||||
new Promise<never>((_resolve, reject) => {
|
||||
signal?.addEventListener('abort', () => reject(signal.reason), { once: true });
|
||||
}),
|
||||
);
|
||||
const handler = createPrimingSkillHandler('cancelled-skill', batchUploadCodeEnvFiles);
|
||||
const resultPromise = new Promise<ToolExecuteResult[]>((resolve, reject) => {
|
||||
handler.handle('on_tool_execute', {
|
||||
toolCalls: [
|
||||
{
|
||||
id: 'call_cancelled_skill',
|
||||
name: Constants.SKILL_TOOL,
|
||||
args: { skillName: 'cancelled-skill' },
|
||||
},
|
||||
],
|
||||
signal: controller.signal,
|
||||
resolve,
|
||||
reject,
|
||||
} as ToolExecuteBatchRequest);
|
||||
});
|
||||
setTimeout(() => controller.abort(), 10);
|
||||
|
||||
const [result] = await resultPromise;
|
||||
|
||||
expect(result.status).toBe('error');
|
||||
expect(result.content).not.toContain('Skill "cancelled-skill" loaded');
|
||||
});
|
||||
|
||||
it("read_file pins lookup to the primed skill's _id when manually invoked this turn (no shadowing on collision)", async () => {
|
||||
/* Same-name collision corner: the resolver primed a specific doc
|
||||
(its `_id` is in `skillPrimedIdsByName`). If read_file used
|
||||
|
|
@ -5248,6 +5372,7 @@ describe('createToolExecuteHandler', () => {
|
|||
listWorkspaceFiles?: ToolExecuteOptions['listWorkspaceFiles'];
|
||||
readSandboxFile?: ToolExecuteOptions['readSandboxFile'];
|
||||
readSandboxImage?: ToolExecuteOptions['readSandboxImage'];
|
||||
runSignal?: AbortSignal;
|
||||
getSkillByName?: ToolExecuteOptions['getSkillByName'];
|
||||
getAuthorSkillByName?: ToolExecuteOptions['getAuthorSkillByName'];
|
||||
}) {
|
||||
|
|
@ -5265,6 +5390,7 @@ describe('createToolExecuteHandler', () => {
|
|||
}));
|
||||
return createToolExecuteHandler({
|
||||
loadTools,
|
||||
runSignal: params.runSignal,
|
||||
getSkillByName: params.getSkillByName,
|
||||
getAuthorSkillByName: params.getAuthorSkillByName,
|
||||
readWorkspaceFile: params.readWorkspaceFile,
|
||||
|
|
@ -6900,6 +7026,7 @@ describe('createToolExecuteHandler', () => {
|
|||
* bytes, which was the matplotlib-shape mojibake regression.
|
||||
*/
|
||||
it('returns a sandbox image as an image_url artifact the model can see', async () => {
|
||||
const controller = new AbortController();
|
||||
const readSandboxFile = jest.fn();
|
||||
const readSandboxImage = jest.fn(async () => ({ base64: PNG_B64, bytes: pngBytes }));
|
||||
const handler = makeReadFileHandler({
|
||||
|
|
@ -6909,14 +7036,21 @@ describe('createToolExecuteHandler', () => {
|
|||
readSandboxImage,
|
||||
});
|
||||
|
||||
const [result] = await invokeHandler(handler, [
|
||||
{
|
||||
id: 'call_png',
|
||||
name: Constants.READ_FILE,
|
||||
args: { path: '/mnt/data/simple_graph.png' },
|
||||
codeSessionContext: { session_id: 'sess-Z', files: [] },
|
||||
} as unknown as ToolCallRequest,
|
||||
]);
|
||||
const [result] = await new Promise<ToolExecuteResult[]>((resolve, reject) => {
|
||||
handler.handle('on_tool_execute', {
|
||||
toolCalls: [
|
||||
{
|
||||
id: 'call_png',
|
||||
name: Constants.READ_FILE,
|
||||
args: { path: '/mnt/data/simple_graph.png' },
|
||||
codeSessionContext: { session_id: 'sess-Z', files: [] },
|
||||
} as unknown as ToolCallRequest,
|
||||
],
|
||||
signal: controller.signal,
|
||||
resolve,
|
||||
reject,
|
||||
} as ToolExecuteBatchRequest);
|
||||
});
|
||||
|
||||
expect(readSandboxFile).not.toHaveBeenCalled();
|
||||
expect(readSandboxImage).toHaveBeenCalledWith(
|
||||
|
|
@ -6924,6 +7058,7 @@ describe('createToolExecuteHandler', () => {
|
|||
file_path: '/mnt/data/simple_graph.png',
|
||||
session_id: 'sess-Z',
|
||||
maxBytes: expect.any(Number),
|
||||
signal: controller.signal,
|
||||
}),
|
||||
);
|
||||
expect(result.status).toBe('success');
|
||||
|
|
|
|||
|
|
@ -78,6 +78,14 @@ import {
|
|||
SEARCH_WORKSPACE_TOOL_NAME,
|
||||
isCodeSessionToolName,
|
||||
} from './tools';
|
||||
import {
|
||||
createCodeApiRateLimitBudget,
|
||||
isAbortError,
|
||||
logAxiosError,
|
||||
truncateMiddle,
|
||||
runOutsideTracing,
|
||||
getSafeErrorMetadata,
|
||||
} from '~/utils';
|
||||
import {
|
||||
ContentFilterError,
|
||||
contentFilterModelBoundBlockResponse,
|
||||
|
|
@ -87,13 +95,6 @@ import {
|
|||
BACKGROUND_TASK_ABORT_GRACE_MS,
|
||||
BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS,
|
||||
} from './backgroundCompletion';
|
||||
import {
|
||||
isAbortError,
|
||||
logAxiosError,
|
||||
truncateMiddle,
|
||||
runOutsideTracing,
|
||||
getSafeErrorMetadata,
|
||||
} from '~/utils';
|
||||
import {
|
||||
WorkspaceToolHttpError,
|
||||
WORKSPACE_EDIT_MAX_COUNT,
|
||||
|
|
@ -237,6 +238,8 @@ export interface ToolExecuteOptions {
|
|||
configurable?: Record<string, unknown>,
|
||||
/** SDK-owned live caller capability projection for this agent context. */
|
||||
callerCapabilityProjection?: CallerCapabilityProjectionSnapshot,
|
||||
/** Effective cancellation signal for this tool-execute batch. */
|
||||
signal?: AbortSignal,
|
||||
) => Promise<{
|
||||
loadedTools: StructuredToolInterface[];
|
||||
/** Additional configurable properties to merge (e.g., userMCPAuthMap) */
|
||||
|
|
@ -466,6 +469,7 @@ export interface ToolExecuteOptions {
|
|||
codeApiBaseUrl?: string;
|
||||
executionProfile?: CodeExecutionContext['executionProfile'];
|
||||
bridgeWorkerId?: string;
|
||||
signal?: AbortSignal;
|
||||
}) => Promise<{
|
||||
storage_session_id: string;
|
||||
files: Array<{ fileId: string; filename: string }>;
|
||||
|
|
@ -626,6 +630,7 @@ export interface ToolExecuteOptions {
|
|||
/** In-sandbox size cap; files larger than this return `tooLarge` without transferring bytes. */
|
||||
maxBytes?: number;
|
||||
req?: ServerRequest;
|
||||
signal?: AbortSignal;
|
||||
}) => Promise<
|
||||
| { base64: string; bytes: number }
|
||||
/** `size`: over `maxBytes`. `round_trips`: within the byte cap, but more
|
||||
|
|
@ -2119,6 +2124,7 @@ async function handleSandboxImageRead(
|
|||
req?: ServerRequest,
|
||||
codeExecutionContext?: CodeExecutionContext,
|
||||
onSuccess?: () => void,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ToolExecuteResult> {
|
||||
const filtered = filteredBinaryFileResult(tc, req, filePath);
|
||||
if (filtered != null) {
|
||||
|
|
@ -2153,6 +2159,7 @@ async function handleSandboxImageRead(
|
|||
session_id: ctx?.session_id,
|
||||
files: ctx?.files,
|
||||
maxBytes: MAX_SANDBOX_INLINE_IMAGE_BYTES,
|
||||
...(signal ? { signal } : {}),
|
||||
...codeExecutionRequestParams(codeExecutionContext),
|
||||
...(req ? { req } : {}),
|
||||
});
|
||||
|
|
@ -2235,10 +2242,20 @@ async function handleSandboxFileFallback(
|
|||
req?: ServerRequest,
|
||||
codeExecutionContext?: CodeExecutionContext,
|
||||
onSuccess?: () => void,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ToolExecuteResult> {
|
||||
const ext = lowercaseExtension(filePath);
|
||||
if (SANDBOX_IMAGE_EXTENSIONS.has(ext)) {
|
||||
return handleSandboxImageRead(tc, filePath, ext, options, req, codeExecutionContext, onSuccess);
|
||||
return handleSandboxImageRead(
|
||||
tc,
|
||||
filePath,
|
||||
ext,
|
||||
options,
|
||||
req,
|
||||
codeExecutionContext,
|
||||
onSuccess,
|
||||
signal,
|
||||
);
|
||||
}
|
||||
const filteredName = filteredFileNameResult(tc, req, filePath);
|
||||
if (filteredName != null) {
|
||||
|
|
@ -4366,6 +4383,7 @@ async function handleReadFileCall(
|
|||
req,
|
||||
codeExecutionContext,
|
||||
onSandboxReadSuccess,
|
||||
signal,
|
||||
);
|
||||
}
|
||||
return {
|
||||
|
|
@ -4402,6 +4420,7 @@ async function handleReadFileCall(
|
|||
req,
|
||||
codeExecutionContext,
|
||||
onSandboxReadSuccess,
|
||||
signal,
|
||||
);
|
||||
}
|
||||
return {
|
||||
|
|
@ -4429,6 +4448,7 @@ async function handleReadFileCall(
|
|||
req,
|
||||
codeExecutionContext,
|
||||
onSandboxReadSuccess,
|
||||
signal,
|
||||
);
|
||||
}
|
||||
return {
|
||||
|
|
@ -4498,6 +4518,7 @@ async function handleReadFileCall(
|
|||
req,
|
||||
codeExecutionContext,
|
||||
onSandboxReadSuccess,
|
||||
signal,
|
||||
);
|
||||
}
|
||||
return {
|
||||
|
|
@ -4544,6 +4565,7 @@ async function handleReadFileCall(
|
|||
req,
|
||||
codeExecutionContext,
|
||||
onSandboxReadSuccess,
|
||||
signal,
|
||||
);
|
||||
}
|
||||
return {
|
||||
|
|
@ -4869,6 +4891,8 @@ async function handleSkillToolCall(
|
|||
options: ToolExecuteOptions,
|
||||
agentId?: string,
|
||||
req?: ServerRequest,
|
||||
signal?: AbortSignal,
|
||||
rateLimitBudget?: import('~/utils').CodeApiRateLimitBudget,
|
||||
): Promise<ToolExecuteResult> {
|
||||
const {
|
||||
getSkillByName,
|
||||
|
|
@ -4984,7 +5008,9 @@ async function handleSkillToolCall(
|
|||
) {
|
||||
let primeResult: PrimeSkillFilesResult | null = null;
|
||||
try {
|
||||
signal?.throwIfAborted();
|
||||
const skillFiles = await listSkillFiles(skill._id);
|
||||
signal?.throwIfAborted();
|
||||
primeResult = await primeSkillFiles({
|
||||
skill,
|
||||
skillFiles,
|
||||
|
|
@ -4995,6 +5021,8 @@ async function handleSkillToolCall(
|
|||
checkIfActive,
|
||||
updateSkillFileCodeEnvIds,
|
||||
codeExecutionContext,
|
||||
signal,
|
||||
rateLimitBudget,
|
||||
});
|
||||
if (primeResult) {
|
||||
/* `session_id` at the top of the artifact is the (representative)
|
||||
|
|
@ -5019,6 +5047,9 @@ async function handleSkillToolCall(
|
|||
};
|
||||
}
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) {
|
||||
throw error;
|
||||
}
|
||||
if (isContentFilterError(error)) {
|
||||
return error instanceof ContentFilterError
|
||||
? errorResult(tc, modelBoundContentFilterErrorMessage(error.body))
|
||||
|
|
@ -5303,6 +5334,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
|
|||
agentId,
|
||||
sourceConfigurable,
|
||||
callerCapabilityProjection,
|
||||
runSignal,
|
||||
);
|
||||
const toolMap = new Map(loadedTools.map((t) => [t.name, t]));
|
||||
const loadedConfigurable = toolConfigurable as Record<string, unknown> | undefined;
|
||||
|
|
@ -6186,6 +6218,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
|
|||
};
|
||||
};
|
||||
|
||||
const batchReq = mergedConfigurable?.req as ServerRequest | undefined;
|
||||
const batchCodeApiRateLimitBudget = createCodeApiRateLimitBudget(
|
||||
batchReq?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
|
||||
);
|
||||
const results: ToolExecuteResult[] = await Promise.all(
|
||||
toolCalls.map(async (tc: ToolCallRequest) => {
|
||||
const preloadedNameBlock = preloadedNameBlocks.get(tc);
|
||||
|
|
@ -6478,6 +6514,8 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
|
|||
options,
|
||||
agentId,
|
||||
req,
|
||||
runSignal,
|
||||
batchCodeApiRateLimitBudget,
|
||||
);
|
||||
} else if (tc.name === Constants.READ_FILE) {
|
||||
handlerResult = await handleReadFileCall(
|
||||
|
|
|
|||
|
|
@ -255,6 +255,7 @@ describe('createAgentChatCompletion - MCP permission user propagation', () => {
|
|||
'agent_test',
|
||||
expect.objectContaining({ requestBody: runArgs.requestBody }),
|
||||
undefined,
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ jest.mock('./run', () => ({
|
|||
|
||||
import { Readable } from 'stream';
|
||||
import { Types } from 'mongoose';
|
||||
import { createCodeApiUploadRegistry } from '~/utils';
|
||||
import { primeInvokedSkills, primeInvokedSkillsForProfiles, primeSkillFiles } from './skillFiles';
|
||||
import type {
|
||||
PrimeInvokedSkillsDeps,
|
||||
|
|
@ -23,6 +24,7 @@ import type {
|
|||
|
||||
const SKILL_ID = new Types.ObjectId();
|
||||
const SKILL_VERSION = 7;
|
||||
const uploadRegistry = createCodeApiUploadRegistry();
|
||||
|
||||
function makeDeps(overrides: Partial<PrimeInvokedSkillsDeps> = {}): PrimeInvokedSkillsDeps {
|
||||
const listSkillFiles = jest.fn().mockResolvedValue([]);
|
||||
|
|
@ -31,7 +33,10 @@ function makeDeps(overrides: Partial<PrimeInvokedSkillsDeps> = {}): PrimeInvoked
|
|||
files: [],
|
||||
});
|
||||
return {
|
||||
req: { user: { id: 'user-1' } } as PrimeInvokedSkillsDeps['req'],
|
||||
req: {
|
||||
user: { id: 'user-1', tenantId: 'tenant-1' },
|
||||
app: { locals: { codeApiUploadRegistry: uploadRegistry } },
|
||||
} as unknown as PrimeInvokedSkillsDeps['req'],
|
||||
payload: [{ role: 'assistant', content: [] }],
|
||||
accessibleSkillIds: [SKILL_ID],
|
||||
codeEnvAvailable: true,
|
||||
|
|
@ -90,6 +95,66 @@ describe('primeInvokedSkills — execute_code capability gate', () => {
|
|||
expect(deps.listSkillFiles).toHaveBeenCalledWith(SKILL_ID);
|
||||
});
|
||||
|
||||
it('shares one retry-wait budget across historical skill uploads', async () => {
|
||||
const skillIds = [new Types.ObjectId(), new Types.ObjectId()];
|
||||
const skillNames = ['first-skill', 'second-skill'];
|
||||
mockExtract.mockReturnValue(new Set(skillNames));
|
||||
const getSkillByName = jest.fn(async (name: string) => {
|
||||
const index = skillNames.indexOf(name);
|
||||
return {
|
||||
_id: skillIds[index],
|
||||
name,
|
||||
body: `${name} body`,
|
||||
version: 1,
|
||||
fileCount: 1,
|
||||
};
|
||||
});
|
||||
const listSkillFiles = jest.fn(async () => [
|
||||
{
|
||||
relativePath: 'references/style.md',
|
||||
filename: 'style.md',
|
||||
filepath: '/storage/style.md',
|
||||
source: 's3',
|
||||
bytes: 5,
|
||||
},
|
||||
]);
|
||||
const attempts = new Map<string, number>();
|
||||
const batchUploadCodeEnvFiles = jest.fn(async ({ id }: { id: string }) => {
|
||||
const attempt = (attempts.get(id) ?? 0) + 1;
|
||||
attempts.set(id, attempt);
|
||||
if (attempt === 1) {
|
||||
const error = new Error('Request failed with status code 429') as Error & {
|
||||
isAxiosError: boolean;
|
||||
response: { status: number; headers: Record<string, string> };
|
||||
};
|
||||
error.isAxiosError = true;
|
||||
error.response = { status: 429, headers: { 'retry-after': '0' } };
|
||||
throw error;
|
||||
}
|
||||
const name = skillNames[skillIds.findIndex((skillId) => skillId.toString() === id)];
|
||||
return {
|
||||
storage_session_id: `session-${name}`,
|
||||
files: [{ fileId: `file-${name}`, filename: `skills/${name}/references/style.md` }],
|
||||
};
|
||||
});
|
||||
const deps = makeDeps({
|
||||
getSkillByName,
|
||||
listSkillFiles,
|
||||
getStrategyFunctions: jest.fn().mockReturnValue({
|
||||
getDownloadStream: jest.fn().mockResolvedValue(Readable.from(Buffer.from('style'))),
|
||||
}),
|
||||
batchUploadCodeEnvFiles,
|
||||
});
|
||||
deps.req.config = {
|
||||
endpoints: { agents: { codeApiUploadConcurrency: 1, codeApiMaxRetryWaitMs: 1_000 } },
|
||||
} as never;
|
||||
|
||||
const result = await primeInvokedSkills(deps);
|
||||
|
||||
expect(batchUploadCodeEnvFiles).toHaveBeenCalledTimes(3);
|
||||
expect(result.initialSessions?.get('execute_code')?.files).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('calls batchUploadCodeEnvFiles without an apiKey when files are returned', async () => {
|
||||
const fileRecords = [
|
||||
{
|
||||
|
|
@ -527,6 +592,7 @@ describe('primeInvokedSkills — execute_code capability gate', () => {
|
|||
},
|
||||
deps.req,
|
||||
undefined,
|
||||
undefined,
|
||||
);
|
||||
const codeSession = result.initialSessions?.get('execute_code');
|
||||
expect(codeSession?.files).toEqual([
|
||||
|
|
@ -698,6 +764,82 @@ describe('primeInvokedSkillsForProfiles', () => {
|
|||
).toEqual(['stateful:first', 'stateful:second']);
|
||||
});
|
||||
|
||||
it('shares one retry-wait budget across execution profiles', async () => {
|
||||
const file = {
|
||||
relativePath: 'references/style.md',
|
||||
filename: 'style.md',
|
||||
filepath: '/storage/style.md',
|
||||
source: 's3',
|
||||
bytes: 5,
|
||||
};
|
||||
const attempts = new Map<string, number>();
|
||||
const batchUploadCodeEnvFiles = jest.fn(
|
||||
async ({ executionProfile }: { executionProfile?: string }) => {
|
||||
const profile = executionProfile ?? 'default';
|
||||
const attempt = (attempts.get(profile) ?? 0) + 1;
|
||||
attempts.set(profile, attempt);
|
||||
if (attempt === 1) {
|
||||
if (profile === 'stateful') {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
}
|
||||
const error = Object.assign(new Error('Request failed with status code 429'), {
|
||||
isAxiosError: true,
|
||||
response: { status: 429, headers: { 'retry-after': '0' } },
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
storage_session_id: `${profile}-session`,
|
||||
files: [
|
||||
{
|
||||
fileId: `${profile}-file`,
|
||||
filename: 'skills/brand-guidelines/references/style.md',
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
);
|
||||
const { codeEnvAvailable: _codeEnvAvailable, ...baseDeps } = makeDeps({
|
||||
listSkillFiles: jest.fn().mockResolvedValue([file]),
|
||||
getStrategyFunctions: jest.fn().mockReturnValue({
|
||||
getDownloadStream: jest.fn().mockResolvedValue(Readable.from(Buffer.from('style'))),
|
||||
}),
|
||||
batchUploadCodeEnvFiles,
|
||||
});
|
||||
baseDeps.req.config = {
|
||||
endpoints: { agents: { codeApiMaxRetryWaitMs: 1_000 } },
|
||||
} as never;
|
||||
|
||||
const result = await primeInvokedSkillsForProfiles({
|
||||
...baseDeps,
|
||||
executionProfiles: [
|
||||
{
|
||||
codeExecutionContext: {
|
||||
baseUrl: 'https://default.example.com/v1',
|
||||
codeSessionKey: 'execute_code',
|
||||
executionProfile: 'default',
|
||||
statefulSessions: false,
|
||||
},
|
||||
codeSessionKeys: ['execute_code'],
|
||||
},
|
||||
{
|
||||
codeExecutionContext: {
|
||||
baseUrl: 'https://stateful.example.com/v1',
|
||||
codeSessionKey: 'execute_code:stateful',
|
||||
executionProfile: 'stateful',
|
||||
executionRouteKey: 'stateful:one',
|
||||
statefulSessions: true,
|
||||
},
|
||||
codeSessionKeys: ['execute_code:stateful'],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(batchUploadCodeEnvFiles).toHaveBeenCalledTimes(3);
|
||||
expect(result.initialSessions?.has('execute_code')).toBe(true);
|
||||
expect(result.initialSessions?.has('execute_code:stateful')).toBe(false);
|
||||
});
|
||||
|
||||
it('keeps a successful profile Skill body and identity after another profile lookup fails', async () => {
|
||||
const {
|
||||
codeEnvAvailable: _codeEnvAvailable,
|
||||
|
|
@ -768,7 +910,10 @@ function makeSkillFilesDeps(overrides: Partial<PrimeSkillFilesParams> = {}): Pri
|
|||
version: SKILL_VERSION,
|
||||
},
|
||||
skillFiles: [],
|
||||
req: { user: { id: 'user-1' } } as PrimeSkillFilesParams['req'],
|
||||
req: {
|
||||
user: { id: 'user-1', tenantId: 'tenant-1' },
|
||||
app: { locals: { codeApiUploadRegistry: uploadRegistry } },
|
||||
} as unknown as PrimeSkillFilesParams['req'],
|
||||
getStrategyFunctions: jest.fn().mockReturnValue({
|
||||
getDownloadStream: jest.fn().mockResolvedValue(Readable.from(Buffer.from(''))),
|
||||
}),
|
||||
|
|
@ -861,10 +1006,15 @@ describe('primeSkillFiles — resource identity propagation', () => {
|
|||
const result = await primeSkillFiles(deps);
|
||||
|
||||
expect(batchUploadCodeEnvFiles).not.toHaveBeenCalled();
|
||||
expect(deps.getSessionInfo).toHaveBeenCalledWith(cachedRef, deps.req, {
|
||||
baseUrl: 'https://stateful-code.example.com',
|
||||
executionProfile: 'stateful',
|
||||
});
|
||||
expect(deps.getSessionInfo).toHaveBeenCalledWith(
|
||||
cachedRef,
|
||||
deps.req,
|
||||
{
|
||||
baseUrl: 'https://stateful-code.example.com',
|
||||
executionProfile: 'stateful',
|
||||
},
|
||||
undefined,
|
||||
);
|
||||
expect(result?.files).toEqual([
|
||||
{
|
||||
id: 'file-cached',
|
||||
|
|
@ -969,7 +1119,8 @@ describe('primeSkillFiles — resource identity propagation', () => {
|
|||
const batchUploadCodeEnvFiles = jest.fn();
|
||||
const deps = makeSkillFilesDeps({
|
||||
req: {
|
||||
user: { id: 'user-1' },
|
||||
user: { id: 'user-1', tenantId: 'tenant-1' },
|
||||
app: { locals: { codeApiUploadRegistry: uploadRegistry } },
|
||||
config: {
|
||||
filters: {
|
||||
skills: {
|
||||
|
|
@ -980,7 +1131,7 @@ describe('primeSkillFiles — resource identity propagation', () => {
|
|||
},
|
||||
},
|
||||
},
|
||||
} as PrimeSkillFilesParams['req'],
|
||||
} as unknown as PrimeSkillFilesParams['req'],
|
||||
skillFiles: [
|
||||
{
|
||||
relativePath: 'references/style.md',
|
||||
|
|
@ -1019,7 +1170,8 @@ describe('primeSkillFiles — resource identity propagation', () => {
|
|||
const batchUploadCodeEnvFiles = jest.fn();
|
||||
const deps = makeSkillFilesDeps({
|
||||
req: {
|
||||
user: { id: 'user-1' },
|
||||
user: { id: 'user-1', tenantId: 'tenant-1' },
|
||||
app: { locals: { codeApiUploadRegistry: uploadRegistry } },
|
||||
config: {
|
||||
filters: {
|
||||
files: {
|
||||
|
|
@ -1029,7 +1181,7 @@ describe('primeSkillFiles — resource identity propagation', () => {
|
|||
},
|
||||
},
|
||||
},
|
||||
} as PrimeSkillFilesParams['req'],
|
||||
} as unknown as PrimeSkillFilesParams['req'],
|
||||
skillFiles: [
|
||||
{
|
||||
relativePath: 'references/style.md',
|
||||
|
|
@ -1077,7 +1229,7 @@ describe('primeSkillFiles — resource identity propagation', () => {
|
|||
},
|
||||
},
|
||||
},
|
||||
} as PrimeSkillFilesParams['req'],
|
||||
} as unknown as PrimeSkillFilesParams['req'],
|
||||
skillFiles: [
|
||||
{
|
||||
relativePath: 'references/sk-private-name.md',
|
||||
|
|
@ -1123,7 +1275,7 @@ describe('primeSkillFiles — resource identity propagation', () => {
|
|||
},
|
||||
},
|
||||
},
|
||||
} as PrimeSkillFilesParams['req'],
|
||||
} as unknown as PrimeSkillFilesParams['req'],
|
||||
skillFiles: [
|
||||
{
|
||||
relativePath: 'references/style.md',
|
||||
|
|
@ -1374,7 +1526,7 @@ describe('primeSkillFiles — resource identity propagation', () => {
|
|||
},
|
||||
},
|
||||
},
|
||||
} as PrimeSkillFilesParams['req'],
|
||||
} as unknown as PrimeSkillFilesParams['req'],
|
||||
skillFiles: [
|
||||
{
|
||||
relativePath: 'references/unavailable.txt',
|
||||
|
|
@ -1505,7 +1657,8 @@ describe('primeSkillFiles — resource identity propagation', () => {
|
|||
});
|
||||
const deps = makeSkillFilesDeps({
|
||||
req: {
|
||||
user: { id: 'user-1' },
|
||||
user: { id: 'user-1', tenantId: 'tenant-1' },
|
||||
app: { locals: { codeApiUploadRegistry: uploadRegistry } },
|
||||
config: {
|
||||
filters: {
|
||||
files: {
|
||||
|
|
@ -1517,7 +1670,7 @@ describe('primeSkillFiles — resource identity propagation', () => {
|
|||
},
|
||||
},
|
||||
},
|
||||
} as PrimeSkillFilesParams['req'],
|
||||
} as unknown as PrimeSkillFilesParams['req'],
|
||||
skillFiles: [
|
||||
{
|
||||
relativePath: 'references/archive.bin',
|
||||
|
|
@ -1679,7 +1832,72 @@ describe('primeSkillFiles — upload rate-limit resilience', () => {
|
|||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it('bounds concurrent batch uploads to 3 process-wide slots', async () => {
|
||||
it('does not upload a partial skill bundle when stream acquisition is canceled', async () => {
|
||||
const controller = new AbortController();
|
||||
const batchUploadCodeEnvFiles = jest.fn();
|
||||
const acquiredStream = Readable.from(Buffer.from('style'));
|
||||
const destroy = jest.spyOn(acquiredStream, 'destroy');
|
||||
const getDownloadStream = jest.fn(async () => {
|
||||
controller.abort();
|
||||
return acquiredStream;
|
||||
});
|
||||
|
||||
await expect(
|
||||
primeSkillFiles(
|
||||
makeSkillFilesDeps({
|
||||
signal: controller.signal,
|
||||
skillFiles: [styleFileRecord()],
|
||||
batchUploadCodeEnvFiles,
|
||||
getStrategyFunctions: jest.fn().mockReturnValue({ getDownloadStream }),
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({ name: 'AbortError' });
|
||||
expect(getDownloadStream).toHaveBeenCalledWith(expect.anything(), expect.any(String), {
|
||||
signal: controller.signal,
|
||||
});
|
||||
expect(destroy).toHaveBeenCalled();
|
||||
expect(batchUploadCodeEnvFiles).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not return a cache hit when cancellation arrives during its liveness check', async () => {
|
||||
const controller = new AbortController();
|
||||
const batchUploadCodeEnvFiles = jest.fn();
|
||||
const getSessionInfo = jest.fn(async () => {
|
||||
controller.abort();
|
||||
return '2026-05-06T00:00:00Z';
|
||||
});
|
||||
const cachedFile = {
|
||||
...styleFileRecord(),
|
||||
codeEnvRef: {
|
||||
kind: 'skill' as const,
|
||||
id: SKILL_ID.toString(),
|
||||
storage_session_id: 'session-cached',
|
||||
file_id: 'file-cached',
|
||||
version: SKILL_VERSION,
|
||||
},
|
||||
};
|
||||
|
||||
await expect(
|
||||
primeSkillFiles(
|
||||
makeSkillFilesDeps({
|
||||
signal: controller.signal,
|
||||
skillFiles: [cachedFile],
|
||||
batchUploadCodeEnvFiles,
|
||||
getSessionInfo,
|
||||
checkIfActive: jest.fn().mockReturnValue(true),
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({ name: 'AbortError' });
|
||||
expect(getSessionInfo).toHaveBeenCalledWith(
|
||||
expect.any(Object),
|
||||
expect.any(Object),
|
||||
undefined,
|
||||
controller.signal,
|
||||
);
|
||||
expect(batchUploadCodeEnvFiles).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('bounds concurrent batch uploads to 3 application-scoped slots', async () => {
|
||||
const gates = Array.from({ length: 5 }, () => deferred<ReturnType<typeof uploadResult>>());
|
||||
let uploadIndex = 0;
|
||||
const batchUploadCodeEnvFiles = jest
|
||||
|
|
|
|||
|
|
@ -18,7 +18,14 @@ import {
|
|||
inspectContent,
|
||||
UninspectableFileError,
|
||||
} from '~/protection';
|
||||
import { createConcurrencyLimiter, getCodeApiRetryAfterMs, getSafeErrorMetadata } from '~/utils';
|
||||
import {
|
||||
createCodeApiRateLimitBudget,
|
||||
getCodeApiUploadOptions,
|
||||
getSafeErrorMetadata,
|
||||
isAbortError,
|
||||
type CodeApiRateLimitBudget,
|
||||
withCodeApiUploadRecovery,
|
||||
} from '~/utils';
|
||||
import { seedCodeFilesIntoSessions, type CodeExecutionProfileRoute } from './codeFilesSession';
|
||||
import { ContentFilterError, isContentFilterError } from '~/middleware/contentFilter';
|
||||
import { getCodeExecutionRouteKey, type CodeExecutionContext } from './execution';
|
||||
|
|
@ -55,8 +62,16 @@ export interface PrimeSkillFilesParams {
|
|||
};
|
||||
skillFiles: SkillFileRecord[];
|
||||
req: ServerRequest;
|
||||
/** Optional operation-wide wait allowance shared by sibling skill primes. */
|
||||
rateLimitBudget?: CodeApiRateLimitBudget;
|
||||
/** Effective foreground cancellation signal for acquisition, waits, and transport. */
|
||||
signal?: AbortSignal;
|
||||
getStrategyFunctions: (source: string) => {
|
||||
getDownloadStream?: (req: ServerRequest, filepath: string) => Promise<NodeJS.ReadableStream>;
|
||||
getDownloadStream?: (
|
||||
req: ServerRequest,
|
||||
filepath: string,
|
||||
options?: { signal?: AbortSignal },
|
||||
) => Promise<NodeJS.ReadableStream>;
|
||||
[key: string]: unknown;
|
||||
};
|
||||
batchUploadCodeEnvFiles: (params: {
|
||||
|
|
@ -76,6 +91,7 @@ export interface PrimeSkillFilesParams {
|
|||
codeApiBaseUrl?: string;
|
||||
executionProfile?: CodeExecutionContext['executionProfile'];
|
||||
bridgeWorkerId?: string;
|
||||
signal?: AbortSignal;
|
||||
}) => Promise<{
|
||||
storage_session_id: string;
|
||||
files: Array<{ fileId: string; filename: string }>;
|
||||
|
|
@ -89,6 +105,7 @@ export interface PrimeSkillFilesParams {
|
|||
executionProfile?: CodeExecutionContext['executionProfile'];
|
||||
bridgeWorkerId?: string;
|
||||
},
|
||||
signal?: AbortSignal,
|
||||
) => Promise<string | null>;
|
||||
/** Trusted Code API route selected for the executing agent. */
|
||||
codeExecutionContext?: Pick<
|
||||
|
|
@ -127,16 +144,21 @@ export interface PrimeSkillFilesResult {
|
|||
}>;
|
||||
}
|
||||
|
||||
/** Cap on concurrent skill batch uploads per process. Bounds burst pressure
|
||||
* on codeapi's per-user upload limiter (default 30 requests / 5 min). */
|
||||
const SKILL_UPLOAD_CONCURRENCY = 3;
|
||||
|
||||
/** Retry a 429'd upload only when the server's Retry-After fits under this
|
||||
* cap; a longer wait would stall a live chat turn worse than degrading. */
|
||||
const MAX_RETRY_AFTER_MS = 15_000;
|
||||
|
||||
const uploadSlots = createConcurrencyLimiter(SKILL_UPLOAD_CONCURRENCY);
|
||||
const inflightPrimes = new Map<string, Promise<PrimeSkillFilesResult | null>>();
|
||||
const signalIds = new WeakMap<AbortSignal, number>();
|
||||
let nextSignalId = 1;
|
||||
|
||||
function getSignalScope(signal?: AbortSignal): string {
|
||||
if (!signal) {
|
||||
return 'unscoped';
|
||||
}
|
||||
let id = signalIds.get(signal);
|
||||
if (id == null) {
|
||||
id = nextSignalId++;
|
||||
signalIds.set(signal, id);
|
||||
}
|
||||
return String(id);
|
||||
}
|
||||
|
||||
type SkillUploadFiles = Array<{ stream: NodeJS.ReadableStream; filename: string }>;
|
||||
type SkillCodeEnvRef = Extract<CodeEnvRef, { kind: 'skill' }>;
|
||||
|
|
@ -148,19 +170,16 @@ function isCurrentSkillRef(
|
|||
return ref?.kind === 'skill' && ref.version === skillVersion;
|
||||
}
|
||||
|
||||
/** Single retry on 429, honoring Retry-After up to MAX_RETRY_AFTER_MS.
|
||||
* Runs inside an upload slot so the wait also brakes queued uploads. */
|
||||
async function retryOn429<T>(attempt: () => Promise<T>, label: string): Promise<T> {
|
||||
function destroySkillUploadStream(stream: NodeJS.ReadableStream): void {
|
||||
try {
|
||||
return await attempt();
|
||||
} catch (error) {
|
||||
const retryAfterMs = getCodeApiRetryAfterMs(error);
|
||||
if (retryAfterMs == null || retryAfterMs > MAX_RETRY_AFTER_MS) {
|
||||
throw error;
|
||||
if (
|
||||
'destroy' in stream &&
|
||||
typeof (stream as NodeJS.ReadableStream & { destroy?: () => void }).destroy === 'function'
|
||||
) {
|
||||
(stream as NodeJS.ReadableStream & { destroy: () => void }).destroy();
|
||||
}
|
||||
logger.warn(`[primeSkillFiles] Rate-limited priming ${label}; retrying in ${retryAfterMs}ms`);
|
||||
await new Promise((resolve) => setTimeout(resolve, retryAfterMs));
|
||||
return attempt();
|
||||
} catch {
|
||||
/* Preserve the run cancellation even if a provider stream rejects cleanup. */
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -171,7 +190,8 @@ async function collectSkillUploadFiles(
|
|||
params: PrimeSkillFilesParams,
|
||||
inspectedBuffers: ReadonlyMap<SkillFileRecord, Buffer>,
|
||||
): Promise<SkillUploadFiles> {
|
||||
const { skill, skillFiles, req, getStrategyFunctions } = params;
|
||||
const { skill, skillFiles, req, getStrategyFunctions, signal } = params;
|
||||
signal?.throwIfAborted();
|
||||
const filesToUpload: SkillUploadFiles = [];
|
||||
|
||||
// SKILL.md from the skill body
|
||||
|
|
@ -184,6 +204,7 @@ async function collectSkillUploadFiles(
|
|||
// Bundled files from storage (parallel stream acquisition)
|
||||
const streamResults = await Promise.allSettled(
|
||||
skillFiles.map(async (file) => {
|
||||
signal?.throwIfAborted();
|
||||
const inspected = inspectedBuffers.get(file);
|
||||
if (inspected != null) {
|
||||
return {
|
||||
|
|
@ -198,10 +219,28 @@ async function collectSkillUploadFiles(
|
|||
);
|
||||
return null;
|
||||
}
|
||||
const stream = await strategy.getDownloadStream(req, resolveDownloadPath(file));
|
||||
const stream = await strategy.getDownloadStream(req, resolveDownloadPath(file), { signal });
|
||||
if (signal?.aborted) {
|
||||
destroySkillUploadStream(stream);
|
||||
signal.throwIfAborted();
|
||||
}
|
||||
return { stream, filename: `${SKILL_FILE_PREFIX}${skill.name}/${file.relativePath}` };
|
||||
}),
|
||||
);
|
||||
/* Do not let allSettled turn foreground cancellation into a skipped bundle
|
||||
* member. A partial skill upload can look successful while leaving required
|
||||
* files unavailable to the sandbox. */
|
||||
if (signal?.aborted) {
|
||||
for (const file of filesToUpload) {
|
||||
destroySkillUploadStream(file.stream);
|
||||
}
|
||||
for (const result of streamResults) {
|
||||
if (result.status === 'fulfilled' && result.value) {
|
||||
destroySkillUploadStream(result.value.stream);
|
||||
}
|
||||
}
|
||||
signal.throwIfAborted();
|
||||
}
|
||||
for (const result of streamResults) {
|
||||
if (result.status === 'fulfilled' && result.value) {
|
||||
filesToUpload.push(result.value);
|
||||
|
|
@ -308,10 +347,14 @@ function throwIfStoredSkillFileMustBeInspectable(req: ServerRequest): void {
|
|||
}
|
||||
}
|
||||
|
||||
async function bufferSkillFileStream(stream: NodeJS.ReadableStream): Promise<Buffer | null> {
|
||||
async function bufferSkillFileStream(
|
||||
stream: NodeJS.ReadableStream,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Buffer | null> {
|
||||
const chunks: Buffer[] = [];
|
||||
let bytes = 0;
|
||||
for await (const chunk of stream as AsyncIterable<Uint8Array | string>) {
|
||||
signal?.throwIfAborted();
|
||||
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
||||
bytes += buffer.length;
|
||||
if (bytes > MAX_INSPECTABLE_SKILL_FILE_BYTES) {
|
||||
|
|
@ -319,6 +362,7 @@ async function bufferSkillFileStream(stream: NodeJS.ReadableStream): Promise<Buf
|
|||
}
|
||||
chunks.push(buffer);
|
||||
}
|
||||
signal?.throwIfAborted();
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
|
|
@ -333,8 +377,8 @@ async function bufferSkillFileStream(stream: NodeJS.ReadableStream): Promise<Buf
|
|||
* documents for future freshness checks.
|
||||
*
|
||||
* Rate-limit resilience: concurrent primes of the same (skill, version)
|
||||
* share one flight, uploads are bounded process-wide, and a 429 retries
|
||||
* once per the server's Retry-After.
|
||||
* share one flight, uploads are bounded per Code API route and principal,
|
||||
* and 429 responses retry within a capped wait budget.
|
||||
*/
|
||||
export async function primeSkillFiles(
|
||||
params: PrimeSkillFilesParams,
|
||||
|
|
@ -348,7 +392,9 @@ export async function primeSkillFiles(
|
|||
const executionRouteKey = params.codeExecutionContext
|
||||
? getCodeExecutionRouteKey(params.codeExecutionContext)
|
||||
: 'default';
|
||||
const flightKey = `${executionRouteKey}:${params.skill._id}:v:${params.skill.version}`;
|
||||
/* A flight may share cancellation only with callers from the same run.
|
||||
* Cross-run sharing would let one user's Stop abort another live request. */
|
||||
const flightKey = `${executionRouteKey}:${params.skill._id}:v:${params.skill.version}:run:${getSignalScope(params.signal)}`;
|
||||
const inflight = inflightPrimes.get(flightKey);
|
||||
if (inflight) {
|
||||
return inflight;
|
||||
|
|
@ -373,7 +419,10 @@ async function executePrimeSkillFiles(
|
|||
checkIfActive,
|
||||
updateSkillFileCodeEnvIds,
|
||||
codeExecutionContext,
|
||||
rateLimitBudget,
|
||||
signal,
|
||||
} = params;
|
||||
signal?.throwIfAborted();
|
||||
const executionProfile = codeExecutionContext?.executionProfile ?? 'default';
|
||||
const executionRouteKey = codeExecutionContext
|
||||
? getCodeExecutionRouteKey(codeExecutionContext)
|
||||
|
|
@ -386,12 +435,14 @@ async function executePrimeSkillFiles(
|
|||
|
||||
if (inspectStoredMetadata) {
|
||||
for (const file of skillFiles) {
|
||||
signal?.throwIfAborted();
|
||||
assertStoredSkillFileNameAllowed(file, req);
|
||||
}
|
||||
}
|
||||
|
||||
if (inspectBundledFileContent) {
|
||||
for (const file of skillFiles) {
|
||||
signal?.throwIfAborted();
|
||||
if (file.bytes > MAX_INSPECTABLE_SKILL_FILE_BYTES) {
|
||||
throwIfStoredSkillFileMustBeInspectable(req);
|
||||
continue;
|
||||
|
|
@ -403,8 +454,10 @@ async function executePrimeSkillFiles(
|
|||
logger.warn('[primeSkillFiles] No download stream for stored skill file');
|
||||
continue;
|
||||
}
|
||||
const sourceStream = await strategy.getDownloadStream(req, resolveDownloadPath(file));
|
||||
const buffer = await bufferSkillFileStream(sourceStream);
|
||||
const sourceStream = await strategy.getDownloadStream(req, resolveDownloadPath(file), {
|
||||
signal,
|
||||
});
|
||||
const buffer = await bufferSkillFileStream(sourceStream, signal);
|
||||
if (buffer == null) {
|
||||
throwIfStoredSkillFileMustBeInspectable(req);
|
||||
continue;
|
||||
|
|
@ -412,6 +465,9 @@ async function executePrimeSkillFiles(
|
|||
assertStoredSkillFileAllowed(file, buffer, req);
|
||||
inspectedBuffers.set(file, buffer);
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) {
|
||||
throw error;
|
||||
}
|
||||
if (isContentFilterError(error)) {
|
||||
throw error;
|
||||
}
|
||||
|
|
@ -445,10 +501,11 @@ async function executePrimeSkillFiles(
|
|||
try {
|
||||
const checkResults = await Promise.all(
|
||||
Array.from(refsBySession.values()).map(async (ref) => {
|
||||
const lastModified = await getSessionInfo(ref, req, codeExecutionContext);
|
||||
const lastModified = await getSessionInfo(ref, req, codeExecutionContext, signal);
|
||||
return !!(lastModified && checkIfActive(lastModified));
|
||||
}),
|
||||
);
|
||||
signal?.throwIfAborted();
|
||||
const allActive = checkResults.every(Boolean);
|
||||
|
||||
if (allActive) {
|
||||
|
|
@ -480,7 +537,10 @@ async function executePrimeSkillFiles(
|
|||
return { storage_session_id: files[0].storage_session_id, files };
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) {
|
||||
throw error;
|
||||
}
|
||||
// Session check failed — fall through to re-upload
|
||||
}
|
||||
}
|
||||
|
|
@ -488,12 +548,20 @@ async function executePrimeSkillFiles(
|
|||
|
||||
const entityId = skill._id.toString();
|
||||
try {
|
||||
/* Streams open inside the slot (not while queued) and inside the retry
|
||||
* closure (a failed attempt consumes them). The slot bounds concurrent
|
||||
* uploads process-wide across both prime call sites. */
|
||||
const uploaded = await uploadSlots(() =>
|
||||
retryOn429(async () => {
|
||||
const filesToUpload = await collectSkillUploadFiles(params, inspectedBuffers);
|
||||
const uploadOptions = getCodeApiUploadOptions(req, executionRouteKey);
|
||||
const uploaded = await withCodeApiUploadRecovery({
|
||||
registry: req.app.locals.codeApiUploadRegistry,
|
||||
scope: uploadOptions.scope,
|
||||
concurrency: uploadOptions.concurrency,
|
||||
label: `priming skill "${skill.name}"`,
|
||||
budget: rateLimitBudget ?? createCodeApiRateLimitBudget(uploadOptions.retryWaitMs),
|
||||
signal,
|
||||
onWait: (waitMs) =>
|
||||
logger.warn(
|
||||
`[primeSkillFiles] Rate-limited priming skill "${skill.name}"; retrying in ${waitMs}ms`,
|
||||
),
|
||||
openSource: () => collectSkillUploadFiles(params, inspectedBuffers),
|
||||
upload: async (filesToUpload) => {
|
||||
if (filesToUpload.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
|
@ -503,7 +571,7 @@ async function executePrimeSkillFiles(
|
|||
/* Resource identity for codeapi's sessionKey: skill files share
|
||||
* cross-user-within-tenant under `<tenant>:skill:<id>:v:<version>`.
|
||||
* Bumping `skill.version` on edit naturally invalidates the prior
|
||||
* cache entry under the new sessionKey. */
|
||||
* cache entry under the new version's sessionKey. */
|
||||
kind: 'skill',
|
||||
id: entityId,
|
||||
version: skill.version,
|
||||
|
|
@ -518,10 +586,11 @@ async function executePrimeSkillFiles(
|
|||
codeApiBaseUrl: codeExecutionContext?.baseUrl,
|
||||
executionProfile: codeExecutionContext?.executionProfile,
|
||||
bridgeWorkerId: codeExecutionContext?.bridgeWorkerId,
|
||||
signal,
|
||||
});
|
||||
return { filesToUpload, result };
|
||||
}, `skill "${skill.name}"`),
|
||||
);
|
||||
},
|
||||
});
|
||||
if (uploaded == null) {
|
||||
return null;
|
||||
}
|
||||
|
|
@ -608,6 +677,9 @@ async function executePrimeSkillFiles(
|
|||
|
||||
return { storage_session_id: result.storage_session_id, files };
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) {
|
||||
throw error;
|
||||
}
|
||||
logger.error('[primeSkillFiles] Batch upload failed', getSafeErrorMetadata(error));
|
||||
return null;
|
||||
}
|
||||
|
|
@ -615,6 +687,8 @@ async function executePrimeSkillFiles(
|
|||
|
||||
export interface PrimeInvokedSkillsDeps {
|
||||
req: ServerRequest;
|
||||
/** Effective cancellation signal for the historical priming operation. */
|
||||
signal?: AbortSignal;
|
||||
/** Raw message payload (before formatAgentMessages). Used to extract invoked skill names. */
|
||||
payload?: Array<Partial<{ role: string; content: unknown }>>;
|
||||
/** Explicit durable names used by a validated event-actor preflight. */
|
||||
|
|
@ -642,6 +716,7 @@ export interface PrimeInvokedSkillsDeps {
|
|||
checkIfActive?: PrimeSkillFilesParams['checkIfActive'];
|
||||
updateSkillFileCodeEnvIds?: PrimeSkillFilesParams['updateSkillFileCodeEnvIds'];
|
||||
codeExecutionContext?: PrimeSkillFilesParams['codeExecutionContext'];
|
||||
rateLimitBudget?: CodeApiRateLimitBudget;
|
||||
}
|
||||
|
||||
export interface PrimeInvokedSkillsResult {
|
||||
|
|
@ -781,13 +856,19 @@ export async function primeInvokedSkills(
|
|||
ref,
|
||||
deps.req,
|
||||
deps.codeExecutionContext,
|
||||
deps.signal,
|
||||
);
|
||||
deps.signal?.throwIfAborted();
|
||||
return !!(lastModified && deps.checkIfActive?.(lastModified));
|
||||
} catch {
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) {
|
||||
throw error;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}),
|
||||
);
|
||||
deps.signal?.throwIfAborted();
|
||||
const allActive = checkResults.every(Boolean);
|
||||
|
||||
if (allActive) {
|
||||
|
|
@ -844,6 +925,9 @@ export async function primeInvokedSkills(
|
|||
kind: 'skill';
|
||||
version: number;
|
||||
}> = [];
|
||||
const uploadOptions = getCodeApiUploadOptions(deps.req, executionRouteKey);
|
||||
const rateLimitBudget =
|
||||
deps.rateLimitBudget ?? createCodeApiRateLimitBudget(uploadOptions.retryWaitMs);
|
||||
const primeResults = await Promise.allSettled(
|
||||
fileListResults.map(async ({ skill, files }) => {
|
||||
const result = await primeSkillFiles({
|
||||
|
|
@ -856,6 +940,8 @@ export async function primeInvokedSkills(
|
|||
checkIfActive: deps.checkIfActive,
|
||||
updateSkillFileCodeEnvIds: deps.updateSkillFileCodeEnvIds,
|
||||
codeExecutionContext: deps.codeExecutionContext,
|
||||
rateLimitBudget,
|
||||
signal: deps.signal,
|
||||
});
|
||||
return { skill, result };
|
||||
}),
|
||||
|
|
@ -876,7 +962,7 @@ export async function primeInvokedSkills(
|
|||
});
|
||||
}
|
||||
} else if (r.status === 'rejected') {
|
||||
if (isContentFilterError(r.reason)) {
|
||||
if (isContentFilterError(r.reason) || isAbortError(r.reason)) {
|
||||
throw r.reason;
|
||||
}
|
||||
logger.warn(
|
||||
|
|
@ -921,6 +1007,10 @@ export async function primeInvokedSkillsForProfiles(
|
|||
return primeInvokedSkills({ ...deps, codeEnvAvailable: false });
|
||||
}
|
||||
|
||||
const rateLimitBudget = createCodeApiRateLimitBudget(
|
||||
deps.req.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
|
||||
);
|
||||
|
||||
const profileResults = await Promise.all(
|
||||
deps.executionProfiles.map(async (profile) => ({
|
||||
profile,
|
||||
|
|
@ -929,6 +1019,8 @@ export async function primeInvokedSkillsForProfiles(
|
|||
codeEnvAvailable: true,
|
||||
codeExecutionContext: profile.codeExecutionContext,
|
||||
updateSkillFileCodeEnvIds: deps.updateSkillFileCodeEnvIds,
|
||||
rateLimitBudget,
|
||||
signal: deps.signal,
|
||||
}),
|
||||
})),
|
||||
);
|
||||
|
|
|
|||
|
|
@ -353,6 +353,83 @@ describe('createProvisionFilesCallback', () => {
|
|||
expect(provisionToCodeEnv).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('preserves cancellation raised during code provisioning and skips search work', async () => {
|
||||
const controller = new AbortController();
|
||||
const codeImpl = jest.fn(async () => {
|
||||
controller.abort();
|
||||
controller.signal.throwIfAborted();
|
||||
});
|
||||
const vectorImpl = jest.fn();
|
||||
const { provisionFiles } = buildHarness({
|
||||
contexts: [
|
||||
['agent-a', { provisionState: state([makeFile()], [makeFile({ file_id: 'search-1' })]) }],
|
||||
],
|
||||
codeImpl,
|
||||
vectorImpl,
|
||||
});
|
||||
|
||||
await expect(
|
||||
provisionFiles([Constants.EXECUTE_CODE, 'file_search'], 'agent-a', controller.signal),
|
||||
).rejects.toMatchObject({ name: 'AbortError' });
|
||||
expect(vectorImpl).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('preserves cancellation raised during vector provisioning', async () => {
|
||||
const controller = new AbortController();
|
||||
const vectorImpl = jest.fn(async () => {
|
||||
controller.abort();
|
||||
controller.signal.throwIfAborted();
|
||||
});
|
||||
const { provisionFiles } = buildHarness({
|
||||
contexts: [['agent-a', { provisionState: state([], [makeFile()]) }]],
|
||||
vectorImpl,
|
||||
});
|
||||
|
||||
await expect(
|
||||
provisionFiles(['file_search'], 'agent-a', controller.signal),
|
||||
).rejects.toMatchObject({ name: 'AbortError' });
|
||||
});
|
||||
|
||||
it('preserves a homogeneous Code API rate-limit failure from lazy provisioning', async () => {
|
||||
const rateLimit = Object.assign(new Error('rate limited'), {
|
||||
name: 'CodeApiRateLimitError',
|
||||
code: 'CODE_API_RATE_LIMITED',
|
||||
status: 429,
|
||||
statusCode: 429,
|
||||
retryAfterMs: 30_000,
|
||||
});
|
||||
const { provisionFiles } = buildHarness({
|
||||
contexts: [['agent-a', { provisionState: state([makeFile()], []) }]],
|
||||
codeImpl: jest.fn().mockRejectedValue(rateLimit),
|
||||
});
|
||||
|
||||
await expect(provisionFiles([Constants.EXECUTE_CODE], 'agent-a')).rejects.toBe(rateLimit);
|
||||
});
|
||||
|
||||
it('retains every failure when lazy code provisioning has mixed causes', async () => {
|
||||
const first = new Error('storage unavailable');
|
||||
const second = new Error('code api unavailable');
|
||||
const { provisionFiles } = buildHarness({
|
||||
contexts: [
|
||||
[
|
||||
'agent-a',
|
||||
{
|
||||
provisionState: state(
|
||||
[makeFile({ file_id: 'first' }), makeFile({ file_id: 'second' })],
|
||||
[],
|
||||
),
|
||||
},
|
||||
],
|
||||
],
|
||||
codeImpl: jest.fn().mockRejectedValueOnce(first).mockRejectedValueOnce(second),
|
||||
});
|
||||
|
||||
await expect(provisionFiles([Constants.EXECUTE_CODE], 'agent-a')).rejects.toMatchObject({
|
||||
cause: first,
|
||||
errors: [first, second],
|
||||
});
|
||||
});
|
||||
|
||||
it('shares embedding work across agents queueing the same search file', async () => {
|
||||
const shared = makeFile();
|
||||
const { provisionFiles, provisionToVectorDB, addEmbeddedEntity } = buildHarness({
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import type { CodeEnvRefUpdate, CodeExecutionRoute, ProvisionService } from './s
|
|||
import type { ProvisionState } from '~/agents/resources';
|
||||
import type { ServerRequest } from '~/types';
|
||||
import { claimCodeDestination, createCodeDestinationSet } from '~/files/code/destinations';
|
||||
import { createCodeApiRateLimitBudget, isCodeApiRateLimitError } from '~/utils';
|
||||
import { isCodeFileToolName } from '~/agents/tools';
|
||||
|
||||
/** Deferred database write produced by a successful provisioning call. */
|
||||
|
|
@ -238,6 +239,7 @@ export function createProvisionFilesCallback({
|
|||
/** Files whose provisioning rejected this turn; kept queued so a transient
|
||||
* outage can retry next turn instead of being silently dropped. */
|
||||
const failedCodeFiles: TFile[] = [];
|
||||
const codeFailureReasons: unknown[] = [];
|
||||
const failedVectorFiles: TFile[] = [];
|
||||
/* The graph seeded each tool call's code-session context from the sessions that
|
||||
* existed at run start, which predate this upload. Returning the refs lets the
|
||||
|
|
@ -248,6 +250,11 @@ export function createProvisionFilesCallback({
|
|||
: [];
|
||||
if (needsCode && provisionState.codeEnvFiles.length > 0) {
|
||||
const queuedCodeFiles = provisionState.codeEnvFiles;
|
||||
/** Every file in this tool-load batch shares one wait allowance. This
|
||||
* prevents a large recovery set from multiplying the live-turn delay. */
|
||||
const codeApiRateLimitBudget = createCodeApiRateLimitBudget(
|
||||
req.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
|
||||
);
|
||||
const destinations = createCodeDestinationSet();
|
||||
const existingCodeFiles = (
|
||||
ctx.tool_resources as Record<string, { files?: TFile[] } | undefined>
|
||||
|
|
@ -287,6 +294,7 @@ export function createProvisionFilesCallback({
|
|||
route: ctx.codeExecutionContext,
|
||||
sandboxFilename,
|
||||
signal,
|
||||
rateLimitBudget: codeApiRateLimitBudget,
|
||||
});
|
||||
signal?.throwIfAborted();
|
||||
/* primeCodeFiles re-reads the database and skips files without a stored
|
||||
|
|
@ -334,10 +342,15 @@ export function createProvisionFilesCallback({
|
|||
}
|
||||
}),
|
||||
);
|
||||
/* allSettled keeps independent file failures retryable, but cancellation belongs
|
||||
* to the whole tool run. Preserve it before translating rejections into queued
|
||||
* files or starting search provisioning. */
|
||||
signal?.throwIfAborted();
|
||||
results.forEach((result, index) => {
|
||||
if (result.status === 'rejected') {
|
||||
logger.error('[provisionFiles] Code env provisioning failed', result.reason);
|
||||
failedCodeFiles.push(queuedCodeFiles[index]);
|
||||
codeFailureReasons.push(result.reason);
|
||||
}
|
||||
});
|
||||
provisionState.codeEnvFiles = failedCodeFiles;
|
||||
|
|
@ -403,6 +416,7 @@ export function createProvisionFilesCallback({
|
|||
throw new Error(`Vector store did not embed "${file.filename}" (${file.file_id})`);
|
||||
}),
|
||||
);
|
||||
signal?.throwIfAborted();
|
||||
results.forEach((result, index) => {
|
||||
if (result.status === 'rejected') {
|
||||
logger.error('[provisionFiles] Vector DB provisioning failed', result.reason);
|
||||
|
|
@ -419,8 +433,15 @@ export function createProvisionFilesCallback({
|
|||
/* Provisioning failed outright, so the sandbox or vector store does not have the
|
||||
* attachment; running the tool anyway would answer from missing input. */
|
||||
if (failedCodeFiles.length > 0) {
|
||||
throw new Error(
|
||||
`Failed to provision ${failedCodeFiles.length} file(s) to the code environment; aborting tool execution rather than running without them`,
|
||||
if (codeFailureReasons.length > 0 && codeFailureReasons.every(isCodeApiRateLimitError)) {
|
||||
throw codeFailureReasons[0];
|
||||
}
|
||||
throw Object.assign(
|
||||
new Error(
|
||||
`Failed to provision ${failedCodeFiles.length} file(s) to the code environment; aborting tool execution rather than running without them`,
|
||||
codeFailureReasons[0] !== undefined ? { cause: codeFailureReasons[0] } : undefined,
|
||||
),
|
||||
{ errors: codeFailureReasons },
|
||||
);
|
||||
}
|
||||
if (failedVectorFiles.length > 0) {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { Readable } from 'node:stream';
|
||||
import { AxiosError, AxiosHeaders } from 'axios';
|
||||
import type { TFile } from 'librechat-data-provider';
|
||||
import type { ServerRequest } from '~/types';
|
||||
|
||||
|
|
@ -20,9 +21,13 @@ jest.mock('@librechat/agents', () => ({
|
|||
getCodeBaseURL: () => 'http://code.test/v1',
|
||||
}));
|
||||
|
||||
import { createCodeApiUploadRegistry } from '~/utils';
|
||||
import { createProvisionService } from './service';
|
||||
|
||||
const req = { user: { id: 'u1' } } as unknown as ServerRequest;
|
||||
const req = {
|
||||
user: { id: 'u1' },
|
||||
app: { locals: { codeApiUploadRegistry: createCodeApiUploadRegistry() } },
|
||||
} as unknown as ServerRequest;
|
||||
|
||||
const makeFile = (overrides: Partial<TFile> = {}): TFile =>
|
||||
({
|
||||
|
|
@ -73,6 +78,48 @@ describe('createProvisionService', () => {
|
|||
});
|
||||
|
||||
describe('provisionToCodeEnv', () => {
|
||||
it('waits out Code API throttling and reopens the upload stream', async () => {
|
||||
const rateLimit = new AxiosError('Request failed', 'ERR_BAD_REQUEST');
|
||||
rateLimit.response = {
|
||||
status: 429,
|
||||
statusText: 'Too Many Requests',
|
||||
headers: new AxiosHeaders({ 'retry-after': '0' }),
|
||||
config: { headers: new AxiosHeaders() },
|
||||
data: {},
|
||||
};
|
||||
const streams = [Readable.from('first'), Readable.from('second')];
|
||||
const getDownloadStream = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(streams[0])
|
||||
.mockResolvedValueOnce(streams[1]);
|
||||
const uploadCodeEnvFile = jest
|
||||
.fn()
|
||||
.mockRejectedValueOnce(rateLimit)
|
||||
.mockResolvedValueOnce({ storage_session_id: 's1', file_id: 'remote-1' });
|
||||
const { service } = buildService({
|
||||
getStrategyFunctions: (source: string) =>
|
||||
source === 'execute_code'
|
||||
? { handleFileUpload: uploadCodeEnvFile }
|
||||
: { getDownloadStream },
|
||||
});
|
||||
|
||||
await expect(
|
||||
service.provisionToCodeEnv({
|
||||
req,
|
||||
file: makeFile(),
|
||||
rateLimitBudget: {
|
||||
limitMs: 2_000,
|
||||
waitedMs: 0,
|
||||
activeWaitEnds: new Set(),
|
||||
},
|
||||
}),
|
||||
).resolves.toMatchObject({ referenceSet: { codeEnvRef: { file_id: 'remote-1' } } });
|
||||
|
||||
expect(uploadCodeEnvFile).toHaveBeenCalledTimes(2);
|
||||
expect(getDownloadStream).toHaveBeenCalledTimes(2);
|
||||
expect(uploadCodeEnvFile.mock.calls.map(([args]) => args.stream)).toEqual(streams);
|
||||
});
|
||||
|
||||
it('renames a converted image to match its stored MIME type', async () => {
|
||||
const { service, uploadCodeEnvFile } = buildService();
|
||||
|
||||
|
|
@ -112,9 +159,14 @@ describe('createProvisionService', () => {
|
|||
it('cancels storage acquisition and destroys a stream returned after cancellation', async () => {
|
||||
const controller = new AbortController();
|
||||
let resolveDownload!: (stream: Readable) => void;
|
||||
let markDownloadStarted!: () => void;
|
||||
const downloadStarted = new Promise<void>((resolve) => {
|
||||
markDownloadStarted = resolve;
|
||||
});
|
||||
const getDownloadStream = jest.fn(
|
||||
() =>
|
||||
new Promise<Readable>((resolve) => {
|
||||
markDownloadStarted();
|
||||
resolveDownload = resolve;
|
||||
}),
|
||||
);
|
||||
|
|
@ -128,6 +180,7 @@ describe('createProvisionService', () => {
|
|||
signal: controller.signal,
|
||||
});
|
||||
|
||||
await downloadStarted;
|
||||
controller.abort();
|
||||
const stream = new Readable({ read() {} });
|
||||
const destroy = jest.spyOn(stream, 'destroy');
|
||||
|
|
|
|||
|
|
@ -14,12 +14,16 @@ import {
|
|||
import type { CodeEnvRef, CodeEnvRefMap, TFile } from 'librechat-data-provider';
|
||||
import type { Readable } from 'node:stream';
|
||||
import type { CodeEnvIdentity } from '~/files/code/identity';
|
||||
import type { CodeApiRateLimitBudget } from '~/utils';
|
||||
import type { ServerRequest } from '~/types';
|
||||
import {
|
||||
logAxiosError,
|
||||
createAxiosInstance,
|
||||
createCodeApiRateLimitBudget,
|
||||
codeServerHttpAgent,
|
||||
codeServerHttpsAgent,
|
||||
getCodeApiUploadOptions,
|
||||
withCodeApiUploadRecovery,
|
||||
} from '~/utils';
|
||||
import { buildCodeEnvIdentityParams } from '~/files/code/identity';
|
||||
import { getCodeApiAuthHeaders } from '~/auth/codeapi';
|
||||
|
|
@ -89,6 +93,7 @@ export interface ProvisionService {
|
|||
route?: CodeExecutionRoute;
|
||||
sandboxFilename?: string;
|
||||
signal?: AbortSignal;
|
||||
rateLimitBudget?: CodeApiRateLimitBudget;
|
||||
}) => Promise<{
|
||||
referenceSet: CodeEnvReferenceSetResult;
|
||||
refUpdate: CodeEnvRefUpdate;
|
||||
|
|
@ -237,6 +242,7 @@ export function createProvisionService({
|
|||
route,
|
||||
sandboxFilename: requestedSandboxFilename,
|
||||
signal,
|
||||
rateLimitBudget,
|
||||
}: {
|
||||
req: ServerRequest;
|
||||
file: TFile;
|
||||
|
|
@ -244,20 +250,13 @@ export function createProvisionService({
|
|||
route?: CodeExecutionRoute;
|
||||
sandboxFilename?: string;
|
||||
signal?: AbortSignal;
|
||||
rateLimitBudget?: CodeApiRateLimitBudget;
|
||||
}) {
|
||||
signal?.throwIfAborted();
|
||||
const { handleFileUpload: uploadCodeEnvFile } = getStrategyFunctions(FileSources.execute_code);
|
||||
if (!uploadCodeEnvFile) {
|
||||
throw new Error('Code environment upload strategy is unavailable');
|
||||
}
|
||||
const stream = await getStorageStream(file, req, signal);
|
||||
signal?.throwIfAborted();
|
||||
if (!stream) {
|
||||
throw new Error(
|
||||
`Cannot provision file "${file.filename}" to code env: storage source "${file.source}" does not support download streams`,
|
||||
);
|
||||
}
|
||||
|
||||
const kind: 'agent' | 'user' = entity_id ? 'agent' : 'user';
|
||||
const id = entity_id ?? (req.user?.id as string);
|
||||
|
||||
|
|
@ -268,15 +267,42 @@ export function createProvisionService({
|
|||
const executionProfile = route?.executionProfile ?? 'default';
|
||||
const sandboxFilename =
|
||||
requestedSandboxFilename ?? resolveSandboxFilename(file.filename, file.type);
|
||||
const uploaded = await uploadCodeEnvFile({
|
||||
const uploadOptions = getCodeApiUploadOptions(
|
||||
req,
|
||||
stream,
|
||||
filename: sandboxFilename,
|
||||
kind,
|
||||
id,
|
||||
...(route?.baseUrl ? { codeApiBaseUrl: route.baseUrl } : {}),
|
||||
executionProfile,
|
||||
route?.executionRouteKey ?? route?.baseUrl ?? executionProfile,
|
||||
);
|
||||
const uploaded = await withCodeApiUploadRecovery({
|
||||
registry: req.app.locals.codeApiUploadRegistry,
|
||||
scope: uploadOptions.scope,
|
||||
concurrency: uploadOptions.concurrency,
|
||||
label: `uploading "${file.filename}" to the code environment`,
|
||||
budget: rateLimitBudget ?? createCodeApiRateLimitBudget(uploadOptions.retryWaitMs),
|
||||
signal,
|
||||
onWait: (waitMs) =>
|
||||
logger.warn(
|
||||
`[provisionToCodeEnv] Rate-limited uploading file ${file.file_id}; retrying in ${waitMs}ms`,
|
||||
),
|
||||
openSource: async () => {
|
||||
const stream = await getStorageStream(file, req, signal);
|
||||
signal?.throwIfAborted();
|
||||
if (!stream) {
|
||||
throw new Error(
|
||||
`Cannot provision file "${file.filename}" to code env: storage source "${file.source}" does not support download streams`,
|
||||
);
|
||||
}
|
||||
return stream;
|
||||
},
|
||||
upload: (stream) =>
|
||||
uploadCodeEnvFile({
|
||||
req,
|
||||
stream,
|
||||
filename: sandboxFilename,
|
||||
kind,
|
||||
id,
|
||||
...(route?.baseUrl ? { codeApiBaseUrl: route.baseUrl } : {}),
|
||||
executionProfile,
|
||||
signal,
|
||||
}),
|
||||
});
|
||||
signal?.throwIfAborted();
|
||||
|
||||
|
|
|
|||
|
|
@ -87,6 +87,19 @@ export const logAxiosError = ({
|
|||
return logMessage;
|
||||
};
|
||||
|
||||
/** Adds upload context without discarding the Axios response metadata used by
|
||||
* Code API recovery. Transport-specific branching remains in this typed module. */
|
||||
export function wrapCodeApiUploadError(error: unknown, message: string): Error {
|
||||
const wrapped = new Error(logAxiosError({ message, error }), { cause: error });
|
||||
if (axios.isAxiosError(error)) {
|
||||
Object.assign(wrapped, {
|
||||
isAxiosError: true,
|
||||
response: error.response,
|
||||
});
|
||||
}
|
||||
return wrapped;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates and configures an Axios instance with optional proxy settings.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,14 @@
|
|||
import { AxiosError, AxiosHeaders } from 'axios';
|
||||
import { getCodeApiRetryAfterMs, withCodeApiRateLimit, createCodeApiRateLimitBudget } from './code';
|
||||
import {
|
||||
getCodeApiRetryAfterMs,
|
||||
getCodeApiUploadOptions,
|
||||
withCodeApiRateLimit,
|
||||
withCodeApiUploadRecovery,
|
||||
withCodeApiUploadSlot,
|
||||
createCodeApiRateLimitBudget,
|
||||
createCodeApiUploadRegistry,
|
||||
} from './code';
|
||||
import { wrapCodeApiUploadError } from './axios';
|
||||
|
||||
/** Builds the error axios raises for a Code API rate-limit response. */
|
||||
function rateLimited({
|
||||
|
|
@ -22,6 +31,18 @@ function rateLimited({
|
|||
return error;
|
||||
}
|
||||
|
||||
describe('wrapCodeApiUploadError', () => {
|
||||
it('preserves Axios rate-limit metadata on the contextual error', () => {
|
||||
const cause = rateLimited({ headers: { 'retry-after': '7' } });
|
||||
|
||||
expect(wrapCodeApiUploadError(cause, 'Upload failed')).toMatchObject({
|
||||
cause,
|
||||
isAxiosError: true,
|
||||
response: { status: 429 },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('getCodeApiRetryAfterMs', () => {
|
||||
it('reads the standard Retry-After header', () => {
|
||||
expect(getCodeApiRetryAfterMs(rateLimited({ headers: { 'retry-after': '17' } }))).toBe(17_000);
|
||||
|
|
@ -88,7 +109,7 @@ describe('withCodeApiRateLimit', () => {
|
|||
|
||||
expect(attempts).toBe(2);
|
||||
expect(waits).toEqual([1_000]);
|
||||
expect(budget.remainingMs).toBe(4_000);
|
||||
expect(budget.waitedMs).toBeGreaterThanOrEqual(1_000);
|
||||
});
|
||||
|
||||
it('names the limit when the wait exceeds the remaining budget', async () => {
|
||||
|
|
@ -99,9 +120,15 @@ describe('withCodeApiRateLimit', () => {
|
|||
|
||||
await expect(
|
||||
withCodeApiRateLimit({ attempt, label: 'reading "x.png"', budget }),
|
||||
).rejects.toThrow(/rate limit reached while reading "x\.png" \(retry in 300s\)/);
|
||||
).rejects.toMatchObject({
|
||||
name: 'CodeApiRateLimitError',
|
||||
code: 'CODE_API_RATE_LIMITED',
|
||||
status: 429,
|
||||
statusCode: 429,
|
||||
retryAfterMs: 300_000,
|
||||
});
|
||||
expect(attempt).toHaveBeenCalledTimes(1);
|
||||
expect(budget.remainingMs).toBe(5_000);
|
||||
expect(budget.waitedMs).toBe(0);
|
||||
});
|
||||
|
||||
it('names the limit even when the response carries no usable delay', async () => {
|
||||
|
|
@ -117,10 +144,15 @@ describe('withCodeApiRateLimit', () => {
|
|||
label: 'reading',
|
||||
budget: createCodeApiRateLimitBudget(60_000),
|
||||
}),
|
||||
).rejects.toThrow(/rate limit reached while reading\./);
|
||||
).rejects.toMatchObject({
|
||||
name: 'CodeApiRateLimitError',
|
||||
code: 'CODE_API_RATE_LIMITED',
|
||||
status: 429,
|
||||
statusCode: 429,
|
||||
});
|
||||
});
|
||||
|
||||
it('charges at least a second so a zero delay cannot spin', async () => {
|
||||
it('waits at least a second so a zero delay cannot spin', async () => {
|
||||
const budget = createCodeApiRateLimitBudget(2_500);
|
||||
const waits: number[] = [];
|
||||
let attempts = 0;
|
||||
|
|
@ -136,7 +168,6 @@ describe('withCodeApiRateLimit', () => {
|
|||
withCodeApiRateLimit({ attempt, label: 'reading', budget, onWait: (ms) => waits.push(ms) }),
|
||||
).resolves.toBe('ok');
|
||||
expect(waits).toEqual([1_000, 1_000]);
|
||||
expect(budget.remainingMs).toBe(500);
|
||||
});
|
||||
|
||||
it('rethrows anything that is not a rate limit', async () => {
|
||||
|
|
@ -147,4 +178,250 @@ describe('withCodeApiRateLimit', () => {
|
|||
'Request failed',
|
||||
);
|
||||
});
|
||||
|
||||
it('stops waiting when the calling operation is cancelled', async () => {
|
||||
const controller = new AbortController();
|
||||
const attempt = jest.fn(async () => {
|
||||
throw rateLimited({ headers: { 'retry-after': '10' } });
|
||||
});
|
||||
|
||||
const pending = withCodeApiRateLimit({
|
||||
attempt,
|
||||
label: 'uploading',
|
||||
budget: createCodeApiRateLimitBudget(20_000),
|
||||
signal: controller.signal,
|
||||
onWait: () => controller.abort(),
|
||||
});
|
||||
|
||||
await expect(pending).rejects.toMatchObject({ name: 'AbortError' });
|
||||
expect(attempt).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('shares elapsed wall time across concurrent waits', async () => {
|
||||
const budget = createCodeApiRateLimitBudget(2_500);
|
||||
const attempts = [0, 0, 0];
|
||||
const run = (index: number) =>
|
||||
withCodeApiRateLimit({
|
||||
budget,
|
||||
label: `uploading ${index}`,
|
||||
attempt: async () => {
|
||||
if (attempts[index]++ === 0) {
|
||||
throw rateLimited({ headers: { 'retry-after': '0' } });
|
||||
}
|
||||
return index;
|
||||
},
|
||||
});
|
||||
|
||||
await expect(Promise.all([run(0), run(1), run(2)])).resolves.toEqual([0, 1, 2]);
|
||||
});
|
||||
|
||||
it('does not charge successful request execution against the wait budget', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const budget = createCodeApiRateLimitBudget(1_500);
|
||||
let attempts = 0;
|
||||
const pending = withCodeApiRateLimit({
|
||||
budget,
|
||||
label: 'reading windows',
|
||||
attempt: async () => {
|
||||
attempts++;
|
||||
if (attempts === 1) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 30_000));
|
||||
throw rateLimited({ headers: { 'retry-after': '1' } });
|
||||
}
|
||||
return 'ok';
|
||||
},
|
||||
});
|
||||
|
||||
await jest.advanceTimersByTimeAsync(30_000);
|
||||
await jest.advanceTimersByTimeAsync(1_000);
|
||||
await expect(pending).resolves.toBe('ok');
|
||||
expect(budget.waitedMs).toBe(1_000);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('withCodeApiUploadSlot', () => {
|
||||
it('removes a canceled caller from a busy scope immediately', async () => {
|
||||
const registry = createCodeApiUploadRegistry();
|
||||
let release!: () => void;
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const first = withCodeApiUploadSlot({
|
||||
registry,
|
||||
scope: 'route:user-a',
|
||||
concurrency: 1,
|
||||
task: () => gate,
|
||||
});
|
||||
const controller = new AbortController();
|
||||
const queuedTask = jest.fn(async () => undefined);
|
||||
const queued = withCodeApiUploadSlot({
|
||||
registry,
|
||||
scope: 'route:user-a',
|
||||
concurrency: 1,
|
||||
signal: controller.signal,
|
||||
task: queuedTask,
|
||||
});
|
||||
|
||||
controller.abort();
|
||||
await expect(queued).rejects.toMatchObject({ name: 'AbortError' });
|
||||
expect(queuedTask).not.toHaveBeenCalled();
|
||||
release();
|
||||
await first;
|
||||
});
|
||||
|
||||
it('does not block a different principal and route scope', async () => {
|
||||
const registry = createCodeApiUploadRegistry();
|
||||
let release!: () => void;
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const first = withCodeApiUploadSlot({
|
||||
registry,
|
||||
scope: 'route-a:user-a',
|
||||
concurrency: 1,
|
||||
task: () => gate,
|
||||
});
|
||||
const independent = jest.fn(async () => 'ok');
|
||||
|
||||
await expect(
|
||||
withCodeApiUploadSlot({
|
||||
registry,
|
||||
scope: 'route-b:user-b',
|
||||
concurrency: 1,
|
||||
task: independent,
|
||||
}),
|
||||
).resolves.toBe('ok');
|
||||
expect(independent).toHaveBeenCalledTimes(1);
|
||||
release();
|
||||
await first;
|
||||
});
|
||||
});
|
||||
|
||||
describe('withCodeApiUploadRecovery', () => {
|
||||
it('disposes a failed source before opening the retry source', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const first = { destroy: jest.fn() };
|
||||
const second = { destroy: jest.fn() };
|
||||
const openSource = jest
|
||||
.fn<Promise<typeof first>, []>()
|
||||
.mockResolvedValueOnce(first)
|
||||
.mockResolvedValueOnce(second);
|
||||
const upload = jest
|
||||
.fn<Promise<string>, [typeof first]>()
|
||||
.mockRejectedValueOnce(rateLimited({ headers: { 'retry-after': '0' } }))
|
||||
.mockResolvedValueOnce('ok');
|
||||
|
||||
const pending = withCodeApiUploadRecovery({
|
||||
registry: createCodeApiUploadRegistry(),
|
||||
scope: 'route:user',
|
||||
budget: createCodeApiRateLimitBudget(2_000),
|
||||
label: 'uploading',
|
||||
openSource,
|
||||
upload,
|
||||
});
|
||||
await jest.advanceTimersByTimeAsync(1_000);
|
||||
|
||||
await expect(pending).resolves.toBe('ok');
|
||||
expect(first.destroy).toHaveBeenCalledTimes(1);
|
||||
expect(second.destroy).not.toHaveBeenCalled();
|
||||
expect(upload.mock.calls.map(([source]) => source)).toEqual([first, second]);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('disposes every stream in a failed batch source', async () => {
|
||||
const first = { stream: { destroy: jest.fn() } };
|
||||
const second = { stream: { destroy: jest.fn() } };
|
||||
|
||||
await expect(
|
||||
withCodeApiUploadRecovery({
|
||||
registry: createCodeApiUploadRegistry(),
|
||||
scope: 'route:user',
|
||||
budget: createCodeApiRateLimitBudget(0),
|
||||
label: 'uploading',
|
||||
openSource: async () => [first, second],
|
||||
upload: async () => {
|
||||
throw rateLimited({ headers: { 'retry-after': '30' } });
|
||||
},
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 'CODE_API_RATE_LIMITED' });
|
||||
expect(first.stream.destroy).toHaveBeenCalledTimes(1);
|
||||
expect(second.stream.destroy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not classify a backing-storage 429 as Code API throttling', async () => {
|
||||
const storageRateLimit = rateLimited({ headers: { 'retry-after': '1' } });
|
||||
const upload = jest.fn();
|
||||
|
||||
await expect(
|
||||
withCodeApiUploadRecovery({
|
||||
registry: createCodeApiUploadRegistry(),
|
||||
scope: 'route:user',
|
||||
budget: createCodeApiRateLimitBudget(2_000),
|
||||
label: 'uploading',
|
||||
openSource: async () => {
|
||||
throw storageRateLimit;
|
||||
},
|
||||
upload,
|
||||
}),
|
||||
).rejects.toBe(storageRateLimit);
|
||||
expect(upload).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('disposes a source acquired as cancellation arrives', async () => {
|
||||
const controller = new AbortController();
|
||||
const source = { destroy: jest.fn() };
|
||||
const upload = jest.fn();
|
||||
|
||||
await expect(
|
||||
withCodeApiUploadRecovery({
|
||||
registry: createCodeApiUploadRegistry(),
|
||||
scope: 'route:user',
|
||||
signal: controller.signal,
|
||||
label: 'uploading',
|
||||
openSource: async () => {
|
||||
controller.abort();
|
||||
return source;
|
||||
},
|
||||
upload,
|
||||
}),
|
||||
).rejects.toMatchObject({ name: 'AbortError' });
|
||||
expect(source.destroy).toHaveBeenCalledTimes(1);
|
||||
expect(upload).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getCodeApiUploadOptions', () => {
|
||||
it('uses the configured limit and isolates tenants, principals, and routes', () => {
|
||||
const first = getCodeApiUploadOptions(
|
||||
{
|
||||
user: { id: 'user-a', tenantId: 'tenant-a' },
|
||||
config: { endpoints: { agents: { codeApiUploadConcurrency: 7 } } },
|
||||
} as never,
|
||||
'route-a',
|
||||
);
|
||||
const second = getCodeApiUploadOptions(
|
||||
{ user: { id: 'user-a', tenantId: 'tenant-b' } } as never,
|
||||
'route-a',
|
||||
);
|
||||
|
||||
expect(first.concurrency).toBe(7);
|
||||
expect(second.concurrency).toBe(3);
|
||||
expect(first.retryWaitMs).toBe(20_000);
|
||||
expect(first.scope).not.toBe(second.scope);
|
||||
expect(first.scope).not.toBe(
|
||||
getCodeApiUploadOptions({ user: { id: 'user-b', tenantId: 'tenant-a' } } as never, 'route-a')
|
||||
.scope,
|
||||
);
|
||||
expect(first.scope).not.toBe(
|
||||
getCodeApiUploadOptions({ user: { id: 'user-a', tenantId: 'tenant-a' } } as never, 'route-b')
|
||||
.scope,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import http from 'http';
|
||||
import https from 'https';
|
||||
import { isAxiosError } from 'axios';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import type { ServerRequest } from '~/types';
|
||||
|
||||
/**
|
||||
* Dedicated agents for code-server requests, preventing socket pool contamination.
|
||||
|
|
@ -41,18 +43,223 @@ export function getCodeApiRetryAfterMs(error: unknown): number | null {
|
|||
}
|
||||
|
||||
/** Total time one operation may spend waiting out Code API rate limits.
|
||||
* Sized to cover a single limiter window without stalling a chat turn. */
|
||||
* Keeps recovery bounded inside a live chat turn. */
|
||||
export const MAX_CODE_API_RATE_LIMIT_WAIT_MS = 20_000;
|
||||
export const CODE_API_UPLOAD_CONCURRENCY_DEFAULT = 3;
|
||||
|
||||
/** Remaining wait allowance, shared by every request of one operation. */
|
||||
export interface CodeApiUploadRegistry {
|
||||
scopes: Map<string, UploadScopeState>;
|
||||
}
|
||||
|
||||
export function createCodeApiUploadRegistry(): CodeApiUploadRegistry {
|
||||
return { scopes: new Map() };
|
||||
}
|
||||
|
||||
export function createCodeApiUploadScope(params: {
|
||||
route: string;
|
||||
principalId: string;
|
||||
tenantId?: string;
|
||||
}): string {
|
||||
return JSON.stringify([params.route, params.tenantId ?? 'legacy', params.principalId]);
|
||||
}
|
||||
|
||||
export function getCodeApiUploadOptions(
|
||||
req: ServerRequest,
|
||||
route: string,
|
||||
): { scope: string; concurrency: number; retryWaitMs: number } {
|
||||
const user = req.user as typeof req.user & {
|
||||
_id?: { toString(): string } | string;
|
||||
tenantId?: unknown;
|
||||
orgId?: unknown;
|
||||
};
|
||||
const principalId = user?.id ?? user?._id?.toString();
|
||||
if (!principalId) {
|
||||
throw new Error('Code API upload recovery requires an authenticated principal');
|
||||
}
|
||||
return {
|
||||
scope: createCodeApiUploadScope({
|
||||
route,
|
||||
principalId,
|
||||
tenantId:
|
||||
user.tenantId == null && user.orgId == null
|
||||
? undefined
|
||||
: String(user.tenantId ?? user.orgId),
|
||||
}),
|
||||
concurrency:
|
||||
req.config?.endpoints?.agents?.codeApiUploadConcurrency ??
|
||||
CODE_API_UPLOAD_CONCURRENCY_DEFAULT,
|
||||
retryWaitMs:
|
||||
req.config?.endpoints?.agents?.codeApiMaxRetryWaitMs ?? MAX_CODE_API_RATE_LIMIT_WAIT_MS,
|
||||
};
|
||||
}
|
||||
|
||||
interface QueuedUpload {
|
||||
run: () => void;
|
||||
reject: (error: unknown) => void;
|
||||
signal?: AbortSignal;
|
||||
onAbort?: () => void;
|
||||
}
|
||||
|
||||
interface UploadScopeState {
|
||||
active: number;
|
||||
concurrency: number;
|
||||
queue: QueuedUpload[];
|
||||
}
|
||||
|
||||
interface CodeApiRateLimitWaitToken {
|
||||
endAt: number;
|
||||
}
|
||||
|
||||
function abortReason(signal: AbortSignal): unknown {
|
||||
try {
|
||||
signal.throwIfAborted();
|
||||
} catch (error) {
|
||||
return error;
|
||||
}
|
||||
return new Error('The operation was aborted');
|
||||
}
|
||||
|
||||
function drainCodeApiUploadScope(
|
||||
registry: CodeApiUploadRegistry,
|
||||
scope: string,
|
||||
state: UploadScopeState,
|
||||
): void {
|
||||
while (state.active < state.concurrency && state.queue.length > 0) {
|
||||
const queued = state.queue.shift();
|
||||
if (!queued) {
|
||||
break;
|
||||
}
|
||||
queued.signal?.removeEventListener('abort', queued.onAbort!);
|
||||
if (queued.signal?.aborted) {
|
||||
queued.reject(abortReason(queued.signal));
|
||||
continue;
|
||||
}
|
||||
queued.run();
|
||||
}
|
||||
if (state.active === 0 && state.queue.length === 0) {
|
||||
registry.scopes.delete(scope);
|
||||
}
|
||||
}
|
||||
|
||||
/** Limits uploads within one Code API route and principal. A throttled bucket
|
||||
* holds only its own slots, and canceled callers leave the queue immediately. */
|
||||
export function withCodeApiUploadSlot<T>(params: {
|
||||
registry: CodeApiUploadRegistry;
|
||||
task: () => Promise<T>;
|
||||
scope: string;
|
||||
concurrency?: number;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<T> {
|
||||
const {
|
||||
registry,
|
||||
task,
|
||||
scope,
|
||||
concurrency = CODE_API_UPLOAD_CONCURRENCY_DEFAULT,
|
||||
signal,
|
||||
} = params;
|
||||
if (!Number.isInteger(concurrency) || concurrency < 1) {
|
||||
throw new Error(`Code API upload concurrency must be a positive integer (got ${concurrency})`);
|
||||
}
|
||||
signal?.throwIfAborted();
|
||||
let state = registry.scopes.get(scope);
|
||||
if (!state) {
|
||||
state = { active: 0, concurrency, queue: [] };
|
||||
registry.scopes.set(scope, state);
|
||||
} else {
|
||||
state.concurrency = concurrency;
|
||||
}
|
||||
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const run = (): void => {
|
||||
state.active++;
|
||||
Promise.resolve()
|
||||
.then(task)
|
||||
.then(resolve, reject)
|
||||
.finally(() => {
|
||||
state.active--;
|
||||
drainCodeApiUploadScope(registry, scope, state);
|
||||
});
|
||||
};
|
||||
if (state.active < state.concurrency) {
|
||||
run();
|
||||
return;
|
||||
}
|
||||
const queued: QueuedUpload = { run, reject, signal };
|
||||
queued.onAbort = () => {
|
||||
const index = state.queue.indexOf(queued);
|
||||
if (index >= 0) {
|
||||
state.queue.splice(index, 1);
|
||||
reject(abortReason(signal!));
|
||||
drainCodeApiUploadScope(registry, scope, state);
|
||||
}
|
||||
};
|
||||
signal?.addEventListener('abort', queued.onAbort, { once: true });
|
||||
state.queue.push(queued);
|
||||
});
|
||||
}
|
||||
|
||||
/** Actual rate-limit wait time shared by every request of one operation. */
|
||||
export interface CodeApiRateLimitBudget {
|
||||
remainingMs: number;
|
||||
limitMs: number;
|
||||
waitedMs: number;
|
||||
waitingSince?: number;
|
||||
activeWaitEnds: Set<CodeApiRateLimitWaitToken>;
|
||||
}
|
||||
|
||||
export function createCodeApiRateLimitBudget(
|
||||
totalMs: number = MAX_CODE_API_RATE_LIMIT_WAIT_MS,
|
||||
): CodeApiRateLimitBudget {
|
||||
return { remainingMs: totalMs };
|
||||
return { limitMs: totalMs, waitedMs: 0, activeWaitEnds: new Set() };
|
||||
}
|
||||
|
||||
function beginCodeApiRateLimitWait(
|
||||
budget: CodeApiRateLimitBudget,
|
||||
waitMs: number,
|
||||
): CodeApiRateLimitWaitToken | null {
|
||||
const now = Date.now();
|
||||
const activeElapsed = budget.waitingSince == null ? 0 : now - budget.waitingSince;
|
||||
const currentWaitEnd = Math.max(
|
||||
now,
|
||||
...Array.from(budget.activeWaitEnds, (token) => token.endAt),
|
||||
);
|
||||
const projectedWaitedMs =
|
||||
budget.waitedMs + activeElapsed + Math.max(currentWaitEnd, now + waitMs) - now;
|
||||
if (projectedWaitedMs > budget.limitMs) {
|
||||
return null;
|
||||
}
|
||||
const token = { endAt: now + waitMs };
|
||||
if (budget.activeWaitEnds.size === 0) {
|
||||
budget.waitingSince = now;
|
||||
}
|
||||
budget.activeWaitEnds.add(token);
|
||||
return token;
|
||||
}
|
||||
|
||||
function finishCodeApiRateLimitWait(
|
||||
budget: CodeApiRateLimitBudget,
|
||||
token: CodeApiRateLimitWaitToken,
|
||||
): void {
|
||||
if (!budget.activeWaitEnds.delete(token)) {
|
||||
return;
|
||||
}
|
||||
if (budget.activeWaitEnds.size === 0 && budget.waitingSince != null) {
|
||||
budget.waitedMs += Date.now() - budget.waitingSince;
|
||||
budget.waitingSince = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function codeApiRateLimitError(label: string, cause: unknown, retryAfterMs?: number): Error {
|
||||
const retrySuffix = retryAfterMs == null ? '' : ` (retry in ${Math.ceil(retryAfterMs / 1000)}s)`;
|
||||
return Object.assign(
|
||||
new Error(`Code API rate limit reached while ${label}${retrySuffix}.`, { cause }),
|
||||
{
|
||||
name: 'CodeApiRateLimitError',
|
||||
code: 'CODE_API_RATE_LIMITED',
|
||||
status: 429,
|
||||
statusCode: 429,
|
||||
...(retryAfterMs == null ? {} : { retryAfterMs }),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -69,30 +276,134 @@ export async function withCodeApiRateLimit<T>(params: {
|
|||
label: string;
|
||||
budget?: CodeApiRateLimitBudget;
|
||||
onWait?: (waitMs: number) => void;
|
||||
signal?: AbortSignal;
|
||||
/** Narrows retry classification when an attempt also performs non-CodeAPI work. */
|
||||
isRateLimit?: (error: unknown) => boolean;
|
||||
}): Promise<T> {
|
||||
const { attempt, label, budget, onWait } = params;
|
||||
const {
|
||||
attempt,
|
||||
label,
|
||||
budget,
|
||||
onWait,
|
||||
signal,
|
||||
isRateLimit = (error) => isAxiosError(error) && error.response?.status === 429,
|
||||
} = params;
|
||||
for (;;) {
|
||||
signal?.throwIfAborted();
|
||||
try {
|
||||
return await attempt();
|
||||
} catch (error) {
|
||||
if (!isAxiosError(error) || error.response?.status !== 429) {
|
||||
if (!isRateLimit(error)) {
|
||||
throw error;
|
||||
}
|
||||
const retryAfterMs = getCodeApiRetryAfterMs(error);
|
||||
if (retryAfterMs == null) {
|
||||
throw new Error(`Code API rate limit reached while ${label}.`);
|
||||
throw codeApiRateLimitError(label, error);
|
||||
}
|
||||
/* A zero delay would spin; charge at least a second so the budget
|
||||
* always drains and the loop terminates. */
|
||||
/* A zero delay would spin. Wait at least a second; the actual-wait
|
||||
* budget still guarantees that repeated zero hints terminate. */
|
||||
const waitMs = Math.max(retryAfterMs, 1000);
|
||||
if (budget == null || waitMs > budget.remainingMs) {
|
||||
throw new Error(
|
||||
`Code API rate limit reached while ${label} (retry in ${Math.ceil(waitMs / 1000)}s).`,
|
||||
);
|
||||
if (budget == null) {
|
||||
throw codeApiRateLimitError(label, error, waitMs);
|
||||
}
|
||||
const rateLimitBudget = budget;
|
||||
const waitToken = beginCodeApiRateLimitWait(rateLimitBudget, waitMs);
|
||||
if (waitToken == null) {
|
||||
throw codeApiRateLimitError(label, error, waitMs);
|
||||
}
|
||||
budget.remainingMs -= waitMs;
|
||||
onWait?.(waitMs);
|
||||
await new Promise((resolve) => setTimeout(resolve, waitMs));
|
||||
try {
|
||||
await delay(waitMs, undefined, signal ? { signal } : undefined);
|
||||
} finally {
|
||||
finishCodeApiRateLimitWait(rateLimitBudget, waitToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Owns upload admission, retry waiting, and source reopening for every Code API
|
||||
* upload producer. Callers supply storage and transport adapters only. */
|
||||
export function withCodeApiUploadRecovery<S, T>(params: {
|
||||
registry: CodeApiUploadRegistry;
|
||||
scope: string;
|
||||
concurrency?: number;
|
||||
budget?: CodeApiRateLimitBudget;
|
||||
signal?: AbortSignal;
|
||||
label: string;
|
||||
onWait?: (waitMs: number) => void;
|
||||
openSource: () => Promise<S>;
|
||||
upload: (source: S) => Promise<T>;
|
||||
}): Promise<T> {
|
||||
const { registry, scope, concurrency, budget, signal, label, onWait, openSource, upload } =
|
||||
params;
|
||||
return withCodeApiUploadSlot({
|
||||
registry,
|
||||
scope,
|
||||
concurrency,
|
||||
signal,
|
||||
task: () => {
|
||||
let uploadStarted = false;
|
||||
return withCodeApiRateLimit({
|
||||
budget,
|
||||
signal,
|
||||
label,
|
||||
onWait,
|
||||
/* Storage backends can also answer 429. Only the CodeAPI transport owns this
|
||||
* retry policy and its operator-facing error identity. */
|
||||
isRateLimit: (error) =>
|
||||
uploadStarted && isAxiosError(error) && error.response?.status === 429,
|
||||
attempt: async () => {
|
||||
uploadStarted = false;
|
||||
let source: S | undefined;
|
||||
try {
|
||||
source = await openSource();
|
||||
signal?.throwIfAborted();
|
||||
uploadStarted = true;
|
||||
return await upload(source);
|
||||
} catch (error) {
|
||||
if (source !== undefined) {
|
||||
disposeCodeApiUploadSource(source);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/** Best-effort disposal for the stream shapes accepted by upload adapters: a
|
||||
* single readable or a batch of `{ stream }` entries. */
|
||||
function disposeCodeApiUploadSource(source: unknown): void {
|
||||
try {
|
||||
if (Array.isArray(source)) {
|
||||
source.forEach(disposeCodeApiUploadSource);
|
||||
return;
|
||||
}
|
||||
if (source == null || typeof source !== 'object') {
|
||||
return;
|
||||
}
|
||||
if ('stream' in source) {
|
||||
disposeCodeApiUploadSource((source as { stream?: unknown }).stream);
|
||||
return;
|
||||
}
|
||||
const destroy = (source as { destroy?: unknown }).destroy;
|
||||
if (typeof destroy === 'function') {
|
||||
destroy.call(source);
|
||||
}
|
||||
} catch {
|
||||
/* Cleanup must not replace the upload or cancellation error being propagated. */
|
||||
}
|
||||
}
|
||||
|
||||
export function isCodeApiRateLimitError(error: unknown): error is Error & {
|
||||
code: 'CODE_API_RATE_LIMITED';
|
||||
status: 429;
|
||||
} {
|
||||
return (
|
||||
error != null &&
|
||||
typeof error === 'object' &&
|
||||
(error as { code?: unknown }).code === 'CODE_API_RATE_LIMITED' &&
|
||||
(error as { status?: unknown }).status === 429
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -68,6 +68,18 @@ describe('isAbortError', () => {
|
|||
expect(isAbortError(wrapped)).toBe(true);
|
||||
});
|
||||
|
||||
it('recognizes an Axios cancellation through an upload wrapper cause', () => {
|
||||
const axiosCancellation = Object.assign(new Error('canceled'), {
|
||||
name: 'CanceledError',
|
||||
code: 'ERR_CANCELED',
|
||||
});
|
||||
const wrapped = new Error('Error uploading code environment file', {
|
||||
cause: axiosCancellation,
|
||||
});
|
||||
|
||||
expect(isAbortError(wrapped)).toBe(true);
|
||||
});
|
||||
|
||||
it('recognizes the SDK message shape that only stringifies the reason', () => {
|
||||
expect(
|
||||
isAbortError(new Error('MCP error -32001: AbortError: This operation was aborted')),
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ export function isAbortError(error: unknown): boolean {
|
|||
if (
|
||||
name === 'AbortError' ||
|
||||
code === 'ABORT_ERR' ||
|
||||
code === 'ERR_CANCELED' ||
|
||||
message.includes('AbortError') ||
|
||||
/(?:operation|request|stream) was aborted/i.test(message)
|
||||
) {
|
||||
|
|
|
|||
|
|
@ -441,6 +441,21 @@ describe('endpointSchema addParams validation', () => {
|
|||
});
|
||||
|
||||
describe('agentsEndpointSchema', () => {
|
||||
it('defaults and bounds Code API upload recovery controls', () => {
|
||||
expect(agentsEndpointSchema.parse({}).codeApiUploadConcurrency).toBe(3);
|
||||
expect(
|
||||
agentsEndpointSchema.parse({ codeApiUploadConcurrency: 8 }).codeApiUploadConcurrency,
|
||||
).toBe(8);
|
||||
expect(agentsEndpointSchema.safeParse({ codeApiUploadConcurrency: 0 }).success).toBe(false);
|
||||
expect(agentsEndpointSchema.safeParse({ codeApiUploadConcurrency: 101 }).success).toBe(false);
|
||||
expect(agentsEndpointSchema.parse({}).codeApiMaxRetryWaitMs).toBe(20_000);
|
||||
expect(
|
||||
agentsEndpointSchema.parse({ codeApiMaxRetryWaitMs: 60_000 }).codeApiMaxRetryWaitMs,
|
||||
).toBe(60_000);
|
||||
expect(agentsEndpointSchema.safeParse({ codeApiMaxRetryWaitMs: -1 }).success).toBe(false);
|
||||
expect(agentsEndpointSchema.safeParse({ codeApiMaxRetryWaitMs: 300_001 }).success).toBe(false);
|
||||
});
|
||||
|
||||
it('accepts a non-empty stateful code environment allowlist', () => {
|
||||
const result = agentsEndpointSchema.safeParse({
|
||||
statefulCodeSessions: { allowedEnvironments: ['user', 'agent-user'] },
|
||||
|
|
|
|||
|
|
@ -1201,6 +1201,10 @@ export const agentsEndpointSchema = baseEndpointSchema
|
|||
.max(MAX_SUBAGENTS_CEILING)
|
||||
.optional()
|
||||
.default(MAX_SUBAGENTS),
|
||||
/** Maximum concurrent Code API uploads per route and authenticated principal. */
|
||||
codeApiUploadConcurrency: z.number().int().min(1).max(100).optional().default(3),
|
||||
/** Maximum wall-clock time spent waiting on Code API rate limits per operation. */
|
||||
codeApiMaxRetryWaitMs: z.number().int().min(0).max(300_000).optional().default(20_000),
|
||||
allowedProviders: z.array(z.union([z.string(), eModelEndpointSchema])).optional(),
|
||||
capabilities: z
|
||||
.array(z.nativeEnum(AgentCapabilities))
|
||||
|
|
|
|||
|
|
@ -632,6 +632,10 @@ export type TConfig = {
|
|||
};
|
||||
/** Effective subagents-per-agent cap served from `endpoints.agents.maxSubagents`. */
|
||||
maxSubagents?: number;
|
||||
/** Concurrent Code API uploads allowed per route and authenticated principal. */
|
||||
codeApiUploadConcurrency?: number;
|
||||
/** Milliseconds one operation may spend waiting on Code API rate limits. */
|
||||
codeApiMaxRetryWaitMs?: number;
|
||||
customParams?: {
|
||||
defaultParamsEndpoint?: string;
|
||||
reasoningFormat?: ReasoningParameterFormat;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue