CERBERUS/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2026-08-27 04:07:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

🛟 fix: Stop Agents When Code Resources Cannot Recover (#14651)

Browse source 
* fix: Block Agents When Code Resources Cannot Recover

* fix: Preserve Resource Recovery Failures Across Agent Paths

* fix: Centralize Fatal Agent Initialization

* chore: sort agent imports
This commit is contained in:
Danny Avila 2026-08-07 00:33:28 -04:00 • committed by GitHub
parent 6d2f29266c
commit 5ff46d8c67
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
24 changed files with 629 additions and 65 deletions
Download patch file Download diff file Expand all files Collapse all files

36
api/server/controllers/agents/__tests__/openai.spec.js
View file

@ -3,7 +3,7 @@
* Tests that recordCollectedUsage is called correctly for token spending
*/
const { ResourceType } = require('librechat-data-provider');
const { ErrorTypes, ResourceType } = require('librechat-data-provider');
const mockProcessStream = jest.fn().mockResolvedValue(undefined);
const mockSpendTokens = jest.fn().mockResolvedValue({});
@ -196,8 +196,8 @@ jest.mock('~/cache', () => ({
jest.mock('~/server/services/ToolService', () => ({
loadAgentTools: jest.fn().mockResolvedValue([]),
loadToolsForExecution: jest.fn().mockResolvedValue([]),
isExpectedMCPToolsUnavailableError: (error) =>
error?.code === 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE',
isFatalAgentInitializationError: (error) =>
['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 'resource_recovery_required'].includes(error?.code),
}));
const mockGetMultiplier = jest.fn().mockReturnValue(1);
@ -420,6 +420,36 @@ describe('OpenAIChatCompletionController', () => {
expect(res.status).toHaveBeenCalledWith(503);
});
it('returns the resource recovery status and code before model invocation', async () => {
const { createErrorResponse, initializeAgent } = require('@librechat/api');
const { loadAgentTools } = require('~/server/services/ToolService');
const toolError = Object.assign(new Error('resource recovery required'), {
code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
status: 409,
statusCode: 409,
});
loadAgentTools.mockRejectedValueOnce(toolError);
initializeAgent.mockImplementationOnce(async ({ req, res, loadTools, agent }) => {
await loadTools({
req,
res,
tools: ['execute_code'],
model: agent.model,
agentId: agent.id,
provider: agent.provider,
});
});
await OpenAIChatCompletionController(req, res);
expect(res.status).toHaveBeenCalledWith(409);
expect(createErrorResponse).toHaveBeenCalledWith(
'resource recovery required',
'invalid_request_error',
ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
);
});
});
describe('execution envelope', () => {

68
api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
View file

@ -211,6 +211,7 @@ jest.mock('~/models', () => ({
}));
const AgentController = require('../request');
const { ErrorTypes } = require('librechat-data-provider');
const { disposeClient: mockDisposeClient } = require('~/server/cleanup');
const { getMCPRequestContext } = require('~/server/services/MCPRequestContext');
@ -1749,6 +1750,73 @@ describe('ResumableAgentController resume metadata', () => {
expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
});
it('returns a typed recovery conflict before acknowledging generation startup', async () => {
const recoveryError = new Error('Attached resources could not be restored');
recoveryError.code = ErrorTypes.RESOURCE_RECOVERY_REQUIRED;
mockGenerationJobManager.createJob.mockRejectedValue(recoveryError);
const req = {
user: { id: 'user-123' },
body: {
text: 'Describe the attached image.',
messageId: 'user-msg',
clientRequestId: 'req-abc',
conversationId: 'conversation-123',
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
},
config: {},
};
const res = createResumableResponse();
await AgentController(req, res, jest.fn(), jest.fn(), null);
expect(res.status).toHaveBeenCalledWith(409);
expect(res.json).toHaveBeenCalledWith({
status: 409,
code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
error: 'Attached resources could not be restored',
generationProtocolVersion: 1,
});
expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
});
it('preserves the recovery code in the durable error after acknowledging startup', async () => {
const recoveryError = new Error('Attached resources could not be restored');
recoveryError.code = ErrorTypes.RESOURCE_RECOVERY_REQUIRED;
const initializeClient = jest.fn().mockRejectedValue(recoveryError);
const req = {
user: { id: 'user-123' },
body: {
text: 'Describe the attached image.',
messageId: 'user-msg',
clientRequestId: 'req-abc',
conversationId: 'conversation-123',
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
},
config: {},
};
const res = createResumableResponse();
await AgentController(req, res, jest.fn(), initializeClient, null);
expect(res.status).toHaveBeenCalledWith(200);
expect(res.json).toHaveBeenCalledWith({
streamId: 'conversation-123',
conversationId: 'conversation-123',
generationCreatedAt: 1000,
status: 'started',
generationProtocolVersion: 1,
});
expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
'conversation-123',
JSON.stringify({
status: 409,
code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
error: 'Attached resources could not be restored',
}),
1000,
);
});
it('returns a recovery conflict when the atomic store rejects changed source content', async () => {
const mismatch = new Error('recovery mismatch');
mismatch.code = 'RECOVERY_PAYLOAD_MISMATCH';

38
api/server/controllers/agents/__tests__/responses.unit.spec.js
View file

@ -3,7 +3,7 @@
* Tests that recordCollectedUsage is called correctly for token spending
*/
const { ResourceType } = require('librechat-data-provider');
const { ErrorTypes, ResourceType } = require('librechat-data-provider');
const mockSpendTokens = jest.fn().mockResolvedValue({});
const mockSpendStructuredTokens = jest.fn().mockResolvedValue({});
@ -211,8 +211,8 @@ jest.mock('@librechat/api', () => ({
jest.mock('~/server/services/ToolService', () => ({
loadAgentTools: jest.fn().mockResolvedValue([]),
loadToolsForExecution: jest.fn().mockResolvedValue([]),
isExpectedMCPToolsUnavailableError: (error) =>
error?.code === 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE',
isFatalAgentInitializationError: (error) =>
['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 'resource_recovery_required'].includes(error?.code),
}));
const mockGetMultiplier = jest.fn().mockReturnValue(1);
@ -372,6 +372,38 @@ describe('createResponse controller', () => {
503,
'Expected MCP tools are unavailable',
'server_error',
'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE',
);
});
it('returns the resource recovery status and code before model invocation', async () => {
const { initializeAgent, sendResponsesErrorResponse } = require('@librechat/api');
const { loadAgentTools } = require('~/server/services/ToolService');
const toolError = Object.assign(new Error('resource recovery required'), {
code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
status: 409,
statusCode: 409,
});
loadAgentTools.mockRejectedValueOnce(toolError);
initializeAgent.mockImplementationOnce(async ({ req, res, loadTools, agent }) => {
await loadTools({
req,
res,
tools: ['execute_code'],
model: agent.model,
agentId: agent.id,
provider: agent.provider,
});
});
await createResponse(req, res);
expect(sendResponsesErrorResponse).toHaveBeenCalledWith(
res,
409,
'resource recovery required',
'invalid_request',
ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
);
});

9
api/server/controllers/agents/openai.js
View file

@ -49,7 +49,7 @@ const {
loadAgentTools,
loadToolsForExecution,
getAccessibleMcpServerNames,
isExpectedMCPToolsUnavailableError,
isFatalAgentInitializationError,
} = require('~/server/services/ToolService');
const {
findAccessibleResources,
@ -103,10 +103,10 @@ function createToolLoader(signal, definitionsOnly = true) {
streamId: null, // No resumable stream for OpenAI compat
});
} catch (error) {
logger.error('Error loading tools for agent ' + agentId, error);
if (isExpectedMCPToolsUnavailableError(error)) {
if (isFatalAgentInitializationError(error)) {
throw error;
}
logger.error('Error loading tools for agent ' + agentId, error);
}
};
}
@ -891,7 +891,8 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
: 500;
const errorType =
statusCode >= 400 && statusCode < 500 ? 'invalid_request_error' : 'server_error';
sendErrorResponse(res, statusCode, errorMessage, errorType);
const errorCode = typeof error?.code === 'string' ? error.code : null;
sendErrorResponse(res, statusCode, errorMessage, errorType, errorCode);
}
}
};

25
api/server/controllers/agents/request.js
View file

@ -3,6 +3,7 @@ const { v5: uuidv5 } = require('uuid');
const {
Constants,
EModelEndpoint,
ErrorTypes,
ViolationTypes,
isEphemeralAgentId,
} = require('librechat-data-provider');
@ -50,6 +51,18 @@ function sendGenerationJson(res, status, body, generationProtocolVersion) {
return res.status(status).json({ ...body, generationProtocolVersion });
}
function getResourceRecoveryFailure(error) {
if (error?.code !== ErrorTypes.RESOURCE_RECOVERY_REQUIRED) {
return null;
}
return {
status: 409,
code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
error: error.message || 'Attached resources must be restored before retrying.',
};
}
function createCloseHandler(abortController) {
return function (manual) {
if (!manual) {
@ -1837,6 +1850,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
});
} catch (error) {
logger.error('[ResumableAgentController] Initialization error:', error);
const resourceRecoveryFailure = getResourceRecoveryFailure(error);
try {
if (!res.headersSent) {
if (error?.code === 'GENERATION_PREDECESSOR_MISMATCH') {
@ -1877,6 +1891,13 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
},
generationProtocolVersion,
);
} else if (resourceRecoveryFailure) {
sendGenerationJson(
res,
resourceRecoveryFailure.status,
resourceRecoveryFailure,
generationProtocolVersion,
);
} else {
sendGenerationJson(
res,
@ -1905,7 +1926,9 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
// and the concurrency slot leaks — so swallow its error. (A failed completeJob did not
// finalize anything, so releasing afterward can't let it abort a later replacement.)
if (jobCreatedAt != null) {
const initializationError = error.message || 'Failed to start generation';
const initializationError = resourceRecoveryFailure
? JSON.stringify(resourceRecoveryFailure)
: error.message || 'Failed to start generation';
await GenerationJobManager.completeJob(streamId, initializationError, jobCreatedAt).catch(
(completeErr) => {
logger.warn(

13
api/server/controllers/agents/responses.js
View file

@ -60,7 +60,7 @@ const {
const {
loadAgentTools,
loadToolsForExecution,
isExpectedMCPToolsUnavailableError,
isFatalAgentInitializationError,
} = require('~/server/services/ToolService');
const {
findAccessibleResources,
@ -116,10 +116,10 @@ function createToolLoader(signal, definitionsOnly = true) {
streamId: null,
});
} catch (error) {
logger.error('Error loading tools for agent ' + agentId, error);
if (isExpectedMCPToolsUnavailableError(error)) {
if (isFatalAgentInitializationError(error)) {
throw error;
}
logger.error('Error loading tools for agent ' + agentId, error);
}
};
}
@ -1093,7 +1093,12 @@ const executeResponse = async (envelope, { req, res }) => {
? error.status
: 500;
const errorType = statusCode >= 400 && statusCode < 500 ? 'invalid_request' : 'server_error';
sendResponsesErrorResponse(res, statusCode, errorMessage, errorType);
const errorCode = typeof error?.code === 'string' ? error.code : undefined;
if (errorCode === undefined) {
sendResponsesErrorResponse(res, statusCode, errorMessage, errorType);
} else {
sendResponsesErrorResponse(res, statusCode, errorMessage, errorType, errorCode);
}
}
}
};

4
api/server/services/Endpoints/agents/addedConvo.js
View file

@ -11,7 +11,7 @@ const { isEphemeralAgentId } = require('librechat-data-provider');
const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions');
const { getMCPServerTools } = require('~/server/services/Config');
const { getAccessibleMcpServerNames } = require('~/server/services/MCP');
const { isExpectedMCPToolsUnavailableError } = require('~/server/services/ToolService');
const { isFatalAgentInitializationError } = require('~/server/services/ToolService');
const { getSkillDbMethods, canAuthorSkillFiles } = require('./skillDeps');
const db = require('~/models');
@ -227,7 +227,7 @@ const processAddedConvo = async ({
return { userMCPAuthMap };
} catch (err) {
if (isExpectedMCPToolsUnavailableError(err)) {
if (isFatalAgentInitializationError(err)) {
throw err;
}
logger.error('[processAddedConvo] Error processing addedConvo for parallel agent', err);

17
api/server/services/Endpoints/agents/addedConvo.spec.js
View file

@ -42,8 +42,8 @@ jest.mock('~/server/services/MCP', () => ({
}));
jest.mock('~/server/services/ToolService', () => ({
isExpectedMCPToolsUnavailableError: (error) =>
error?.code === 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE',
isFatalAgentInitializationError: (error) =>
['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 'resource_recovery_required'].includes(error?.code),
}));
jest.mock('./skillDeps', () => ({
@ -59,7 +59,7 @@ jest.mock('~/models', () => ({
}));
const { processAddedConvo } = require('./addedConvo');
const { Constants } = require('librechat-data-provider');
const { Constants, ErrorTypes } = require('librechat-data-provider');
const makeReq = () => ({ user: { id: 'u1', role: 'USER' } });
@ -143,10 +143,13 @@ describe('processAddedConvo', () => {
);
});
it('propagates an expected-MCP-tools failure from an added parallel agent', async () => {
const toolError = Object.assign(new Error('Added agent expected MCP tools are unavailable'), {
code: 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE',
statusCode: 503,
it.each([
['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 503],
[ErrorTypes.RESOURCE_RECOVERY_REQUIRED, 409],
])('propagates fatal %s failures from an added parallel agent', async (code, statusCode) => {
const toolError = Object.assign(new Error(`Added agent failed with ${code}`), {
code,
statusCode,
});
mockInitializeAgent.mockRejectedValueOnce(toolError);

8
api/server/services/Endpoints/agents/initialize.js
View file

@ -40,7 +40,7 @@ const {
loadAgentTools,
loadToolsForExecution,
getAccessibleMcpServerNames,
isExpectedMCPToolsUnavailableError,
isFatalAgentInitializationError,
} = require('~/server/services/ToolService');
const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions');
const {
@ -109,10 +109,10 @@ function createToolLoader(signal, streamId = null, definitionsOnly = false, jobC
accessibleMcpServerNames,
});
} catch (error) {
logger.error('Error loading tools for agent ' + agentId, error);
if (isExpectedMCPToolsUnavailableError(error)) {
if (isFatalAgentInitializationError(error)) {
throw error;
}
logger.error('Error loading tools for agent ' + agentId, error);
}
};
}
@ -804,7 +804,7 @@ const initializeClient = async ({
agentToolContexts.set(agentId, buildAgentToolContext({ agent, config }));
return config;
} catch (err) {
if (isExpectedMCPToolsUnavailableError(err)) {
if (isFatalAgentInitializationError(err)) {
throw err;
}
logger.error(`[processAgent] Error processing subagent ${agentId}:`, err);

39
api/server/services/Endpoints/agents/initialize.spec.js
View file

@ -7,6 +7,7 @@ const {
MAX_SUBAGENT_DEPTH,
MAX_SUBAGENT_GRAPH_NODES,
Constants,
ErrorTypes,
} = require('librechat-data-provider');
const { MongoMemoryServer } = require('mongodb-memory-server');
@ -58,8 +59,8 @@ const mockLoadToolsForExecution = jest.fn();
jest.mock('~/server/services/ToolService', () => ({
loadAgentTools: jest.fn(),
loadToolsForExecution: (...args) => mockLoadToolsForExecution(...args),
isExpectedMCPToolsUnavailableError: (error) =>
error?.code === 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE',
isFatalAgentInitializationError: (error) =>
['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 'resource_recovery_required'].includes(error?.code),
}));
jest.mock('~/server/controllers/ModelController', () => ({
@ -711,6 +712,40 @@ describe('initializeClient — subagent loading', () => {
).rejects.toBe(toolError);
});
it('aborts the run when a pure subagent requires CodeAPI resource recovery', async () => {
const subAgent = await createAgent({
id: SUBAGENT_ID,
name: 'Code Subagent',
provider: 'openai',
model: 'gpt-4',
author: new mongoose.Types.ObjectId(),
tools: ['execute_code'],
});
await grantView(subAgent);
const resourceRecoveryError = Object.assign(new Error('resource recovery required'), {
code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
status: 409,
statusCode: 409,
});
mockInitializeAgent
.mockResolvedValueOnce(
makePrimaryConfig({
subagents: { enabled: true, allowSelf: true, agent_ids: [SUBAGENT_ID] },
}),
)
.mockRejectedValueOnce(resourceRecoveryError);
await expect(
initializeClient({
req: makeSubagentReq(),
res: {},
signal: new AbortController().signal,
endpointOption: makeEndpointOption(),
}),
).rejects.toBe(resourceRecoveryError);
});
it('loads a configured subagent, populates `subagentAgentConfigs`, and keeps it out of `agentConfigs`', async () => {
const subAgent = await createAgent({
id: SUBAGENT_ID,

92
api/server/services/Files/Code/process.js
View file

@ -29,6 +29,7 @@ const {
inferMimeType,
EToolResources,
EModelEndpoint,
ErrorTypes,
mergeFileConfig,
getEndpointFileConfig,
} = require('librechat-data-provider');
@ -784,10 +785,8 @@ async function getSessionInfo(ref, req) {
});
return response.data?.lastModified;
} catch (error) {
logger.debug(
`[getSessionInfo] session lookup failed (treating as cache miss): ${error?.message ?? String(error)}`,
);
} catch (_error) {
logger.debug('[getSessionInfo] session lookup failed (treating as cache miss)');
return null;
}
}
@ -827,6 +826,53 @@ const appendVisibleCodeFileContext = (toolContext, contextLine) => {
return `- Note: The following files are available in the "${Tools.execute_code}" tool environment:${contextLine}`;
};
class CodeResourceRecoveryError extends Error {
constructor({ required, primed, failed }) {
super(JSON.stringify({ type: ErrorTypes.RESOURCE_RECOVERY_REQUIRED }));
this.name = 'CodeResourceRecoveryError';
this.code = ErrorTypes.RESOURCE_RECOVERY_REQUIRED;
this.status = 409;
this.statusCode = 409;
this.details = { required, primed, failed };
this.required = required;
this.primed = primed;
this.failed = failed;
}
}
const getPrimingCorrelation = (req) => ({
requestId: req?.requestId ?? req?.id ?? 'unknown',
runId: req?.body?.messageId ?? req?.body?.conversationId ?? 'unknown',
});
const getReuploadFailureCategory = (error) => {
const status =
error?.response?.status ??
error?.statusCode ??
error?.status ??
error?.$metadata?.httpStatusCode;
const code = error?.code ?? error?.name;
if (
status === 404 ||
code === 'NoSuchKey' ||
code === 'NotFound' ||
code === 'BlobNotFound' ||
code === 'ResourceNotFound'
) {
return 'missing_backing_object';
}
if (
status === 401 ||
status === 403 ||
code === 'AccessDenied' ||
code === 'AccessDeniedException' ||
code === 'Forbidden'
) {
return 'resource_access_denied';
}
return 'reupload_failed';
};
/**
*
* @param {Object} options
@ -888,6 +934,8 @@ const primeFiles = async (options) => {
* paths taken, and the final dispatch summary in one trace. */
let skippedNoRef = 0;
let reuploadFailures = 0;
let requiredCodeFiles = 0;
const reuploadFailureCategories = new Set();
for (let i = 0; i < dbFiles.length; i++) {
const file = dbFiles[i];
@ -898,11 +946,10 @@ const primeFiles = async (options) => {
const ref = file.metadata?.codeEnvRef;
if (!ref) {
skippedNoRef += 1;
logger.debug(
`[primeCodeFiles] file=${file.file_id} path=skip reason=no-codeenvref filename=${file.filename}`,
);
logger.debug(`[primeCodeFiles] file=${file.file_id} path=skip reason=no-codeenvref`);
continue;
}
requiredCodeFiles += 1;
const session_id = ref.storage_session_id;
const id = ref.file_id;
@ -1010,9 +1057,12 @@ const primeFiles = async (options) => {
);
} catch (error) {
reuploadFailures += 1;
const failureCategory = getReuploadFailureCategory(error);
reuploadFailureCategories.add(failureCategory);
const { requestId, runId } = getPrimingCorrelation(req);
logger.error(
`[primeCodeFiles] file=${file.file_id} path=reupload-failed session=${session_id}: ${error.message}`,
error,
`[primeCodeFiles] reupload-failed requestId=${requestId} runId=${runId} ` +
`category=${failureCategory}`,
);
}
};
@ -1043,11 +1093,32 @@ const primeFiles = async (options) => {
/* Dispatch summary — emitted unconditionally so a single grep on
* `[primeCodeFiles] out` always shows the final state, not only
* the per-path trail leading up to it. */
const primedCodeFiles = files.length;
const allRequiredResourcesFailed =
requiredCodeFiles > 0 && primedCodeFiles === 0 && reuploadFailures === requiredCodeFiles;
const { requestId, runId } = getPrimingCorrelation(req);
logger.debug(
`[primeCodeFiles] out: returned=${files.length} ` +
`skippedNoRef=${skippedNoRef} reuploadFailures=${reuploadFailures}`,
`required=${requiredCodeFiles} skippedNoRef=${skippedNoRef} reuploadFailures=${reuploadFailures}`,
);
if (allRequiredResourcesFailed) {
const failureCategory =
reuploadFailureCategories.size === 1
? Array.from(reuploadFailureCategories)[0]
: 'mixed_reupload_failure';
logger.warn(
`[primeCodeFiles] resource-recovery-required requestId=${requestId} runId=${runId} ` +
`required=${requiredCodeFiles} primed=${primedCodeFiles} failed=${reuploadFailures} ` +
`category=${failureCategory}`,
);
throw new CodeResourceRecoveryError({
required: requiredCodeFiles,
primed: primedCodeFiles,
failed: reuploadFailures,
});
}
return { files, toolContext };
};
@ -1457,6 +1528,7 @@ async function writeSandboxFile({
}
module.exports = {
CodeResourceRecoveryError,
primeFiles,
checkIfActive,
getSessionInfo,

71
api/server/services/Files/Code/process.spec.js
View file

@ -29,7 +29,7 @@ jest.mock('librechat-data-provider', () => {
};
});
const { FileContext, ResourceType } = require('librechat-data-provider');
const { ErrorTypes, FileContext, ResourceType } = require('librechat-data-provider');
// Mock uuid
jest.mock('uuid', () => ({
@ -1927,6 +1927,7 @@ describe('Code Process', () => {
agentId: 'agent-123',
resourceType: ResourceType.REMOTE_AGENT,
});
expect(JSON.stringify(logger.debug.mock.calls)).not.toContain(files[0].filename);
});
it('does not read a runtime file record that has no authorized database record', async () => {
@ -2204,6 +2205,74 @@ describe('Code Process', () => {
},
]);
});
it.each([
['Axios/CloudFront 404', { response: { status: 404 } }, 'missing_backing_object'],
[
'AWS SDK NoSuchKey',
{ name: 'NoSuchKey', $metadata: { httpStatusCode: 404 } },
'missing_backing_object',
],
['Azure BlobNotFound', { code: 'BlobNotFound', statusCode: 404 }, 'missing_backing_object'],
['storage access denied', { code: 'AccessDenied', status: 403 }, 'resource_access_denied'],
])(
'fails with a typed recovery error for %s',
async (_errorShape, downloadError, expectedCategory) => {
const dbFile = {
file_id: 'librechat-file-id',
filename: 'cross-region-report.png',
filepath: 'https://storage.us-east.example.test/missing-object',
source: 'local',
context: 'execute_code',
metadata: {
codeEnvRef: {
kind: 'user',
id: 'user-123',
storage_session_id: 'US_EAST_SESSION',
file_id: 'MISSING_OBJECT',
},
},
};
const getDownloadStream = jest.fn().mockRejectedValue(downloadError);
getFiles.mockResolvedValue([dbFile]);
getStrategyFunctions.mockImplementation((source) =>
source === 'execute_code' ? { handleFileUpload: jest.fn() } : { getDownloadStream },
);
mockAxios.mockResolvedValue({ data: null });
await expect(
primeFiles({
req: {
id: 'request-123',
body: { messageId: 'run-123' },
user: { id: 'user-123', role: 'USER' },
},
tool_resources: {
execute_code: { file_ids: ['librechat-file-id'], files: [] },
},
agentId: 'agent-id',
}),
).rejects.toMatchObject({
code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
status: 409,
statusCode: 409,
details: { required: 1, primed: 0, failed: 1 },
required: 1,
primed: 0,
failed: 1,
});
expect(logger.warn).toHaveBeenCalledWith(
expect.stringContaining(
`resource-recovery-required requestId=request-123 runId=run-123 required=1 primed=0 failed=1 category=${expectedCategory}`,
),
);
const failureLogs = logger.error.mock.calls.map(([message]) => message).join('\n');
expect(failureLogs).toContain(`category=${expectedCategory}`);
expect(failureLogs).not.toContain(dbFile.filename);
expect(failureLogs).not.toContain(dbFile.filepath);
},
);
});
describe('primeFiles toolContext for model-visible code files', () => {

13
api/server/services/ToolService.js
View file

@ -33,6 +33,8 @@ const {
buildServerNameAliases,
findShadowedServerNames,
isNormalizationSensitiveName,
AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE,
isFatalAgentInitializationError,
} = require('@librechat/api');
const {
Time,
@ -529,16 +531,15 @@ const isExpectedMCPTool = (toolName) =>
toolName?.includes(Constants.mcp_delimiter) &&
!toolName.startsWith(mcpServerPinPrefix) &&
!isActionTool(toolName);
const expectedMCPToolsUnavailableCode = 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE';
const isExpectedMCPToolsUnavailableError = (error) =>
error?.code === expectedMCPToolsUnavailableCode;
error?.code === AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE;
const createExpectedMCPToolsUnavailableError = (agentName, cause) => {
const subject = agentName ? `Agent "${agentName}"` : 'The agent';
const error = new Error(
`${subject} is configured to use MCP tools, but none are available. Verify that the MCP server is connected and this agent can access its selected tools, then try again.`,
);
error.name = 'AgentToolInitializationError';
error.code = expectedMCPToolsUnavailableCode;
error.code = AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE;
error.status = 503;
error.statusCode = 503;
if (cause != null) {
@ -1165,6 +1166,9 @@ async function loadToolDefinitionsWrapper({
primedCodeFiles = files;
}
} catch (error) {
if (isFatalAgentInitializationError(error)) {
throw error;
}
logger.error('[loadToolDefinitionsWrapper] Error priming code files:', error);
}
}
@ -1269,7 +1273,7 @@ async function loadAgentTools({
accessibleMcpServerNames,
});
} catch (error) {
if (isExpectedMCPToolsUnavailableError(error) || !agent.tools?.some(isExpectedMCPTool)) {
if (isFatalAgentInitializationError(error) || !agent.tools?.some(isExpectedMCPTool)) {
throw error;
}
throw createExpectedMCPToolsUnavailableError(agent.name, error);
@ -2040,6 +2044,7 @@ module.exports = {
loadToolsForExecution,
processRequiredActions,
resolveAgentCapabilities,
isFatalAgentInitializationError,
isExpectedMCPToolsUnavailableError,
/** Re-exported for controllers that already depend on (and mock) this
* module, avoiding a fresh heavy `services/MCP` require chain there. */

28
api/server/services/__tests__/ToolService.spec.js
View file

@ -3,6 +3,7 @@ const {
Tools,
Constants,
ResourceType,
ErrorTypes,
EModelEndpoint,
isActionTool,
actionDelimiter,
@ -25,6 +26,9 @@ const mockLoadToolDefinitions = jest.fn();
const mockGetUserMCPAuthMap = jest.fn();
jest.mock('@librechat/api', () => ({
...jest.requireActual('@librechat/api'),
AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE: 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE',
isFatalAgentInitializationError: (error) =>
['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 'resource_recovery_required'].includes(error?.code),
loadToolDefinitions: (...args) => mockLoadToolDefinitions(...args),
getUserMCPAuthMap: (...args) => mockGetUserMCPAuthMap(...args),
sendEvent: (...args) => mockSendEvent(...args),
@ -266,6 +270,30 @@ describe('ToolService - Action Capability Gating', () => {
expect(primeCodeFiles).toHaveBeenCalledWith(expectedParams);
});
it('propagates a typed CodeAPI resource recovery failure before model invocation', async () => {
const capabilities = [AgentCapabilities.tools, AgentCapabilities.execute_code];
const req = createMockReq(capabilities);
const resourceRecoveryError = Object.assign(new Error('resource recovery required'), {
code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
});
const { primeFiles: primeCodeFiles } = require('~/server/services/Files/Code/process');
primeCodeFiles.mockRejectedValueOnce(resourceRecoveryError);
mockGetEndpointsConfig.mockResolvedValue(createEndpointsConfig(capabilities));
await expect(
loadAgentTools({
req,
res: {},
agent: {
id: 'agent_123',
tools: [Tools.execute_code, 'run_query_mcp_warehouse'],
},
tool_resources: { execute_code: { file_ids: ['stale-file'] } },
definitionsOnly: true,
}),
).rejects.toBe(resourceRecoveryError);
});
it('should exclude action tools from definitions when actions capability is disabled', async () => {
const capabilities = [AgentCapabilities.tools, AgentCapabilities.web_search];
const req = createMockReq(capabilities);