CERBERUS/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2026-08-27 04:07:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

🛟 fix: Stop Agents When Code Resources Cannot Recover (#14651)

Browse source 
* fix: Block Agents When Code Resources Cannot Recover

* fix: Preserve Resource Recovery Failures Across Agent Paths

* fix: Centralize Fatal Agent Initialization

* chore: sort agent imports
This commit is contained in:
Danny Avila 2026-08-07 00:33:28 -04:00 • committed by GitHub
parent 6d2f29266c
commit 5ff46d8c67
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
24 changed files with 629 additions and 65 deletions
Download patch file Download diff file Expand all files Collapse all files

36
api/server/controllers/agents/__tests__/openai.spec.js
View file

@ -3,7 +3,7 @@
* Tests that recordCollectedUsage is called correctly for token spending
*/
const { ResourceType } = require('librechat-data-provider');
const { ErrorTypes, ResourceType } = require('librechat-data-provider');
const mockProcessStream = jest.fn().mockResolvedValue(undefined);
const mockSpendTokens = jest.fn().mockResolvedValue({});
@ -196,8 +196,8 @@ jest.mock('~/cache', () => ({
jest.mock('~/server/services/ToolService', () => ({
loadAgentTools: jest.fn().mockResolvedValue([]),
loadToolsForExecution: jest.fn().mockResolvedValue([]),
isExpectedMCPToolsUnavailableError: (error) =>
error?.code === 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE',
isFatalAgentInitializationError: (error) =>
['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 'resource_recovery_required'].includes(error?.code),
}));
const mockGetMultiplier = jest.fn().mockReturnValue(1);
@ -420,6 +420,36 @@ describe('OpenAIChatCompletionController', () => {
expect(res.status).toHaveBeenCalledWith(503);
});
it('returns the resource recovery status and code before model invocation', async () => {
const { createErrorResponse, initializeAgent } = require('@librechat/api');
const { loadAgentTools } = require('~/server/services/ToolService');
const toolError = Object.assign(new Error('resource recovery required'), {
code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
status: 409,
statusCode: 409,
});
loadAgentTools.mockRejectedValueOnce(toolError);
initializeAgent.mockImplementationOnce(async ({ req, res, loadTools, agent }) => {
await loadTools({
req,
res,
tools: ['execute_code'],
model: agent.model,
agentId: agent.id,
provider: agent.provider,
});
});
await OpenAIChatCompletionController(req, res);
expect(res.status).toHaveBeenCalledWith(409);
expect(createErrorResponse).toHaveBeenCalledWith(
'resource recovery required',
'invalid_request_error',
ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
);
});
});
describe('execution envelope', () => {

68
api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
View file

@ -211,6 +211,7 @@ jest.mock('~/models', () => ({
}));
const AgentController = require('../request');
const { ErrorTypes } = require('librechat-data-provider');
const { disposeClient: mockDisposeClient } = require('~/server/cleanup');
const { getMCPRequestContext } = require('~/server/services/MCPRequestContext');
@ -1749,6 +1750,73 @@ describe('ResumableAgentController resume metadata', () => {
expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
});
it('returns a typed recovery conflict before acknowledging generation startup', async () => {
const recoveryError = new Error('Attached resources could not be restored');
recoveryError.code = ErrorTypes.RESOURCE_RECOVERY_REQUIRED;
mockGenerationJobManager.createJob.mockRejectedValue(recoveryError);
const req = {
user: { id: 'user-123' },
body: {
text: 'Describe the attached image.',
messageId: 'user-msg',
clientRequestId: 'req-abc',
conversationId: 'conversation-123',
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
},
config: {},
};
const res = createResumableResponse();
await AgentController(req, res, jest.fn(), jest.fn(), null);
expect(res.status).toHaveBeenCalledWith(409);
expect(res.json).toHaveBeenCalledWith({
status: 409,
code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
error: 'Attached resources could not be restored',
generationProtocolVersion: 1,
});
expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
});
it('preserves the recovery code in the durable error after acknowledging startup', async () => {
const recoveryError = new Error('Attached resources could not be restored');
recoveryError.code = ErrorTypes.RESOURCE_RECOVERY_REQUIRED;
const initializeClient = jest.fn().mockRejectedValue(recoveryError);
const req = {
user: { id: 'user-123' },
body: {
text: 'Describe the attached image.',
messageId: 'user-msg',
clientRequestId: 'req-abc',
conversationId: 'conversation-123',
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
},
config: {},
};
const res = createResumableResponse();
await AgentController(req, res, jest.fn(), initializeClient, null);
expect(res.status).toHaveBeenCalledWith(200);
expect(res.json).toHaveBeenCalledWith({
streamId: 'conversation-123',
conversationId: 'conversation-123',
generationCreatedAt: 1000,
status: 'started',
generationProtocolVersion: 1,
});
expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
'conversation-123',
JSON.stringify({
status: 409,
code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
error: 'Attached resources could not be restored',
}),
1000,
);
});
it('returns a recovery conflict when the atomic store rejects changed source content', async () => {
const mismatch = new Error('recovery mismatch');
mismatch.code = 'RECOVERY_PAYLOAD_MISMATCH';

38
api/server/controllers/agents/__tests__/responses.unit.spec.js
View file

@ -3,7 +3,7 @@
* Tests that recordCollectedUsage is called correctly for token spending
*/
const { ResourceType } = require('librechat-data-provider');
const { ErrorTypes, ResourceType } = require('librechat-data-provider');
const mockSpendTokens = jest.fn().mockResolvedValue({});
const mockSpendStructuredTokens = jest.fn().mockResolvedValue({});
@ -211,8 +211,8 @@ jest.mock('@librechat/api', () => ({
jest.mock('~/server/services/ToolService', () => ({
loadAgentTools: jest.fn().mockResolvedValue([]),
loadToolsForExecution: jest.fn().mockResolvedValue([]),
isExpectedMCPToolsUnavailableError: (error) =>
error?.code === 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE',
isFatalAgentInitializationError: (error) =>
['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 'resource_recovery_required'].includes(error?.code),
}));
const mockGetMultiplier = jest.fn().mockReturnValue(1);
@ -372,6 +372,38 @@ describe('createResponse controller', () => {
503,
'Expected MCP tools are unavailable',
'server_error',
'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE',
);
});
it('returns the resource recovery status and code before model invocation', async () => {
const { initializeAgent, sendResponsesErrorResponse } = require('@librechat/api');
const { loadAgentTools } = require('~/server/services/ToolService');
const toolError = Object.assign(new Error('resource recovery required'), {
code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
status: 409,
statusCode: 409,
});
loadAgentTools.mockRejectedValueOnce(toolError);
initializeAgent.mockImplementationOnce(async ({ req, res, loadTools, agent }) => {
await loadTools({
req,
res,
tools: ['execute_code'],
model: agent.model,
agentId: agent.id,
provider: agent.provider,
});
});
await createResponse(req, res);
expect(sendResponsesErrorResponse).toHaveBeenCalledWith(
res,
409,
'resource recovery required',
'invalid_request',
ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
);
});

9
api/server/controllers/agents/openai.js
View file

@ -49,7 +49,7 @@ const {
loadAgentTools,
loadToolsForExecution,
getAccessibleMcpServerNames,
isExpectedMCPToolsUnavailableError,
isFatalAgentInitializationError,
} = require('~/server/services/ToolService');
const {
findAccessibleResources,
@ -103,10 +103,10 @@ function createToolLoader(signal, definitionsOnly = true) {
streamId: null, // No resumable stream for OpenAI compat
});
} catch (error) {
logger.error('Error loading tools for agent ' + agentId, error);
if (isExpectedMCPToolsUnavailableError(error)) {
if (isFatalAgentInitializationError(error)) {
throw error;
}
logger.error('Error loading tools for agent ' + agentId, error);
}
};
}
@ -891,7 +891,8 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
: 500;
const errorType =
statusCode >= 400 && statusCode < 500 ? 'invalid_request_error' : 'server_error';
sendErrorResponse(res, statusCode, errorMessage, errorType);
const errorCode = typeof error?.code === 'string' ? error.code : null;
sendErrorResponse(res, statusCode, errorMessage, errorType, errorCode);
}
}
};

25
api/server/controllers/agents/request.js
View file

@ -3,6 +3,7 @@ const { v5: uuidv5 } = require('uuid');
const {
Constants,
EModelEndpoint,
ErrorTypes,
ViolationTypes,
isEphemeralAgentId,
} = require('librechat-data-provider');
@ -50,6 +51,18 @@ function sendGenerationJson(res, status, body, generationProtocolVersion) {
return res.status(status).json({ ...body, generationProtocolVersion });
}
function getResourceRecoveryFailure(error) {
if (error?.code !== ErrorTypes.RESOURCE_RECOVERY_REQUIRED) {
return null;
}
return {
status: 409,
code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
error: error.message || 'Attached resources must be restored before retrying.',
};
}
function createCloseHandler(abortController) {
return function (manual) {
if (!manual) {
@ -1837,6 +1850,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
});
} catch (error) {
logger.error('[ResumableAgentController] Initialization error:', error);
const resourceRecoveryFailure = getResourceRecoveryFailure(error);
try {
if (!res.headersSent) {
if (error?.code === 'GENERATION_PREDECESSOR_MISMATCH') {
@ -1877,6 +1891,13 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
},
generationProtocolVersion,
);
} else if (resourceRecoveryFailure) {
sendGenerationJson(
res,
resourceRecoveryFailure.status,
resourceRecoveryFailure,
generationProtocolVersion,
);
} else {
sendGenerationJson(
res,
@ -1905,7 +1926,9 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
// and the concurrency slot leaks — so swallow its error. (A failed completeJob did not
// finalize anything, so releasing afterward can't let it abort a later replacement.)
if (jobCreatedAt != null) {
const initializationError = error.message || 'Failed to start generation';
const initializationError = resourceRecoveryFailure
? JSON.stringify(resourceRecoveryFailure)
: error.message || 'Failed to start generation';
await GenerationJobManager.completeJob(streamId, initializationError, jobCreatedAt).catch(
(completeErr) => {
logger.warn(

13
api/server/controllers/agents/responses.js
View file

@ -60,7 +60,7 @@ const {
const {
loadAgentTools,
loadToolsForExecution,
isExpectedMCPToolsUnavailableError,
isFatalAgentInitializationError,
} = require('~/server/services/ToolService');
const {
findAccessibleResources,
@ -116,10 +116,10 @@ function createToolLoader(signal, definitionsOnly = true) {
streamId: null,
});
} catch (error) {
logger.error('Error loading tools for agent ' + agentId, error);
if (isExpectedMCPToolsUnavailableError(error)) {
if (isFatalAgentInitializationError(error)) {
throw error;
}
logger.error('Error loading tools for agent ' + agentId, error);
}
};
}
@ -1093,7 +1093,12 @@ const executeResponse = async (envelope, { req, res }) => {
? error.status
: 500;
const errorType = statusCode >= 400 && statusCode < 500 ? 'invalid_request' : 'server_error';
sendResponsesErrorResponse(res, statusCode, errorMessage, errorType);
const errorCode = typeof error?.code === 'string' ? error.code : undefined;
if (errorCode === undefined) {
sendResponsesErrorResponse(res, statusCode, errorMessage, errorType);
} else {
sendResponsesErrorResponse(res, statusCode, errorMessage, errorType, errorCode);
}
}
}
};

4
api/server/services/Endpoints/agents/addedConvo.js
View file

@ -11,7 +11,7 @@ const { isEphemeralAgentId } = require('librechat-data-provider');
const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions');
const { getMCPServerTools } = require('~/server/services/Config');
const { getAccessibleMcpServerNames } = require('~/server/services/MCP');
const { isExpectedMCPToolsUnavailableError } = require('~/server/services/ToolService');
const { isFatalAgentInitializationError } = require('~/server/services/ToolService');
const { getSkillDbMethods, canAuthorSkillFiles } = require('./skillDeps');
const db = require('~/models');
@ -227,7 +227,7 @@ const processAddedConvo = async ({
return { userMCPAuthMap };
} catch (err) {
if (isExpectedMCPToolsUnavailableError(err)) {
if (isFatalAgentInitializationError(err)) {
throw err;
}
logger.error('[processAddedConvo] Error processing addedConvo for parallel agent', err);

17
api/server/services/Endpoints/agents/addedConvo.spec.js
View file

@ -42,8 +42,8 @@ jest.mock('~/server/services/MCP', () => ({
}));
jest.mock('~/server/services/ToolService', () => ({
isExpectedMCPToolsUnavailableError: (error) =>
error?.code === 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE',
isFatalAgentInitializationError: (error) =>
['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 'resource_recovery_required'].includes(error?.code),
}));
jest.mock('./skillDeps', () => ({
@ -59,7 +59,7 @@ jest.mock('~/models', () => ({
}));
const { processAddedConvo } = require('./addedConvo');
const { Constants } = require('librechat-data-provider');
const { Constants, ErrorTypes } = require('librechat-data-provider');
const makeReq = () => ({ user: { id: 'u1', role: 'USER' } });
@ -143,10 +143,13 @@ describe('processAddedConvo', () => {
);
});
it('propagates an expected-MCP-tools failure from an added parallel agent', async () => {
const toolError = Object.assign(new Error('Added agent expected MCP tools are unavailable'), {
code: 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE',
statusCode: 503,
it.each([
['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 503],
[ErrorTypes.RESOURCE_RECOVERY_REQUIRED, 409],
])('propagates fatal %s failures from an added parallel agent', async (code, statusCode) => {
const toolError = Object.assign(new Error(`Added agent failed with ${code}`), {
code,
statusCode,
});
mockInitializeAgent.mockRejectedValueOnce(toolError);

8
api/server/services/Endpoints/agents/initialize.js
View file

@ -40,7 +40,7 @@ const {
loadAgentTools,
loadToolsForExecution,
getAccessibleMcpServerNames,
isExpectedMCPToolsUnavailableError,
isFatalAgentInitializationError,
} = require('~/server/services/ToolService');
const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions');
const {
@ -109,10 +109,10 @@ function createToolLoader(signal, streamId = null, definitionsOnly = false, jobC
accessibleMcpServerNames,
});
} catch (error) {
logger.error('Error loading tools for agent ' + agentId, error);
if (isExpectedMCPToolsUnavailableError(error)) {
if (isFatalAgentInitializationError(error)) {
throw error;
}
logger.error('Error loading tools for agent ' + agentId, error);
}
};
}
@ -804,7 +804,7 @@ const initializeClient = async ({
agentToolContexts.set(agentId, buildAgentToolContext({ agent, config }));
return config;
} catch (err) {
if (isExpectedMCPToolsUnavailableError(err)) {
if (isFatalAgentInitializationError(err)) {
throw err;
}
logger.error(`[processAgent] Error processing subagent ${agentId}:`, err);

39
api/server/services/Endpoints/agents/initialize.spec.js
View file

@ -7,6 +7,7 @@ const {
MAX_SUBAGENT_DEPTH,
MAX_SUBAGENT_GRAPH_NODES,
Constants,
ErrorTypes,
} = require('librechat-data-provider');
const { MongoMemoryServer } = require('mongodb-memory-server');
@ -58,8 +59,8 @@ const mockLoadToolsForExecution = jest.fn();
jest.mock('~/server/services/ToolService', () => ({
loadAgentTools: jest.fn(),
loadToolsForExecution: (...args) => mockLoadToolsForExecution(...args),
isExpectedMCPToolsUnavailableError: (error) =>
error?.code === 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE',
isFatalAgentInitializationError: (error) =>
['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 'resource_recovery_required'].includes(error?.code),
}));
jest.mock('~/server/controllers/ModelController', () => ({
@ -711,6 +712,40 @@ describe('initializeClient — subagent loading', () => {
).rejects.toBe(toolError);
});
it('aborts the run when a pure subagent requires CodeAPI resource recovery', async () => {
const subAgent = await createAgent({
id: SUBAGENT_ID,
name: 'Code Subagent',
provider: 'openai',
model: 'gpt-4',
author: new mongoose.Types.ObjectId(),
tools: ['execute_code'],
});
await grantView(subAgent);
const resourceRecoveryError = Object.assign(new Error('resource recovery required'), {
code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
status: 409,
statusCode: 409,
});
mockInitializeAgent
.mockResolvedValueOnce(
makePrimaryConfig({
subagents: { enabled: true, allowSelf: true, agent_ids: [SUBAGENT_ID] },
}),
)
.mockRejectedValueOnce(resourceRecoveryError);
await expect(
initializeClient({
req: makeSubagentReq(),
res: {},
signal: new AbortController().signal,
endpointOption: makeEndpointOption(),
}),
).rejects.toBe(resourceRecoveryError);
});
it('loads a configured subagent, populates `subagentAgentConfigs`, and keeps it out of `agentConfigs`', async () => {
const subAgent = await createAgent({
id: SUBAGENT_ID,

92
api/server/services/Files/Code/process.js
View file

@ -29,6 +29,7 @@ const {
inferMimeType,
EToolResources,
EModelEndpoint,
ErrorTypes,
mergeFileConfig,
getEndpointFileConfig,
} = require('librechat-data-provider');
@ -784,10 +785,8 @@ async function getSessionInfo(ref, req) {
});
return response.data?.lastModified;
} catch (error) {
logger.debug(
`[getSessionInfo] session lookup failed (treating as cache miss): ${error?.message ?? String(error)}`,
);
} catch (_error) {
logger.debug('[getSessionInfo] session lookup failed (treating as cache miss)');
return null;
}
}
@ -827,6 +826,53 @@ const appendVisibleCodeFileContext = (toolContext, contextLine) => {
return `- Note: The following files are available in the "${Tools.execute_code}" tool environment:${contextLine}`;
};
class CodeResourceRecoveryError extends Error {
constructor({ required, primed, failed }) {
super(JSON.stringify({ type: ErrorTypes.RESOURCE_RECOVERY_REQUIRED }));
this.name = 'CodeResourceRecoveryError';
this.code = ErrorTypes.RESOURCE_RECOVERY_REQUIRED;
this.status = 409;
this.statusCode = 409;
this.details = { required, primed, failed };
this.required = required;
this.primed = primed;
this.failed = failed;
}
}
const getPrimingCorrelation = (req) => ({
requestId: req?.requestId ?? req?.id ?? 'unknown',
runId: req?.body?.messageId ?? req?.body?.conversationId ?? 'unknown',
});
const getReuploadFailureCategory = (error) => {
const status =
error?.response?.status ??
error?.statusCode ??
error?.status ??
error?.$metadata?.httpStatusCode;
const code = error?.code ?? error?.name;
if (
status === 404 ||
code === 'NoSuchKey' ||
code === 'NotFound' ||
code === 'BlobNotFound' ||
code === 'ResourceNotFound'
) {
return 'missing_backing_object';
}
if (
status === 401 ||
status === 403 ||
code === 'AccessDenied' ||
code === 'AccessDeniedException' ||
code === 'Forbidden'
) {
return 'resource_access_denied';
}
return 'reupload_failed';
};
/**
*
* @param {Object} options
@ -888,6 +934,8 @@ const primeFiles = async (options) => {
* paths taken, and the final dispatch summary in one trace. */
let skippedNoRef = 0;
let reuploadFailures = 0;
let requiredCodeFiles = 0;
const reuploadFailureCategories = new Set();
for (let i = 0; i < dbFiles.length; i++) {
const file = dbFiles[i];
@ -898,11 +946,10 @@ const primeFiles = async (options) => {
const ref = file.metadata?.codeEnvRef;
if (!ref) {
skippedNoRef += 1;
logger.debug(
`[primeCodeFiles] file=${file.file_id} path=skip reason=no-codeenvref filename=${file.filename}`,
);
logger.debug(`[primeCodeFiles] file=${file.file_id} path=skip reason=no-codeenvref`);
continue;
}
requiredCodeFiles += 1;
const session_id = ref.storage_session_id;
const id = ref.file_id;
@ -1010,9 +1057,12 @@ const primeFiles = async (options) => {
);
} catch (error) {
reuploadFailures += 1;
const failureCategory = getReuploadFailureCategory(error);
reuploadFailureCategories.add(failureCategory);
const { requestId, runId } = getPrimingCorrelation(req);
logger.error(
`[primeCodeFiles] file=${file.file_id} path=reupload-failed session=${session_id}: ${error.message}`,
error,
`[primeCodeFiles] reupload-failed requestId=${requestId} runId=${runId} ` +
`category=${failureCategory}`,
);
}
};
@ -1043,11 +1093,32 @@ const primeFiles = async (options) => {
/* Dispatch summary — emitted unconditionally so a single grep on
* `[primeCodeFiles] out` always shows the final state, not only
* the per-path trail leading up to it. */
const primedCodeFiles = files.length;
const allRequiredResourcesFailed =
requiredCodeFiles > 0 && primedCodeFiles === 0 && reuploadFailures === requiredCodeFiles;
const { requestId, runId } = getPrimingCorrelation(req);
logger.debug(
`[primeCodeFiles] out: returned=${files.length} ` +
`skippedNoRef=${skippedNoRef} reuploadFailures=${reuploadFailures}`,
`required=${requiredCodeFiles} skippedNoRef=${skippedNoRef} reuploadFailures=${reuploadFailures}`,
);
if (allRequiredResourcesFailed) {
const failureCategory =
reuploadFailureCategories.size === 1
? Array.from(reuploadFailureCategories)[0]
: 'mixed_reupload_failure';
logger.warn(
`[primeCodeFiles] resource-recovery-required requestId=${requestId} runId=${runId} ` +
`required=${requiredCodeFiles} primed=${primedCodeFiles} failed=${reuploadFailures} ` +
`category=${failureCategory}`,
);
throw new CodeResourceRecoveryError({
required: requiredCodeFiles,
primed: primedCodeFiles,
failed: reuploadFailures,
});
}
return { files, toolContext };
};
@ -1457,6 +1528,7 @@ async function writeSandboxFile({
}
module.exports = {
CodeResourceRecoveryError,
primeFiles,
checkIfActive,
getSessionInfo,

71
api/server/services/Files/Code/process.spec.js
View file

@ -29,7 +29,7 @@ jest.mock('librechat-data-provider', () => {
};
});
const { FileContext, ResourceType } = require('librechat-data-provider');
const { ErrorTypes, FileContext, ResourceType } = require('librechat-data-provider');
// Mock uuid
jest.mock('uuid', () => ({
@ -1927,6 +1927,7 @@ describe('Code Process', () => {
agentId: 'agent-123',
resourceType: ResourceType.REMOTE_AGENT,
});
expect(JSON.stringify(logger.debug.mock.calls)).not.toContain(files[0].filename);
});
it('does not read a runtime file record that has no authorized database record', async () => {
@ -2204,6 +2205,74 @@ describe('Code Process', () => {
},
]);
});
it.each([
['Axios/CloudFront 404', { response: { status: 404 } }, 'missing_backing_object'],
[
'AWS SDK NoSuchKey',
{ name: 'NoSuchKey', $metadata: { httpStatusCode: 404 } },
'missing_backing_object',
],
['Azure BlobNotFound', { code: 'BlobNotFound', statusCode: 404 }, 'missing_backing_object'],
['storage access denied', { code: 'AccessDenied', status: 403 }, 'resource_access_denied'],
])(
'fails with a typed recovery error for %s',
async (_errorShape, downloadError, expectedCategory) => {
const dbFile = {
file_id: 'librechat-file-id',
filename: 'cross-region-report.png',
filepath: 'https://storage.us-east.example.test/missing-object',
source: 'local',
context: 'execute_code',
metadata: {
codeEnvRef: {
kind: 'user',
id: 'user-123',
storage_session_id: 'US_EAST_SESSION',
file_id: 'MISSING_OBJECT',
},
},
};
const getDownloadStream = jest.fn().mockRejectedValue(downloadError);
getFiles.mockResolvedValue([dbFile]);
getStrategyFunctions.mockImplementation((source) =>
source === 'execute_code' ? { handleFileUpload: jest.fn() } : { getDownloadStream },
);
mockAxios.mockResolvedValue({ data: null });
await expect(
primeFiles({
req: {
id: 'request-123',
body: { messageId: 'run-123' },
user: { id: 'user-123', role: 'USER' },
},
tool_resources: {
execute_code: { file_ids: ['librechat-file-id'], files: [] },
},
agentId: 'agent-id',
}),
).rejects.toMatchObject({
code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
status: 409,
statusCode: 409,
details: { required: 1, primed: 0, failed: 1 },
required: 1,
primed: 0,
failed: 1,
});
expect(logger.warn).toHaveBeenCalledWith(
expect.stringContaining(
`resource-recovery-required requestId=request-123 runId=run-123 required=1 primed=0 failed=1 category=${expectedCategory}`,
),
);
const failureLogs = logger.error.mock.calls.map(([message]) => message).join('\n');
expect(failureLogs).toContain(`category=${expectedCategory}`);
expect(failureLogs).not.toContain(dbFile.filename);
expect(failureLogs).not.toContain(dbFile.filepath);
},
);
});
describe('primeFiles toolContext for model-visible code files', () => {

13
api/server/services/ToolService.js
View file

@ -33,6 +33,8 @@ const {
buildServerNameAliases,
findShadowedServerNames,
isNormalizationSensitiveName,
AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE,
isFatalAgentInitializationError,
} = require('@librechat/api');
const {
Time,
@ -529,16 +531,15 @@ const isExpectedMCPTool = (toolName) =>
toolName?.includes(Constants.mcp_delimiter) &&
!toolName.startsWith(mcpServerPinPrefix) &&
!isActionTool(toolName);
const expectedMCPToolsUnavailableCode = 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE';
const isExpectedMCPToolsUnavailableError = (error) =>
error?.code === expectedMCPToolsUnavailableCode;
error?.code === AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE;
const createExpectedMCPToolsUnavailableError = (agentName, cause) => {
const subject = agentName ? `Agent "${agentName}"` : 'The agent';
const error = new Error(
`${subject} is configured to use MCP tools, but none are available. Verify that the MCP server is connected and this agent can access its selected tools, then try again.`,
);
error.name = 'AgentToolInitializationError';
error.code = expectedMCPToolsUnavailableCode;
error.code = AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE;
error.status = 503;
error.statusCode = 503;
if (cause != null) {
@ -1165,6 +1166,9 @@ async function loadToolDefinitionsWrapper({
primedCodeFiles = files;
}
} catch (error) {
if (isFatalAgentInitializationError(error)) {
throw error;
}
logger.error('[loadToolDefinitionsWrapper] Error priming code files:', error);
}
}
@ -1269,7 +1273,7 @@ async function loadAgentTools({
accessibleMcpServerNames,
});
} catch (error) {
if (isExpectedMCPToolsUnavailableError(error) || !agent.tools?.some(isExpectedMCPTool)) {
if (isFatalAgentInitializationError(error) || !agent.tools?.some(isExpectedMCPTool)) {
throw error;
}
throw createExpectedMCPToolsUnavailableError(agent.name, error);
@ -2040,6 +2044,7 @@ module.exports = {
loadToolsForExecution,
processRequiredActions,
resolveAgentCapabilities,
isFatalAgentInitializationError,
isExpectedMCPToolsUnavailableError,
/** Re-exported for controllers that already depend on (and mock) this
* module, avoiding a fresh heavy `services/MCP` require chain there. */

28
api/server/services/__tests__/ToolService.spec.js
View file

@ -3,6 +3,7 @@ const {
Tools,
Constants,
ResourceType,
ErrorTypes,
EModelEndpoint,
isActionTool,
actionDelimiter,
@ -25,6 +26,9 @@ const mockLoadToolDefinitions = jest.fn();
const mockGetUserMCPAuthMap = jest.fn();
jest.mock('@librechat/api', () => ({
...jest.requireActual('@librechat/api'),
AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE: 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE',
isFatalAgentInitializationError: (error) =>
['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 'resource_recovery_required'].includes(error?.code),
loadToolDefinitions: (...args) => mockLoadToolDefinitions(...args),
getUserMCPAuthMap: (...args) => mockGetUserMCPAuthMap(...args),
sendEvent: (...args) => mockSendEvent(...args),
@ -266,6 +270,30 @@ describe('ToolService - Action Capability Gating', () => {
expect(primeCodeFiles).toHaveBeenCalledWith(expectedParams);
});
it('propagates a typed CodeAPI resource recovery failure before model invocation', async () => {
const capabilities = [AgentCapabilities.tools, AgentCapabilities.execute_code];
const req = createMockReq(capabilities);
const resourceRecoveryError = Object.assign(new Error('resource recovery required'), {
code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
});
const { primeFiles: primeCodeFiles } = require('~/server/services/Files/Code/process');
primeCodeFiles.mockRejectedValueOnce(resourceRecoveryError);
mockGetEndpointsConfig.mockResolvedValue(createEndpointsConfig(capabilities));
await expect(
loadAgentTools({
req,
res: {},
agent: {
id: 'agent_123',
tools: [Tools.execute_code, 'run_query_mcp_warehouse'],
},
tool_resources: { execute_code: { file_ids: ['stale-file'] } },
definitionsOnly: true,
}),
).rejects.toBe(resourceRecoveryError);
});
it('should exclude action tools from definitions when actions capability is disabled', async () => {
const capabilities = [AgentCapabilities.tools, AgentCapabilities.web_search];
const req = createMockReq(capabilities);

1
client/src/components/Messages/Content/Error.tsx
View file

@ -77,6 +77,7 @@ const errorMessages = {
},
[ErrorTypes.GOOGLE_TOOL_CONFLICT]: 'com_error_google_tool_conflict',
[ErrorTypes.GOOGLE_VIDEO_UNPROCESSABLE]: 'com_error_google_video_unprocessable',
[ErrorTypes.RESOURCE_RECOVERY_REQUIRED]: 'com_error_resource_recovery_required',
[ErrorTypes.STREAM_EXPIRED]: 'com_error_stream_expired',
[ViolationTypes.BAN]:
'Your account has been temporarily banned due to violations of our service.',

1
client/src/locales/en/translation.json
View file

@ -399,6 +399,7 @@
"com_error_no_base_url": "No base URL found. Please provide one and try again.",
"com_error_no_user_key": "No key found. Please provide a key and try again.",
"com_error_refusal": "Response refused by safety filters. Rewrite your message and try again. If you encounter this frequently while using Claude Sonnet 4.5 or Opus 4.1, you can try Sonnet 4, which has different usage restrictions.",
"com_error_resource_recovery_required": "Some files from this conversation could not be restored. Reattach the files and try again.",
"com_error_stream_expired": "The response stream has expired or already completed. Please try again.",
"com_error_token_balance": "Insufficient funds. Balance: {{0}}. Prompt tokens: {{1}}. Cost: {{2}}.",
"com_file_pages": "Pages: {{pages}}",

95
packages/api/src/agents/__tests__/initialize.test.ts
View file

@ -33,7 +33,13 @@ jest.mock('@librechat/agents', () => ({
}));
import { Providers } from '@librechat/agents';
import { Constants, EModelEndpoint, EToolResources, Tools } from 'librechat-data-provider';
import {
Constants,
ErrorTypes,
EModelEndpoint,
EToolResources,
Tools,
} from 'librechat-data-provider';
import type { IMongoFile } from '@librechat/data-schemas';
import type { Agent } from 'librechat-data-provider';
import type { ServerRequest, InitializeResultBase, EndpointTokenConfig } from '~/types';
@ -1505,6 +1511,93 @@ describe('initializeAgent — skill `allowed-tools` union (Phase 6)', () => {
expect(definedNames).not.toContain('mcp__broken__tool');
});
it('does not retry a resource recovery failure when execute_code is skill-added', async () => {
const { agent, req, res, loadTools, db } = createMocks();
agent.tools = ['web_search'];
const { Types } = await import('mongoose');
const skillId = new Types.ObjectId();
const resourceRecoveryError = Object.assign(new Error('resource recovery required'), {
code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
status: 409,
statusCode: 409,
});
loadTools.mockRejectedValue(resourceRecoveryError);
const getSkillByName = buildGetSkillByName(
'code-skill',
[Tools.execute_code],
skillId,
req.user!.id,
);
await expect(
initializeAgent(
{
req,
res,
agent,
loadTools,
endpointOption: { endpoint: EModelEndpoint.agents },
allowedProviders: new Set([Providers.OPENAI]),
isInitialAgent: true,
accessibleSkillIds: [skillId],
manualSkills: ['code-skill'],
},
{ ...db, listSkillsByAccess: emptyListSkillsByAccess, getSkillByName },
),
).rejects.toBe(resourceRecoveryError);
expect(loadTools).toHaveBeenCalledTimes(1);
expect(loadTools.mock.calls[0][0].tools).toEqual(['web_search', Tools.execute_code]);
});
it('still retries when only skill-added tools expect unavailable MCP tools', async () => {
const { agent, req, res, loadTools, db } = createMocks();
agent.tools = ['web_search'];
const { Types } = await import('mongoose');
const skillId = new Types.ObjectId();
const expectedMCPError = Object.assign(new Error('expected MCP tools are unavailable'), {
code: 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE',
statusCode: 503,
});
loadTools.mockRejectedValueOnce(expectedMCPError).mockResolvedValueOnce({
tools: [],
toolContextMap: {},
userMCPAuthMap: undefined,
toolRegistry: undefined,
toolDefinitions: [{ name: 'web_search', description: '', parameters: {} }],
hasDeferredTools: false,
actionsEnabled: undefined,
});
const getSkillByName = buildGetSkillByName(
'mcp-skill',
['mcp__warehouse__query'],
skillId,
req.user!.id,
);
const result = await initializeAgent(
{
req,
res,
agent,
loadTools,
endpointOption: { endpoint: EModelEndpoint.agents },
allowedProviders: new Set([Providers.OPENAI]),
isInitialAgent: true,
accessibleSkillIds: [skillId],
manualSkills: ['mcp-skill'],
},
{ ...db, listSkillsByAccess: emptyListSkillsByAccess, getSkillByName },
);
expect(loadTools).toHaveBeenCalledTimes(2);
expect(loadTools.mock.calls[0][0].tools).toEqual(['web_search', 'mcp__warehouse__query']);
expect(loadTools.mock.calls[1][0].tools).toEqual(['web_search']);
expect(result.toolDefinitions?.map((definition) => definition.name)).toContain('web_search');
});
it('falls back to host-provided skill authoring tools when BOTH loadTools calls return undefined', async () => {
/* Worst-case silent-failure path: production loaders catch errors
and return undefined. If the agent's own tools fail to load AND

47
packages/api/src/agents/discovery.spec.ts
View file

@ -1,4 +1,4 @@
import { EModelEndpoint } from 'librechat-data-provider';
import { ErrorTypes, EModelEndpoint } from 'librechat-data-provider';
import type { Agent, GraphEdge } from 'librechat-data-provider';
import type { Response } from 'express';
import type { InitializedAgent } from './initialize';
@ -1038,11 +1038,11 @@ describe('discoverConnectedAgents', () => {
expect(result.agentConfigs.has('B')).toBe(false);
});
it('propagates an expected-MCP-tools failure from a handoff target', async () => {
const toolError = Object.assign(new Error('Target Agent has no available MCP tools'), {
code: 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE',
statusCode: 503,
});
it.each([
['expected MCP tools are unavailable', 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 503],
['CodeAPI resource recovery is required', ErrorTypes.RESOURCE_RECOVERY_REQUIRED, 409],
])('propagates a fatal handoff initialization error when %s', async (_case, code, statusCode) => {
const toolError = Object.assign(new Error(_case), { code, statusCode });
mockInitializeAgent.mockRejectedValueOnce(toolError);
const primaryConfig = makeConfig('A', [{ from: 'A', to: 'B', edgeType: 'handoff' }]);
@ -1069,6 +1069,41 @@ describe('discoverConnectedAgents', () => {
).rejects.toBe(toolError);
});
it.each([
['expected MCP tools are unavailable', 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 503],
['CodeAPI resource recovery is required', ErrorTypes.RESOURCE_RECOVERY_REQUIRED, 409],
])(
'propagates a fatal legacy-chain initialization error when %s',
async (_case, code, statusCode) => {
const toolError = Object.assign(new Error(_case), { code, statusCode });
mockInitializeAgent.mockRejectedValueOnce(toolError);
const primaryConfig = makeConfig('A');
const getAgent = jest.fn(async () => makeAgent('B', []));
const checkPermission = jest.fn().mockResolvedValue(true);
await expect(
discoverConnectedAgents(
{
req: makeReq(),
res: makeRes(),
primaryConfig,
agent_ids: ['B'],
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
loadTools: jest.fn(),
},
{
getAgent,
checkPermission,
logViolation: jest.fn(),
db: {} as never,
},
),
).rejects.toBe(toolError);
},
);
it('skips when request has no authenticated user', async () => {
const primaryConfig = makeConfig('A', [{ from: 'A', to: 'B', edgeType: 'handoff' }]);

16
packages/api/src/agents/discovery.ts
View file

@ -12,19 +12,9 @@ import type { ServerRequest } from '~/types';
import { validateAgentModel as defaultValidateAgentModel } from './validation';
import { initializeAgent as defaultInitializeAgent } from './initialize';
import { createEdgeCollector, filterOrphanedEdges } from './edges';
import { isFatalAgentInitializationError } from './errors';
import { createSequentialChainEdges } from './chain';
const expectedMCPToolsUnavailableCode = 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE';
function isExpectedMCPToolsUnavailableError(error: unknown): boolean {
return (
typeof error === 'object' &&
error !== null &&
'code' in error &&
error.code === expectedMCPToolsUnavailableCode
);
}
/**
* Callback invoked after a sub-agent is successfully initialized.
* Used by callers that need to track per-agent tool context (e.g., for
@ -338,7 +328,7 @@ export async function discoverConnectedAgents(
collectEdges(agent.edges);
}
} catch (err) {
if (isExpectedMCPToolsUnavailableError(err)) {
if (isFatalAgentInitializationError(err)) {
throw err;
}
logger.error(`[discoverConnectedAgents] Error processing agent ${agentId}:`, err);
@ -355,7 +345,7 @@ export async function discoverConnectedAgents(
try {
await processAgent(agentId);
} catch (err) {
if (isExpectedMCPToolsUnavailableError(err)) {
if (isFatalAgentInitializationError(err)) {
throw err;
}
logger.error(`[discoverConnectedAgents] Error processing chain agent ${agentId}:`, err);

28
packages/api/src/agents/errors.spec.ts Normal file
View file

@ -0,0 +1,28 @@
import { ErrorTypes } from 'librechat-data-provider';
import { AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE, isFatalAgentInitializationError } from './errors';
describe('isFatalAgentInitializationError', () => {
it.each([ErrorTypes.RESOURCE_RECOVERY_REQUIRED, AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE])(
'classifies %s as fatal',
(code) => {
expect(isFatalAgentInitializationError({ code })).toBe(true);
},
);
it('allows skill-added MCP tools to fall back while keeping resource recovery fatal', () => {
const options = { allowExpectedMCPFallback: true };
expect(
isFatalAgentInitializationError({ code: AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE }, options),
).toBe(false);
expect(
isFatalAgentInitializationError({ code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED }, options),
).toBe(true);
});
it.each([undefined, null, new Error('optional tool failed'), { code: 'OPTIONAL_TOOL_FAILED' }])(
'keeps non-fatal failures eligible for legacy soft handling',
(error) => {
expect(isFatalAgentInitializationError(error)).toBe(false);
},
);
});

36
packages/api/src/agents/errors.ts Normal file
View file

@ -0,0 +1,36 @@
import { ErrorTypes } from 'librechat-data-provider';
export const AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE = 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE';
export interface FatalAgentInitializationOptions {
/**
* Skill `allowed-tools` may add an MCP tool beyond the agent's configured
* baseline. That union load is allowed to retry without the skill extras;
* a second failure from the baseline still propagates normally.
*/
allowExpectedMCPFallback?: boolean;
}
function getErrorCode(error: unknown): unknown {
if (error == null || typeof error !== 'object') {
return undefined;
}
return (error as { code?: unknown }).code;
}
/**
* Returns whether agent initialization must abort instead of using the
* legacy soft-failure behavior for unavailable optional tools or agents.
* Keep fatal initialization policy centralized here so every topology and
* ingress path makes the same decision when new invariant errors are added.
*/
export function isFatalAgentInitializationError(
error: unknown,
options: FatalAgentInitializationOptions = {},
): boolean {
const code = getErrorCode(error);
return (
code === ErrorTypes.RESOURCE_RECOVERY_REQUIRED ||
(code === AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE && options.allowExpectedMCPFallback !== true)
);
}

1
packages/api/src/agents/index.ts
View file

@ -9,6 +9,7 @@ export * from './context';
export * from './conversation';
export * from './discovery';
export * from './edges';
export * from './errors';
export * from './envelope';
export * from './handlers';
export * from './harvest';

4
packages/api/src/agents/initialize.ts
View file

@ -64,6 +64,7 @@ import {
} from './tools';
import { registerMemoryTools, memoryToolUsageGuard } from './memory';
import { applyIntentLabels, sanitizeIntentLabels } from './intent';
import { isFatalAgentInitializationError } from './errors';
import { applyBackgroundToolCalls } from './background';
import { filterFilesByEndpointConfig } from '~/files';
import { generateArtifactsPrompt } from '~/prompts';
@ -1024,6 +1025,9 @@ export async function initializeAgent(
try {
loadToolsResult = await callLoadTools(requestedToolNames);
} catch (err) {
if (isFatalAgentInitializationError(err, { allowExpectedMCPFallback: true })) {
throw err;
}
if (extraAllowedToolNames.length > 0) {
logger.warn(
`[allowedTools] loadTools threw with skill-added extras [${extraAllowedToolNames.join(', ')}]; retrying without them:`,

4
packages/data-provider/src/config.ts
View file

@ -2636,6 +2636,10 @@ export enum ErrorTypes {
* Google provider could not process a linked video (most often longer than the model accepts)
*/
GOOGLE_VIDEO_UNPROCESSABLE = 'google_video_unprocessable',
/**
* Required CodeAPI resources could not be restored before model invocation.
*/
RESOURCE_RECOVERY_REQUIRED = 'resource_recovery_required',
/**
* Invalid Agent Provider (excluded by Admin)
*/