diff --git a/api/server/controllers/agents/__tests__/openai.spec.js b/api/server/controllers/agents/__tests__/openai.spec.js index c82053e54e..e11532a969 100644 --- a/api/server/controllers/agents/__tests__/openai.spec.js +++ b/api/server/controllers/agents/__tests__/openai.spec.js @@ -3,7 +3,7 @@ * Tests that recordCollectedUsage is called correctly for token spending */ -const { ResourceType } = require('librechat-data-provider'); +const { ErrorTypes, ResourceType } = require('librechat-data-provider'); const mockProcessStream = jest.fn().mockResolvedValue(undefined); const mockSpendTokens = jest.fn().mockResolvedValue({}); @@ -196,8 +196,8 @@ jest.mock('~/cache', () => ({ jest.mock('~/server/services/ToolService', () => ({ loadAgentTools: jest.fn().mockResolvedValue([]), loadToolsForExecution: jest.fn().mockResolvedValue([]), - isExpectedMCPToolsUnavailableError: (error) => - error?.code === 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', + isFatalAgentInitializationError: (error) => + ['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 'resource_recovery_required'].includes(error?.code), })); const mockGetMultiplier = jest.fn().mockReturnValue(1); @@ -420,6 +420,36 @@ describe('OpenAIChatCompletionController', () => { expect(res.status).toHaveBeenCalledWith(503); }); + + it('returns the resource recovery status and code before model invocation', async () => { + const { createErrorResponse, initializeAgent } = require('@librechat/api'); + const { loadAgentTools } = require('~/server/services/ToolService'); + const toolError = Object.assign(new Error('resource recovery required'), { + code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED, + status: 409, + statusCode: 409, + }); + loadAgentTools.mockRejectedValueOnce(toolError); + initializeAgent.mockImplementationOnce(async ({ req, res, loadTools, agent }) => { + await loadTools({ + req, + res, + tools: ['execute_code'], + model: agent.model, + agentId: agent.id, + provider: agent.provider, + }); + }); + + await OpenAIChatCompletionController(req, res); + + expect(res.status).toHaveBeenCalledWith(409); + expect(createErrorResponse).toHaveBeenCalledWith( + 'resource recovery required', + 'invalid_request_error', + ErrorTypes.RESOURCE_RECOVERY_REQUIRED, + ); + }); }); describe('execution envelope', () => { diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js index 124ae0368c..eda065cdf4 100644 --- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js +++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js @@ -211,6 +211,7 @@ jest.mock('~/models', () => ({ })); const AgentController = require('../request'); +const { ErrorTypes } = require('librechat-data-provider'); const { disposeClient: mockDisposeClient } = require('~/server/cleanup'); const { getMCPRequestContext } = require('~/server/services/MCPRequestContext'); @@ -1749,6 +1750,73 @@ describe('ResumableAgentController resume metadata', () => { expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123'); }); + it('returns a typed recovery conflict before acknowledging generation startup', async () => { + const recoveryError = new Error('Attached resources could not be restored'); + recoveryError.code = ErrorTypes.RESOURCE_RECOVERY_REQUIRED; + mockGenerationJobManager.createJob.mockRejectedValue(recoveryError); + const req = { + user: { id: 'user-123' }, + body: { + text: 'Describe the attached image.', + messageId: 'user-msg', + clientRequestId: 'req-abc', + conversationId: 'conversation-123', + endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } }, + }, + config: {}, + }; + const res = createResumableResponse(); + + await AgentController(req, res, jest.fn(), jest.fn(), null); + + expect(res.status).toHaveBeenCalledWith(409); + expect(res.json).toHaveBeenCalledWith({ + status: 409, + code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED, + error: 'Attached resources could not be restored', + generationProtocolVersion: 1, + }); + expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled(); + }); + + it('preserves the recovery code in the durable error after acknowledging startup', async () => { + const recoveryError = new Error('Attached resources could not be restored'); + recoveryError.code = ErrorTypes.RESOURCE_RECOVERY_REQUIRED; + const initializeClient = jest.fn().mockRejectedValue(recoveryError); + const req = { + user: { id: 'user-123' }, + body: { + text: 'Describe the attached image.', + messageId: 'user-msg', + clientRequestId: 'req-abc', + conversationId: 'conversation-123', + endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } }, + }, + config: {}, + }; + const res = createResumableResponse(); + + await AgentController(req, res, jest.fn(), initializeClient, null); + + expect(res.status).toHaveBeenCalledWith(200); + expect(res.json).toHaveBeenCalledWith({ + streamId: 'conversation-123', + conversationId: 'conversation-123', + generationCreatedAt: 1000, + status: 'started', + generationProtocolVersion: 1, + }); + expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith( + 'conversation-123', + JSON.stringify({ + status: 409, + code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED, + error: 'Attached resources could not be restored', + }), + 1000, + ); + }); + it('returns a recovery conflict when the atomic store rejects changed source content', async () => { const mismatch = new Error('recovery mismatch'); mismatch.code = 'RECOVERY_PAYLOAD_MISMATCH'; diff --git a/api/server/controllers/agents/__tests__/responses.unit.spec.js b/api/server/controllers/agents/__tests__/responses.unit.spec.js index 621dce6799..35f25e3758 100644 --- a/api/server/controllers/agents/__tests__/responses.unit.spec.js +++ b/api/server/controllers/agents/__tests__/responses.unit.spec.js @@ -3,7 +3,7 @@ * Tests that recordCollectedUsage is called correctly for token spending */ -const { ResourceType } = require('librechat-data-provider'); +const { ErrorTypes, ResourceType } = require('librechat-data-provider'); const mockSpendTokens = jest.fn().mockResolvedValue({}); const mockSpendStructuredTokens = jest.fn().mockResolvedValue({}); @@ -211,8 +211,8 @@ jest.mock('@librechat/api', () => ({ jest.mock('~/server/services/ToolService', () => ({ loadAgentTools: jest.fn().mockResolvedValue([]), loadToolsForExecution: jest.fn().mockResolvedValue([]), - isExpectedMCPToolsUnavailableError: (error) => - error?.code === 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', + isFatalAgentInitializationError: (error) => + ['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 'resource_recovery_required'].includes(error?.code), })); const mockGetMultiplier = jest.fn().mockReturnValue(1); @@ -372,6 +372,38 @@ describe('createResponse controller', () => { 503, 'Expected MCP tools are unavailable', 'server_error', + 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', + ); + }); + + it('returns the resource recovery status and code before model invocation', async () => { + const { initializeAgent, sendResponsesErrorResponse } = require('@librechat/api'); + const { loadAgentTools } = require('~/server/services/ToolService'); + const toolError = Object.assign(new Error('resource recovery required'), { + code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED, + status: 409, + statusCode: 409, + }); + loadAgentTools.mockRejectedValueOnce(toolError); + initializeAgent.mockImplementationOnce(async ({ req, res, loadTools, agent }) => { + await loadTools({ + req, + res, + tools: ['execute_code'], + model: agent.model, + agentId: agent.id, + provider: agent.provider, + }); + }); + + await createResponse(req, res); + + expect(sendResponsesErrorResponse).toHaveBeenCalledWith( + res, + 409, + 'resource recovery required', + 'invalid_request', + ErrorTypes.RESOURCE_RECOVERY_REQUIRED, ); }); diff --git a/api/server/controllers/agents/openai.js b/api/server/controllers/agents/openai.js index 897e389a88..ed647e1e62 100644 --- a/api/server/controllers/agents/openai.js +++ b/api/server/controllers/agents/openai.js @@ -49,7 +49,7 @@ const { loadAgentTools, loadToolsForExecution, getAccessibleMcpServerNames, - isExpectedMCPToolsUnavailableError, + isFatalAgentInitializationError, } = require('~/server/services/ToolService'); const { findAccessibleResources, @@ -103,10 +103,10 @@ function createToolLoader(signal, definitionsOnly = true) { streamId: null, // No resumable stream for OpenAI compat }); } catch (error) { - logger.error('Error loading tools for agent ' + agentId, error); - if (isExpectedMCPToolsUnavailableError(error)) { + if (isFatalAgentInitializationError(error)) { throw error; } + logger.error('Error loading tools for agent ' + agentId, error); } }; } @@ -891,7 +891,8 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { : 500; const errorType = statusCode >= 400 && statusCode < 500 ? 'invalid_request_error' : 'server_error'; - sendErrorResponse(res, statusCode, errorMessage, errorType); + const errorCode = typeof error?.code === 'string' ? error.code : null; + sendErrorResponse(res, statusCode, errorMessage, errorType, errorCode); } } }; diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.js index e0aa68f8d4..01effa0049 100644 --- a/api/server/controllers/agents/request.js +++ b/api/server/controllers/agents/request.js @@ -3,6 +3,7 @@ const { v5: uuidv5 } = require('uuid'); const { Constants, EModelEndpoint, + ErrorTypes, ViolationTypes, isEphemeralAgentId, } = require('librechat-data-provider'); @@ -50,6 +51,18 @@ function sendGenerationJson(res, status, body, generationProtocolVersion) { return res.status(status).json({ ...body, generationProtocolVersion }); } +function getResourceRecoveryFailure(error) { + if (error?.code !== ErrorTypes.RESOURCE_RECOVERY_REQUIRED) { + return null; + } + + return { + status: 409, + code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED, + error: error.message || 'Attached resources must be restored before retrying.', + }; +} + function createCloseHandler(abortController) { return function (manual) { if (!manual) { @@ -1837,6 +1850,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit }); } catch (error) { logger.error('[ResumableAgentController] Initialization error:', error); + const resourceRecoveryFailure = getResourceRecoveryFailure(error); try { if (!res.headersSent) { if (error?.code === 'GENERATION_PREDECESSOR_MISMATCH') { @@ -1877,6 +1891,13 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit }, generationProtocolVersion, ); + } else if (resourceRecoveryFailure) { + sendGenerationJson( + res, + resourceRecoveryFailure.status, + resourceRecoveryFailure, + generationProtocolVersion, + ); } else { sendGenerationJson( res, @@ -1905,7 +1926,9 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit // and the concurrency slot leaks — so swallow its error. (A failed completeJob did not // finalize anything, so releasing afterward can't let it abort a later replacement.) if (jobCreatedAt != null) { - const initializationError = error.message || 'Failed to start generation'; + const initializationError = resourceRecoveryFailure + ? JSON.stringify(resourceRecoveryFailure) + : error.message || 'Failed to start generation'; await GenerationJobManager.completeJob(streamId, initializationError, jobCreatedAt).catch( (completeErr) => { logger.warn( diff --git a/api/server/controllers/agents/responses.js b/api/server/controllers/agents/responses.js index 759166fad8..4c0ae894dd 100644 --- a/api/server/controllers/agents/responses.js +++ b/api/server/controllers/agents/responses.js @@ -60,7 +60,7 @@ const { const { loadAgentTools, loadToolsForExecution, - isExpectedMCPToolsUnavailableError, + isFatalAgentInitializationError, } = require('~/server/services/ToolService'); const { findAccessibleResources, @@ -116,10 +116,10 @@ function createToolLoader(signal, definitionsOnly = true) { streamId: null, }); } catch (error) { - logger.error('Error loading tools for agent ' + agentId, error); - if (isExpectedMCPToolsUnavailableError(error)) { + if (isFatalAgentInitializationError(error)) { throw error; } + logger.error('Error loading tools for agent ' + agentId, error); } }; } @@ -1093,7 +1093,12 @@ const executeResponse = async (envelope, { req, res }) => { ? error.status : 500; const errorType = statusCode >= 400 && statusCode < 500 ? 'invalid_request' : 'server_error'; - sendResponsesErrorResponse(res, statusCode, errorMessage, errorType); + const errorCode = typeof error?.code === 'string' ? error.code : undefined; + if (errorCode === undefined) { + sendResponsesErrorResponse(res, statusCode, errorMessage, errorType); + } else { + sendResponsesErrorResponse(res, statusCode, errorMessage, errorType, errorCode); + } } } }; diff --git a/api/server/services/Endpoints/agents/addedConvo.js b/api/server/services/Endpoints/agents/addedConvo.js index eed0df0a84..dee208f3fb 100644 --- a/api/server/services/Endpoints/agents/addedConvo.js +++ b/api/server/services/Endpoints/agents/addedConvo.js @@ -11,7 +11,7 @@ const { isEphemeralAgentId } = require('librechat-data-provider'); const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions'); const { getMCPServerTools } = require('~/server/services/Config'); const { getAccessibleMcpServerNames } = require('~/server/services/MCP'); -const { isExpectedMCPToolsUnavailableError } = require('~/server/services/ToolService'); +const { isFatalAgentInitializationError } = require('~/server/services/ToolService'); const { getSkillDbMethods, canAuthorSkillFiles } = require('./skillDeps'); const db = require('~/models'); @@ -227,7 +227,7 @@ const processAddedConvo = async ({ return { userMCPAuthMap }; } catch (err) { - if (isExpectedMCPToolsUnavailableError(err)) { + if (isFatalAgentInitializationError(err)) { throw err; } logger.error('[processAddedConvo] Error processing addedConvo for parallel agent', err); diff --git a/api/server/services/Endpoints/agents/addedConvo.spec.js b/api/server/services/Endpoints/agents/addedConvo.spec.js index 3033901e0b..a7b391acb5 100644 --- a/api/server/services/Endpoints/agents/addedConvo.spec.js +++ b/api/server/services/Endpoints/agents/addedConvo.spec.js @@ -42,8 +42,8 @@ jest.mock('~/server/services/MCP', () => ({ })); jest.mock('~/server/services/ToolService', () => ({ - isExpectedMCPToolsUnavailableError: (error) => - error?.code === 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', + isFatalAgentInitializationError: (error) => + ['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 'resource_recovery_required'].includes(error?.code), })); jest.mock('./skillDeps', () => ({ @@ -59,7 +59,7 @@ jest.mock('~/models', () => ({ })); const { processAddedConvo } = require('./addedConvo'); -const { Constants } = require('librechat-data-provider'); +const { Constants, ErrorTypes } = require('librechat-data-provider'); const makeReq = () => ({ user: { id: 'u1', role: 'USER' } }); @@ -143,10 +143,13 @@ describe('processAddedConvo', () => { ); }); - it('propagates an expected-MCP-tools failure from an added parallel agent', async () => { - const toolError = Object.assign(new Error('Added agent expected MCP tools are unavailable'), { - code: 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', - statusCode: 503, + it.each([ + ['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 503], + [ErrorTypes.RESOURCE_RECOVERY_REQUIRED, 409], + ])('propagates fatal %s failures from an added parallel agent', async (code, statusCode) => { + const toolError = Object.assign(new Error(`Added agent failed with ${code}`), { + code, + statusCode, }); mockInitializeAgent.mockRejectedValueOnce(toolError); diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js index a9ad6a8103..8375e40d42 100644 --- a/api/server/services/Endpoints/agents/initialize.js +++ b/api/server/services/Endpoints/agents/initialize.js @@ -40,7 +40,7 @@ const { loadAgentTools, loadToolsForExecution, getAccessibleMcpServerNames, - isExpectedMCPToolsUnavailableError, + isFatalAgentInitializationError, } = require('~/server/services/ToolService'); const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions'); const { @@ -109,10 +109,10 @@ function createToolLoader(signal, streamId = null, definitionsOnly = false, jobC accessibleMcpServerNames, }); } catch (error) { - logger.error('Error loading tools for agent ' + agentId, error); - if (isExpectedMCPToolsUnavailableError(error)) { + if (isFatalAgentInitializationError(error)) { throw error; } + logger.error('Error loading tools for agent ' + agentId, error); } }; } @@ -804,7 +804,7 @@ const initializeClient = async ({ agentToolContexts.set(agentId, buildAgentToolContext({ agent, config })); return config; } catch (err) { - if (isExpectedMCPToolsUnavailableError(err)) { + if (isFatalAgentInitializationError(err)) { throw err; } logger.error(`[processAgent] Error processing subagent ${agentId}:`, err); diff --git a/api/server/services/Endpoints/agents/initialize.spec.js b/api/server/services/Endpoints/agents/initialize.spec.js index 59c4b654b0..144d27ff22 100644 --- a/api/server/services/Endpoints/agents/initialize.spec.js +++ b/api/server/services/Endpoints/agents/initialize.spec.js @@ -7,6 +7,7 @@ const { MAX_SUBAGENT_DEPTH, MAX_SUBAGENT_GRAPH_NODES, Constants, + ErrorTypes, } = require('librechat-data-provider'); const { MongoMemoryServer } = require('mongodb-memory-server'); @@ -58,8 +59,8 @@ const mockLoadToolsForExecution = jest.fn(); jest.mock('~/server/services/ToolService', () => ({ loadAgentTools: jest.fn(), loadToolsForExecution: (...args) => mockLoadToolsForExecution(...args), - isExpectedMCPToolsUnavailableError: (error) => - error?.code === 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', + isFatalAgentInitializationError: (error) => + ['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 'resource_recovery_required'].includes(error?.code), })); jest.mock('~/server/controllers/ModelController', () => ({ @@ -711,6 +712,40 @@ describe('initializeClient — subagent loading', () => { ).rejects.toBe(toolError); }); + it('aborts the run when a pure subagent requires CodeAPI resource recovery', async () => { + const subAgent = await createAgent({ + id: SUBAGENT_ID, + name: 'Code Subagent', + provider: 'openai', + model: 'gpt-4', + author: new mongoose.Types.ObjectId(), + tools: ['execute_code'], + }); + await grantView(subAgent); + + const resourceRecoveryError = Object.assign(new Error('resource recovery required'), { + code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED, + status: 409, + statusCode: 409, + }); + mockInitializeAgent + .mockResolvedValueOnce( + makePrimaryConfig({ + subagents: { enabled: true, allowSelf: true, agent_ids: [SUBAGENT_ID] }, + }), + ) + .mockRejectedValueOnce(resourceRecoveryError); + + await expect( + initializeClient({ + req: makeSubagentReq(), + res: {}, + signal: new AbortController().signal, + endpointOption: makeEndpointOption(), + }), + ).rejects.toBe(resourceRecoveryError); + }); + it('loads a configured subagent, populates `subagentAgentConfigs`, and keeps it out of `agentConfigs`', async () => { const subAgent = await createAgent({ id: SUBAGENT_ID, diff --git a/api/server/services/Files/Code/process.js b/api/server/services/Files/Code/process.js index 67c2bf9fec..33b28a2d1d 100644 --- a/api/server/services/Files/Code/process.js +++ b/api/server/services/Files/Code/process.js @@ -29,6 +29,7 @@ const { inferMimeType, EToolResources, EModelEndpoint, + ErrorTypes, mergeFileConfig, getEndpointFileConfig, } = require('librechat-data-provider'); @@ -784,10 +785,8 @@ async function getSessionInfo(ref, req) { }); return response.data?.lastModified; - } catch (error) { - logger.debug( - `[getSessionInfo] session lookup failed (treating as cache miss): ${error?.message ?? String(error)}`, - ); + } catch (_error) { + logger.debug('[getSessionInfo] session lookup failed (treating as cache miss)'); return null; } } @@ -827,6 +826,53 @@ const appendVisibleCodeFileContext = (toolContext, contextLine) => { return `- Note: The following files are available in the "${Tools.execute_code}" tool environment:${contextLine}`; }; +class CodeResourceRecoveryError extends Error { + constructor({ required, primed, failed }) { + super(JSON.stringify({ type: ErrorTypes.RESOURCE_RECOVERY_REQUIRED })); + this.name = 'CodeResourceRecoveryError'; + this.code = ErrorTypes.RESOURCE_RECOVERY_REQUIRED; + this.status = 409; + this.statusCode = 409; + this.details = { required, primed, failed }; + this.required = required; + this.primed = primed; + this.failed = failed; + } +} + +const getPrimingCorrelation = (req) => ({ + requestId: req?.requestId ?? req?.id ?? 'unknown', + runId: req?.body?.messageId ?? req?.body?.conversationId ?? 'unknown', +}); + +const getReuploadFailureCategory = (error) => { + const status = + error?.response?.status ?? + error?.statusCode ?? + error?.status ?? + error?.$metadata?.httpStatusCode; + const code = error?.code ?? error?.name; + if ( + status === 404 || + code === 'NoSuchKey' || + code === 'NotFound' || + code === 'BlobNotFound' || + code === 'ResourceNotFound' + ) { + return 'missing_backing_object'; + } + if ( + status === 401 || + status === 403 || + code === 'AccessDenied' || + code === 'AccessDeniedException' || + code === 'Forbidden' + ) { + return 'resource_access_denied'; + } + return 'reupload_failed'; +}; + /** * * @param {Object} options @@ -888,6 +934,8 @@ const primeFiles = async (options) => { * paths taken, and the final dispatch summary in one trace. */ let skippedNoRef = 0; let reuploadFailures = 0; + let requiredCodeFiles = 0; + const reuploadFailureCategories = new Set(); for (let i = 0; i < dbFiles.length; i++) { const file = dbFiles[i]; @@ -898,11 +946,10 @@ const primeFiles = async (options) => { const ref = file.metadata?.codeEnvRef; if (!ref) { skippedNoRef += 1; - logger.debug( - `[primeCodeFiles] file=${file.file_id} path=skip reason=no-codeenvref filename=${file.filename}`, - ); + logger.debug(`[primeCodeFiles] file=${file.file_id} path=skip reason=no-codeenvref`); continue; } + requiredCodeFiles += 1; const session_id = ref.storage_session_id; const id = ref.file_id; @@ -1010,9 +1057,12 @@ const primeFiles = async (options) => { ); } catch (error) { reuploadFailures += 1; + const failureCategory = getReuploadFailureCategory(error); + reuploadFailureCategories.add(failureCategory); + const { requestId, runId } = getPrimingCorrelation(req); logger.error( - `[primeCodeFiles] file=${file.file_id} path=reupload-failed session=${session_id}: ${error.message}`, - error, + `[primeCodeFiles] reupload-failed requestId=${requestId} runId=${runId} ` + + `category=${failureCategory}`, ); } }; @@ -1043,11 +1093,32 @@ const primeFiles = async (options) => { /* Dispatch summary — emitted unconditionally so a single grep on * `[primeCodeFiles] out` always shows the final state, not only * the per-path trail leading up to it. */ + const primedCodeFiles = files.length; + const allRequiredResourcesFailed = + requiredCodeFiles > 0 && primedCodeFiles === 0 && reuploadFailures === requiredCodeFiles; + const { requestId, runId } = getPrimingCorrelation(req); logger.debug( `[primeCodeFiles] out: returned=${files.length} ` + - `skippedNoRef=${skippedNoRef} reuploadFailures=${reuploadFailures}`, + `required=${requiredCodeFiles} skippedNoRef=${skippedNoRef} reuploadFailures=${reuploadFailures}`, ); + if (allRequiredResourcesFailed) { + const failureCategory = + reuploadFailureCategories.size === 1 + ? Array.from(reuploadFailureCategories)[0] + : 'mixed_reupload_failure'; + logger.warn( + `[primeCodeFiles] resource-recovery-required requestId=${requestId} runId=${runId} ` + + `required=${requiredCodeFiles} primed=${primedCodeFiles} failed=${reuploadFailures} ` + + `category=${failureCategory}`, + ); + throw new CodeResourceRecoveryError({ + required: requiredCodeFiles, + primed: primedCodeFiles, + failed: reuploadFailures, + }); + } + return { files, toolContext }; }; @@ -1457,6 +1528,7 @@ async function writeSandboxFile({ } module.exports = { + CodeResourceRecoveryError, primeFiles, checkIfActive, getSessionInfo, diff --git a/api/server/services/Files/Code/process.spec.js b/api/server/services/Files/Code/process.spec.js index 220b3d2b56..99e4f7a491 100644 --- a/api/server/services/Files/Code/process.spec.js +++ b/api/server/services/Files/Code/process.spec.js @@ -29,7 +29,7 @@ jest.mock('librechat-data-provider', () => { }; }); -const { FileContext, ResourceType } = require('librechat-data-provider'); +const { ErrorTypes, FileContext, ResourceType } = require('librechat-data-provider'); // Mock uuid jest.mock('uuid', () => ({ @@ -1927,6 +1927,7 @@ describe('Code Process', () => { agentId: 'agent-123', resourceType: ResourceType.REMOTE_AGENT, }); + expect(JSON.stringify(logger.debug.mock.calls)).not.toContain(files[0].filename); }); it('does not read a runtime file record that has no authorized database record', async () => { @@ -2204,6 +2205,74 @@ describe('Code Process', () => { }, ]); }); + + it.each([ + ['Axios/CloudFront 404', { response: { status: 404 } }, 'missing_backing_object'], + [ + 'AWS SDK NoSuchKey', + { name: 'NoSuchKey', $metadata: { httpStatusCode: 404 } }, + 'missing_backing_object', + ], + ['Azure BlobNotFound', { code: 'BlobNotFound', statusCode: 404 }, 'missing_backing_object'], + ['storage access denied', { code: 'AccessDenied', status: 403 }, 'resource_access_denied'], + ])( + 'fails with a typed recovery error for %s', + async (_errorShape, downloadError, expectedCategory) => { + const dbFile = { + file_id: 'librechat-file-id', + filename: 'cross-region-report.png', + filepath: 'https://storage.us-east.example.test/missing-object', + source: 'local', + context: 'execute_code', + metadata: { + codeEnvRef: { + kind: 'user', + id: 'user-123', + storage_session_id: 'US_EAST_SESSION', + file_id: 'MISSING_OBJECT', + }, + }, + }; + const getDownloadStream = jest.fn().mockRejectedValue(downloadError); + getFiles.mockResolvedValue([dbFile]); + getStrategyFunctions.mockImplementation((source) => + source === 'execute_code' ? { handleFileUpload: jest.fn() } : { getDownloadStream }, + ); + mockAxios.mockResolvedValue({ data: null }); + + await expect( + primeFiles({ + req: { + id: 'request-123', + body: { messageId: 'run-123' }, + user: { id: 'user-123', role: 'USER' }, + }, + tool_resources: { + execute_code: { file_ids: ['librechat-file-id'], files: [] }, + }, + agentId: 'agent-id', + }), + ).rejects.toMatchObject({ + code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED, + status: 409, + statusCode: 409, + details: { required: 1, primed: 0, failed: 1 }, + required: 1, + primed: 0, + failed: 1, + }); + + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining( + `resource-recovery-required requestId=request-123 runId=run-123 required=1 primed=0 failed=1 category=${expectedCategory}`, + ), + ); + const failureLogs = logger.error.mock.calls.map(([message]) => message).join('\n'); + expect(failureLogs).toContain(`category=${expectedCategory}`); + expect(failureLogs).not.toContain(dbFile.filename); + expect(failureLogs).not.toContain(dbFile.filepath); + }, + ); }); describe('primeFiles toolContext for model-visible code files', () => { diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.js index b29d172c99..db790803a5 100644 --- a/api/server/services/ToolService.js +++ b/api/server/services/ToolService.js @@ -33,6 +33,8 @@ const { buildServerNameAliases, findShadowedServerNames, isNormalizationSensitiveName, + AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE, + isFatalAgentInitializationError, } = require('@librechat/api'); const { Time, @@ -529,16 +531,15 @@ const isExpectedMCPTool = (toolName) => toolName?.includes(Constants.mcp_delimiter) && !toolName.startsWith(mcpServerPinPrefix) && !isActionTool(toolName); -const expectedMCPToolsUnavailableCode = 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE'; const isExpectedMCPToolsUnavailableError = (error) => - error?.code === expectedMCPToolsUnavailableCode; + error?.code === AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE; const createExpectedMCPToolsUnavailableError = (agentName, cause) => { const subject = agentName ? `Agent "${agentName}"` : 'The agent'; const error = new Error( `${subject} is configured to use MCP tools, but none are available. Verify that the MCP server is connected and this agent can access its selected tools, then try again.`, ); error.name = 'AgentToolInitializationError'; - error.code = expectedMCPToolsUnavailableCode; + error.code = AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE; error.status = 503; error.statusCode = 503; if (cause != null) { @@ -1165,6 +1166,9 @@ async function loadToolDefinitionsWrapper({ primedCodeFiles = files; } } catch (error) { + if (isFatalAgentInitializationError(error)) { + throw error; + } logger.error('[loadToolDefinitionsWrapper] Error priming code files:', error); } } @@ -1269,7 +1273,7 @@ async function loadAgentTools({ accessibleMcpServerNames, }); } catch (error) { - if (isExpectedMCPToolsUnavailableError(error) || !agent.tools?.some(isExpectedMCPTool)) { + if (isFatalAgentInitializationError(error) || !agent.tools?.some(isExpectedMCPTool)) { throw error; } throw createExpectedMCPToolsUnavailableError(agent.name, error); @@ -2040,6 +2044,7 @@ module.exports = { loadToolsForExecution, processRequiredActions, resolveAgentCapabilities, + isFatalAgentInitializationError, isExpectedMCPToolsUnavailableError, /** Re-exported for controllers that already depend on (and mock) this * module, avoiding a fresh heavy `services/MCP` require chain there. */ diff --git a/api/server/services/__tests__/ToolService.spec.js b/api/server/services/__tests__/ToolService.spec.js index 850a18b4e2..0d0b60b71e 100644 --- a/api/server/services/__tests__/ToolService.spec.js +++ b/api/server/services/__tests__/ToolService.spec.js @@ -3,6 +3,7 @@ const { Tools, Constants, ResourceType, + ErrorTypes, EModelEndpoint, isActionTool, actionDelimiter, @@ -25,6 +26,9 @@ const mockLoadToolDefinitions = jest.fn(); const mockGetUserMCPAuthMap = jest.fn(); jest.mock('@librechat/api', () => ({ ...jest.requireActual('@librechat/api'), + AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE: 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', + isFatalAgentInitializationError: (error) => + ['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 'resource_recovery_required'].includes(error?.code), loadToolDefinitions: (...args) => mockLoadToolDefinitions(...args), getUserMCPAuthMap: (...args) => mockGetUserMCPAuthMap(...args), sendEvent: (...args) => mockSendEvent(...args), @@ -266,6 +270,30 @@ describe('ToolService - Action Capability Gating', () => { expect(primeCodeFiles).toHaveBeenCalledWith(expectedParams); }); + it('propagates a typed CodeAPI resource recovery failure before model invocation', async () => { + const capabilities = [AgentCapabilities.tools, AgentCapabilities.execute_code]; + const req = createMockReq(capabilities); + const resourceRecoveryError = Object.assign(new Error('resource recovery required'), { + code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED, + }); + const { primeFiles: primeCodeFiles } = require('~/server/services/Files/Code/process'); + primeCodeFiles.mockRejectedValueOnce(resourceRecoveryError); + mockGetEndpointsConfig.mockResolvedValue(createEndpointsConfig(capabilities)); + + await expect( + loadAgentTools({ + req, + res: {}, + agent: { + id: 'agent_123', + tools: [Tools.execute_code, 'run_query_mcp_warehouse'], + }, + tool_resources: { execute_code: { file_ids: ['stale-file'] } }, + definitionsOnly: true, + }), + ).rejects.toBe(resourceRecoveryError); + }); + it('should exclude action tools from definitions when actions capability is disabled', async () => { const capabilities = [AgentCapabilities.tools, AgentCapabilities.web_search]; const req = createMockReq(capabilities); diff --git a/client/src/components/Messages/Content/Error.tsx b/client/src/components/Messages/Content/Error.tsx index f6e262a350..974d54e8af 100644 --- a/client/src/components/Messages/Content/Error.tsx +++ b/client/src/components/Messages/Content/Error.tsx @@ -77,6 +77,7 @@ const errorMessages = { }, [ErrorTypes.GOOGLE_TOOL_CONFLICT]: 'com_error_google_tool_conflict', [ErrorTypes.GOOGLE_VIDEO_UNPROCESSABLE]: 'com_error_google_video_unprocessable', + [ErrorTypes.RESOURCE_RECOVERY_REQUIRED]: 'com_error_resource_recovery_required', [ErrorTypes.STREAM_EXPIRED]: 'com_error_stream_expired', [ViolationTypes.BAN]: 'Your account has been temporarily banned due to violations of our service.', diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.json index 2b06395da8..61986241e9 100644 --- a/client/src/locales/en/translation.json +++ b/client/src/locales/en/translation.json @@ -399,6 +399,7 @@ "com_error_no_base_url": "No base URL found. Please provide one and try again.", "com_error_no_user_key": "No key found. Please provide a key and try again.", "com_error_refusal": "Response refused by safety filters. Rewrite your message and try again. If you encounter this frequently while using Claude Sonnet 4.5 or Opus 4.1, you can try Sonnet 4, which has different usage restrictions.", + "com_error_resource_recovery_required": "Some files from this conversation could not be restored. Reattach the files and try again.", "com_error_stream_expired": "The response stream has expired or already completed. Please try again.", "com_error_token_balance": "Insufficient funds. Balance: {{0}}. Prompt tokens: {{1}}. Cost: {{2}}.", "com_file_pages": "Pages: {{pages}}", diff --git a/packages/api/src/agents/__tests__/initialize.test.ts b/packages/api/src/agents/__tests__/initialize.test.ts index 27bbe3789c..c565227bea 100644 --- a/packages/api/src/agents/__tests__/initialize.test.ts +++ b/packages/api/src/agents/__tests__/initialize.test.ts @@ -33,7 +33,13 @@ jest.mock('@librechat/agents', () => ({ })); import { Providers } from '@librechat/agents'; -import { Constants, EModelEndpoint, EToolResources, Tools } from 'librechat-data-provider'; +import { + Constants, + ErrorTypes, + EModelEndpoint, + EToolResources, + Tools, +} from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { Agent } from 'librechat-data-provider'; import type { ServerRequest, InitializeResultBase, EndpointTokenConfig } from '~/types'; @@ -1505,6 +1511,93 @@ describe('initializeAgent — skill `allowed-tools` union (Phase 6)', () => { expect(definedNames).not.toContain('mcp__broken__tool'); }); + it('does not retry a resource recovery failure when execute_code is skill-added', async () => { + const { agent, req, res, loadTools, db } = createMocks(); + agent.tools = ['web_search']; + const { Types } = await import('mongoose'); + const skillId = new Types.ObjectId(); + const resourceRecoveryError = Object.assign(new Error('resource recovery required'), { + code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED, + status: 409, + statusCode: 409, + }); + loadTools.mockRejectedValue(resourceRecoveryError); + + const getSkillByName = buildGetSkillByName( + 'code-skill', + [Tools.execute_code], + skillId, + req.user!.id, + ); + + await expect( + initializeAgent( + { + req, + res, + agent, + loadTools, + endpointOption: { endpoint: EModelEndpoint.agents }, + allowedProviders: new Set([Providers.OPENAI]), + isInitialAgent: true, + accessibleSkillIds: [skillId], + manualSkills: ['code-skill'], + }, + { ...db, listSkillsByAccess: emptyListSkillsByAccess, getSkillByName }, + ), + ).rejects.toBe(resourceRecoveryError); + + expect(loadTools).toHaveBeenCalledTimes(1); + expect(loadTools.mock.calls[0][0].tools).toEqual(['web_search', Tools.execute_code]); + }); + + it('still retries when only skill-added tools expect unavailable MCP tools', async () => { + const { agent, req, res, loadTools, db } = createMocks(); + agent.tools = ['web_search']; + const { Types } = await import('mongoose'); + const skillId = new Types.ObjectId(); + const expectedMCPError = Object.assign(new Error('expected MCP tools are unavailable'), { + code: 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', + statusCode: 503, + }); + loadTools.mockRejectedValueOnce(expectedMCPError).mockResolvedValueOnce({ + tools: [], + toolContextMap: {}, + userMCPAuthMap: undefined, + toolRegistry: undefined, + toolDefinitions: [{ name: 'web_search', description: '', parameters: {} }], + hasDeferredTools: false, + actionsEnabled: undefined, + }); + + const getSkillByName = buildGetSkillByName( + 'mcp-skill', + ['mcp__warehouse__query'], + skillId, + req.user!.id, + ); + + const result = await initializeAgent( + { + req, + res, + agent, + loadTools, + endpointOption: { endpoint: EModelEndpoint.agents }, + allowedProviders: new Set([Providers.OPENAI]), + isInitialAgent: true, + accessibleSkillIds: [skillId], + manualSkills: ['mcp-skill'], + }, + { ...db, listSkillsByAccess: emptyListSkillsByAccess, getSkillByName }, + ); + + expect(loadTools).toHaveBeenCalledTimes(2); + expect(loadTools.mock.calls[0][0].tools).toEqual(['web_search', 'mcp__warehouse__query']); + expect(loadTools.mock.calls[1][0].tools).toEqual(['web_search']); + expect(result.toolDefinitions?.map((definition) => definition.name)).toContain('web_search'); + }); + it('falls back to host-provided skill authoring tools when BOTH loadTools calls return undefined', async () => { /* Worst-case silent-failure path: production loaders catch errors and return undefined. If the agent's own tools fail to load AND diff --git a/packages/api/src/agents/discovery.spec.ts b/packages/api/src/agents/discovery.spec.ts index fdfe163646..a80833f7c4 100644 --- a/packages/api/src/agents/discovery.spec.ts +++ b/packages/api/src/agents/discovery.spec.ts @@ -1,4 +1,4 @@ -import { EModelEndpoint } from 'librechat-data-provider'; +import { ErrorTypes, EModelEndpoint } from 'librechat-data-provider'; import type { Agent, GraphEdge } from 'librechat-data-provider'; import type { Response } from 'express'; import type { InitializedAgent } from './initialize'; @@ -1038,11 +1038,11 @@ describe('discoverConnectedAgents', () => { expect(result.agentConfigs.has('B')).toBe(false); }); - it('propagates an expected-MCP-tools failure from a handoff target', async () => { - const toolError = Object.assign(new Error('Target Agent has no available MCP tools'), { - code: 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', - statusCode: 503, - }); + it.each([ + ['expected MCP tools are unavailable', 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 503], + ['CodeAPI resource recovery is required', ErrorTypes.RESOURCE_RECOVERY_REQUIRED, 409], + ])('propagates a fatal handoff initialization error when %s', async (_case, code, statusCode) => { + const toolError = Object.assign(new Error(_case), { code, statusCode }); mockInitializeAgent.mockRejectedValueOnce(toolError); const primaryConfig = makeConfig('A', [{ from: 'A', to: 'B', edgeType: 'handoff' }]); @@ -1069,6 +1069,41 @@ describe('discoverConnectedAgents', () => { ).rejects.toBe(toolError); }); + it.each([ + ['expected MCP tools are unavailable', 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 503], + ['CodeAPI resource recovery is required', ErrorTypes.RESOURCE_RECOVERY_REQUIRED, 409], + ])( + 'propagates a fatal legacy-chain initialization error when %s', + async (_case, code, statusCode) => { + const toolError = Object.assign(new Error(_case), { code, statusCode }); + mockInitializeAgent.mockRejectedValueOnce(toolError); + + const primaryConfig = makeConfig('A'); + const getAgent = jest.fn(async () => makeAgent('B', [])); + const checkPermission = jest.fn().mockResolvedValue(true); + + await expect( + discoverConnectedAgents( + { + req: makeReq(), + res: makeRes(), + primaryConfig, + agent_ids: ['B'], + allowedProviders: new Set(), + modelsConfig: { openai: ['gpt-4o'] }, + loadTools: jest.fn(), + }, + { + getAgent, + checkPermission, + logViolation: jest.fn(), + db: {} as never, + }, + ), + ).rejects.toBe(toolError); + }, + ); + it('skips when request has no authenticated user', async () => { const primaryConfig = makeConfig('A', [{ from: 'A', to: 'B', edgeType: 'handoff' }]); diff --git a/packages/api/src/agents/discovery.ts b/packages/api/src/agents/discovery.ts index f093373d8b..b12b8b8d24 100644 --- a/packages/api/src/agents/discovery.ts +++ b/packages/api/src/agents/discovery.ts @@ -12,19 +12,9 @@ import type { ServerRequest } from '~/types'; import { validateAgentModel as defaultValidateAgentModel } from './validation'; import { initializeAgent as defaultInitializeAgent } from './initialize'; import { createEdgeCollector, filterOrphanedEdges } from './edges'; +import { isFatalAgentInitializationError } from './errors'; import { createSequentialChainEdges } from './chain'; -const expectedMCPToolsUnavailableCode = 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE'; - -function isExpectedMCPToolsUnavailableError(error: unknown): boolean { - return ( - typeof error === 'object' && - error !== null && - 'code' in error && - error.code === expectedMCPToolsUnavailableCode - ); -} - /** * Callback invoked after a sub-agent is successfully initialized. * Used by callers that need to track per-agent tool context (e.g., for @@ -338,7 +328,7 @@ export async function discoverConnectedAgents( collectEdges(agent.edges); } } catch (err) { - if (isExpectedMCPToolsUnavailableError(err)) { + if (isFatalAgentInitializationError(err)) { throw err; } logger.error(`[discoverConnectedAgents] Error processing agent ${agentId}:`, err); @@ -355,7 +345,7 @@ export async function discoverConnectedAgents( try { await processAgent(agentId); } catch (err) { - if (isExpectedMCPToolsUnavailableError(err)) { + if (isFatalAgentInitializationError(err)) { throw err; } logger.error(`[discoverConnectedAgents] Error processing chain agent ${agentId}:`, err); diff --git a/packages/api/src/agents/errors.spec.ts b/packages/api/src/agents/errors.spec.ts new file mode 100644 index 0000000000..b2c88a25c7 --- /dev/null +++ b/packages/api/src/agents/errors.spec.ts @@ -0,0 +1,28 @@ +import { ErrorTypes } from 'librechat-data-provider'; +import { AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE, isFatalAgentInitializationError } from './errors'; + +describe('isFatalAgentInitializationError', () => { + it.each([ErrorTypes.RESOURCE_RECOVERY_REQUIRED, AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE])( + 'classifies %s as fatal', + (code) => { + expect(isFatalAgentInitializationError({ code })).toBe(true); + }, + ); + + it('allows skill-added MCP tools to fall back while keeping resource recovery fatal', () => { + const options = { allowExpectedMCPFallback: true }; + expect( + isFatalAgentInitializationError({ code: AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE }, options), + ).toBe(false); + expect( + isFatalAgentInitializationError({ code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED }, options), + ).toBe(true); + }); + + it.each([undefined, null, new Error('optional tool failed'), { code: 'OPTIONAL_TOOL_FAILED' }])( + 'keeps non-fatal failures eligible for legacy soft handling', + (error) => { + expect(isFatalAgentInitializationError(error)).toBe(false); + }, + ); +}); diff --git a/packages/api/src/agents/errors.ts b/packages/api/src/agents/errors.ts new file mode 100644 index 0000000000..b1809a094d --- /dev/null +++ b/packages/api/src/agents/errors.ts @@ -0,0 +1,36 @@ +import { ErrorTypes } from 'librechat-data-provider'; + +export const AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE = 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE'; + +export interface FatalAgentInitializationOptions { + /** + * Skill `allowed-tools` may add an MCP tool beyond the agent's configured + * baseline. That union load is allowed to retry without the skill extras; + * a second failure from the baseline still propagates normally. + */ + allowExpectedMCPFallback?: boolean; +} + +function getErrorCode(error: unknown): unknown { + if (error == null || typeof error !== 'object') { + return undefined; + } + return (error as { code?: unknown }).code; +} + +/** + * Returns whether agent initialization must abort instead of using the + * legacy soft-failure behavior for unavailable optional tools or agents. + * Keep fatal initialization policy centralized here so every topology and + * ingress path makes the same decision when new invariant errors are added. + */ +export function isFatalAgentInitializationError( + error: unknown, + options: FatalAgentInitializationOptions = {}, +): boolean { + const code = getErrorCode(error); + return ( + code === ErrorTypes.RESOURCE_RECOVERY_REQUIRED || + (code === AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE && options.allowExpectedMCPFallback !== true) + ); +} diff --git a/packages/api/src/agents/index.ts b/packages/api/src/agents/index.ts index 283ae4360a..46da9f2a8e 100644 --- a/packages/api/src/agents/index.ts +++ b/packages/api/src/agents/index.ts @@ -9,6 +9,7 @@ export * from './context'; export * from './conversation'; export * from './discovery'; export * from './edges'; +export * from './errors'; export * from './envelope'; export * from './handlers'; export * from './harvest'; diff --git a/packages/api/src/agents/initialize.ts b/packages/api/src/agents/initialize.ts index e1f8c7eb22..b9b5eed8c8 100644 --- a/packages/api/src/agents/initialize.ts +++ b/packages/api/src/agents/initialize.ts @@ -64,6 +64,7 @@ import { } from './tools'; import { registerMemoryTools, memoryToolUsageGuard } from './memory'; import { applyIntentLabels, sanitizeIntentLabels } from './intent'; +import { isFatalAgentInitializationError } from './errors'; import { applyBackgroundToolCalls } from './background'; import { filterFilesByEndpointConfig } from '~/files'; import { generateArtifactsPrompt } from '~/prompts'; @@ -1024,6 +1025,9 @@ export async function initializeAgent( try { loadToolsResult = await callLoadTools(requestedToolNames); } catch (err) { + if (isFatalAgentInitializationError(err, { allowExpectedMCPFallback: true })) { + throw err; + } if (extraAllowedToolNames.length > 0) { logger.warn( `[allowedTools] loadTools threw with skill-added extras [${extraAllowedToolNames.join(', ')}]; retrying without them:`, diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 5128f95abf..18d347662e 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -2636,6 +2636,10 @@ export enum ErrorTypes { * Google provider could not process a linked video (most often longer than the model accepts) */ GOOGLE_VIDEO_UNPROCESSABLE = 'google_video_unprocessable', + /** + * Required CodeAPI resources could not be restored before model invocation. + */ + RESOURCE_RECOVERY_REQUIRED = 'resource_recovery_required', /** * Invalid Agent Provider (excluded by Admin) */