diff --git a/api/app/clients/tools/util/handleTools.js b/api/app/clients/tools/util/handleTools.js index 0d85baf700..d2a5ee02e0 100644 --- a/api/app/clients/tools/util/handleTools.js +++ b/api/app/clients/tools/util/handleTools.js @@ -7,6 +7,7 @@ const { checkToolRolePermission, createSafeUser, createAuthIdentityContext, + selectMCPUpstreamTokenProvider, mcpToolPattern, loadWebSearchAuth, splitMCPToolKey, @@ -638,12 +639,18 @@ const loadTools = async ({ user: options.req?.user, tenantId: getTenantId(), }); - const upstreamTokenProvider = createOpenIDSessionTokenProvider({ - req: options.req, - res: options.res, - user: options.req?.user, - identityContext: oboIdentityContext, - tokenPreference: 'access_token', + const upstreamTokenProviderResolver = options.upstreamTokenProviderResolver; + const upstreamTokenProvider = selectMCPUpstreamTokenProvider({ + upstreamTokenProvider: options.upstreamTokenProvider, + upstreamTokenProviderResolver, + createSessionProvider: () => + createOpenIDSessionTokenProvider({ + req: options.req, + res: options.res, + user: options.req?.user, + identityContext: oboIdentityContext, + tokenPreference: 'access_token', + }), }); for (const [serverName, toolConfigs] of Object.entries(requestedMCPTools)) { @@ -666,6 +673,7 @@ const loadTools = async ({ requestScopedConnections, res: options.res, upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, streamId: options.req?._resumableStreamId || null, jobCreatedAt: options.jobCreatedAt, diff --git a/api/server/controllers/agents/__tests__/openai.spec.js b/api/server/controllers/agents/__tests__/openai.spec.js index e891d6cb44..6fac81d976 100644 --- a/api/server/controllers/agents/__tests__/openai.spec.js +++ b/api/server/controllers/agents/__tests__/openai.spec.js @@ -421,8 +421,9 @@ jest.mock('~/cache', () => ({ jest.mock('~/server/services/ToolService', () => ({ loadAgentTools: jest.fn().mockResolvedValue([]), loadToolsForExecution: jest.fn().mockResolvedValue([]), - isFatalAgentInitializationError: (error) => + isFatalAgentInitializationError: jest.fn((error) => ['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 'resource_recovery_required'].includes(error?.code), + ), })); const mockGetMultiplier = jest.fn().mockReturnValue(1); @@ -630,6 +631,7 @@ describe('OpenAIChatCompletionController', () => { expect.objectContaining({ primaryConfig, rootConfigs: [primaryConfig], + signal: mockExecution.signal, resourceType: ResourceType.REMOTE_AGENT, memoryAvailable: true, }), @@ -1435,6 +1437,9 @@ describe('OpenAIChatCompletionController', () => { await OpenAIChatCompletionController(req, res); expect(res.status).toHaveBeenCalledWith(503); + expect( + require('~/server/services/ToolService').isFatalAgentInitializationError, + ).toHaveBeenCalledWith(toolError, { signal: loadAgentTools.mock.calls.at(-1)[0].signal }); }); it('returns the resource recovery status and code before model invocation', async () => { diff --git a/api/server/controllers/agents/__tests__/responses.unit.spec.js b/api/server/controllers/agents/__tests__/responses.unit.spec.js index b31227f810..45d00b5684 100644 --- a/api/server/controllers/agents/__tests__/responses.unit.spec.js +++ b/api/server/controllers/agents/__tests__/responses.unit.spec.js @@ -462,8 +462,9 @@ jest.mock('@librechat/api', () => ({ jest.mock('~/server/services/ToolService', () => ({ loadAgentTools: jest.fn().mockResolvedValue([]), loadToolsForExecution: jest.fn().mockResolvedValue([]), - isFatalAgentInitializationError: (error) => + isFatalAgentInitializationError: jest.fn((error) => ['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 'resource_recovery_required'].includes(error?.code), + ), })); const mockGetMultiplier = jest.fn().mockReturnValue(1); @@ -840,6 +841,7 @@ describe('createResponse controller', () => { expect.objectContaining({ primaryConfig, rootConfigs: [primaryConfig], + signal: mockExecution.signal, resourceType: ResourceType.REMOTE_AGENT, memoryAvailable: true, }), @@ -920,6 +922,9 @@ describe('createResponse controller', () => { 'server_error', 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', ); + expect( + require('~/server/services/ToolService').isFatalAgentInitializationError, + ).toHaveBeenCalledWith(toolError, { signal: loadAgentTools.mock.calls.at(-1)[0].signal }); }); it('returns the resource recovery status and code before model invocation', async () => { diff --git a/api/server/controllers/agents/openai.js b/api/server/controllers/agents/openai.js index 1c754b3860..6681ba6063 100644 --- a/api/server/controllers/agents/openai.js +++ b/api/server/controllers/agents/openai.js @@ -144,7 +144,7 @@ function createToolLoader({ req, res, signal, definitionsOnly = true }) { streamId: null, // No resumable stream for OpenAI compat }); } catch (error) { - if (isFatalAgentInitializationError(error) || isContentFilterError(error)) { + if (isFatalAgentInitializationError(error, { signal }) || isContentFilterError(error)) { throw error; } logger.error('Error loading tools for agent ' + agentId, getSafeErrorMetadata(error)); @@ -627,6 +627,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { skillStates, defaultActiveOnShare, manualSkills, + signal: execution.signal, }, dbMethods, ); @@ -663,6 +664,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { const discoveryParams = { req, res, + signal: execution.signal, primaryConfig, endpointOption, allowedProviders, diff --git a/api/server/controllers/agents/responses.js b/api/server/controllers/agents/responses.js index 839c5a8a8f..c0b6c8e8b9 100644 --- a/api/server/controllers/agents/responses.js +++ b/api/server/controllers/agents/responses.js @@ -196,7 +196,7 @@ function createToolLoader({ req, res, signal, definitionsOnly = true }) { streamId: null, }); } catch (error) { - if (isFatalAgentInitializationError(error) || isContentFilterError(error)) { + if (isFatalAgentInitializationError(error, { signal }) || isContentFilterError(error)) { throw error; } logger.error('Error loading tools for agent ' + agentId, getSafeErrorMetadata(error)); @@ -883,6 +883,7 @@ const executeResponse = async (envelope, { req, res }) => { skillStates, defaultActiveOnShare, manualSkills, + signal: execution.signal, }, dbMethods, ); @@ -919,6 +920,7 @@ const executeResponse = async (envelope, { req, res }) => { const discoveryParams = { req, res, + signal: execution.signal, primaryConfig, endpointOption, allowedProviders, diff --git a/api/server/services/Endpoints/agents/addedConvo.js b/api/server/services/Endpoints/agents/addedConvo.js index 1764359a21..031e7d5823 100644 --- a/api/server/services/Endpoints/agents/addedConvo.js +++ b/api/server/services/Endpoints/agents/addedConvo.js @@ -66,6 +66,7 @@ const loadAddedAgent = (params) => * `codeEnvAvailable`. @see InitializeAgentParams.fileSearchAvailable. * @param {boolean} [params.statefulSessionsAvailable] - `stateful_code_sessions` * capability flag; forwarded verbatim alongside `codeEnvAvailable`. + * @param {AbortSignal} [params.signal] - Owning run cancellation signal. * @returns {Promise<{userMCPAuthMap: Object|undefined}>} The updated userMCPAuthMap */ const processAddedConvo = async ({ @@ -97,6 +98,7 @@ const processAddedConvo = async ({ toolIntentsAvailable, statefulSessionsAvailable, memoryAvailable, + signal, }) => { const addedConvo = endpointOption.addedConvo; if (addedConvo == null) { @@ -200,6 +202,7 @@ const processAddedConvo = async ({ memoryAvailable, skillStates, defaultActiveOnShare, + signal, }, { getFiles: db.getFiles, @@ -240,7 +243,7 @@ const processAddedConvo = async ({ return { userMCPAuthMap }; } catch (err) { - if (isFatalAgentInitializationError(err)) { + if (isFatalAgentInitializationError(err, { signal })) { throw err; } logger.error('[processAddedConvo] Error processing addedConvo for parallel agent', err); diff --git a/api/server/services/Endpoints/agents/addedConvo.spec.js b/api/server/services/Endpoints/agents/addedConvo.spec.js index c2c6b69121..8224eb4f5d 100644 --- a/api/server/services/Endpoints/agents/addedConvo.spec.js +++ b/api/server/services/Endpoints/agents/addedConvo.spec.js @@ -42,7 +42,8 @@ jest.mock('~/server/services/MCP', () => ({ })); jest.mock('~/server/services/ToolService', () => ({ - isFatalAgentInitializationError: (error) => + isFatalAgentInitializationError: (error, { signal } = {}) => + (signal?.aborted === true && (error === signal.reason || error?.name === 'AbortError')) || ['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 'resource_recovery_required'].includes(error?.code), })); @@ -173,6 +174,19 @@ describe('processAddedConvo', () => { await expect(processAddedConvo(baseParams())).rejects.toBe(toolError); }); + it('forwards and propagates owning-run cancellation from added-agent initialization', async () => { + const controller = new AbortController(); + const reason = new Error('parallel agent stopped'); + controller.abort(reason); + mockInitializeAgent.mockRejectedValueOnce(reason); + + await expect(processAddedConvo(baseParams({ signal: controller.signal }))).rejects.toBe(reason); + expect(mockInitializeAgent).toHaveBeenCalledWith( + expect.objectContaining({ signal: controller.signal }), + expect.anything(), + ); + }); + it('keeps deployment-aware skill metadata on a persisted added-agent config', async () => { const deploymentSkillId = { toString: () => 'deployment-skill' }; const agentConfigs = new Map(); diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js index 326a44b6b1..6c826220c5 100644 --- a/api/server/services/Endpoints/agents/initialize.js +++ b/api/server/services/Endpoints/agents/initialize.js @@ -40,6 +40,7 @@ const { encodeAndFormatAudios, encodeAndFormatVideos, extractFileContext, + createScheduleUpstreamTokenProviderResolver, } = require('@librechat/api'); const { ResourceType, @@ -123,6 +124,8 @@ function createToolLoader( streamId = null, definitionsOnly = false, jobCreatedAt, + upstreamTokenProvider, + upstreamTokenProviderResolver, ) { /** * @param {object} params @@ -164,9 +167,11 @@ function createToolLoader( codeExecutionContext, definitionsOnly, accessibleMcpServerNames, + upstreamTokenProvider, + upstreamTokenProviderResolver, }); } catch (error) { - if (isFatalAgentInitializationError(error) || isContentFilterError(error)) { + if (isFatalAgentInitializationError(error, { signal }) || isContentFilterError(error)) { throw error; } logger.error('Error loading tools for agent ' + agentId, error); @@ -185,8 +190,10 @@ function createToolLoader( * @param {string} [params.checkpointNamespace] Immutable saver-level generation scope * @param {string} [params.foregroundRunId] Canonical response identity for foreground execution * @param {import('@librechat/api').MCPRuntimeRequestBody} [params.requestBody] + * @param {import('@librechat/api').UpstreamTokenProvider} [params.upstreamTokenProvider] + * @param {import('@librechat/api').UpstreamTokenProviderResolver} [params.upstreamTokenProviderResolver] */ -const initializeClient = async ({ +const initializeClientWithProvider = async ({ req, res, signal, @@ -195,6 +202,8 @@ const initializeClient = async ({ checkpointNamespace, foregroundRunId, requestBody, + upstreamTokenProvider, + upstreamTokenProviderResolver, }) => { if (!endpointOption) { throw new Error('Endpoint option not provided'); @@ -463,6 +472,8 @@ const initializeClient = async ({ mcpAvailableTools: ctx.mcpAvailableTools, requestScopedConnections: ctx.requestScopedConnections, userMCPAuthMap: ctx.userMCPAuthMap, + upstreamTokenProvider, + upstreamTokenProviderResolver, tool_resources: ctx.tool_resources, actionsEnabled: ctx.actionsEnabled, accessibleMcpServerNames: ctx.accessibleMcpServerNames, @@ -624,7 +635,16 @@ const initializeClient = async ({ const allowedProviders = new Set(appConfig?.endpoints?.[EModelEndpoint.agents]?.allowedProviders); /** Event-driven mode: only load tool definitions, not full instances */ - const loadTools = createToolLoader(req, res, signal, streamId, true, jobCreatedAt); + const loadTools = createToolLoader( + req, + res, + signal, + streamId, + true, + jobCreatedAt, + upstreamTokenProvider, + upstreamTokenProviderResolver, + ); /** @type {Array} */ const requestFiles = req.body.files ?? []; /** @type {string | undefined} */ @@ -712,6 +732,7 @@ const initializeClient = async ({ skillStates, defaultActiveOnShare, manualSkills, + signal, }, { getFiles: db.getFiles, @@ -760,6 +781,7 @@ const initializeClient = async ({ { req, res, + signal, primaryConfig, agent_ids: primaryConfig.agent_ids, endpointOption, @@ -884,6 +906,7 @@ const initializeClient = async ({ toolIntentsAvailable, statefulSessionsAvailable, memoryAvailable, + signal, }); if (updatedMCPAuthMap) { @@ -1160,7 +1183,7 @@ const initializeClient = async ({ try { return await loading; } catch (error) { - if (isFatalAgentInitializationError(error)) { + if (isFatalAgentInitializationError(error, { signal })) { throw error; } logger.error(`[initializeClient] Error loading subagent metadata ${agentId}:`, error); @@ -1248,7 +1271,16 @@ const initializeClient = async ({ req, res, agent, - loadTools: createToolLoader(req, res, context.signal, streamId, true, jobCreatedAt), + loadTools: createToolLoader( + req, + res, + context.signal, + streamId, + true, + jobCreatedAt, + upstreamTokenProvider, + upstreamTokenProviderResolver, + ), requestFiles, authorizedRunFiles: getAuthorizedRunFileSnapshot({ policy: appConfig.endpoints?.agents?.fileSharing, @@ -1277,6 +1309,7 @@ const initializeClient = async ({ memoryAvailable, skillStates, defaultActiveOnShare, + signal: context.signal, }, { getFiles: db.getFiles, @@ -1461,7 +1494,7 @@ const initializeClient = async ({ graphMemberConfigsById.set(memberId, config); return config; } catch (error) { - if (isFatalAgentInitializationError(error)) { + if (isFatalAgentInitializationError(error, { signal })) { throw error; } logger.error(`[initializeClient] Error initializing graph member ${memberId}:`, error); @@ -1834,4 +1867,25 @@ const initializeClient = async ({ return { client, userMCPAuthMap }; }; -module.exports = { initializeClient }; +/** + * Creates an agent initializer whose host may resolve renewable credentials at + * the execution boundary. The resolver returns a provider closure rather than + * token material so refresh remains owned by the host integration. + * + * @param {object} [dependencies] + * @param {(user: import('@librechat/data-schemas').IUser, options: { signal?: AbortSignal }) => import('@librechat/api').UpstreamTokenProvider | undefined | Promise} [dependencies.resolveUpstreamTokenProvider] + */ +function createInitializeClient(dependencies = {}) { + return async (params) => { + const upstreamTokenProviderResolver = createScheduleUpstreamTokenProviderResolver( + params.req, + dependencies.resolveUpstreamTokenProvider, + params.signal, + ); + return initializeClientWithProvider({ ...params, upstreamTokenProviderResolver }); + }; +} + +const initializeClient = createInitializeClient(); + +module.exports = { createInitializeClient, initializeClient }; diff --git a/api/server/services/Endpoints/agents/initialize.spec.js b/api/server/services/Endpoints/agents/initialize.spec.js index 52241abda3..e76bdbb8c1 100644 --- a/api/server/services/Endpoints/agents/initialize.spec.js +++ b/api/server/services/Endpoints/agents/initialize.spec.js @@ -98,7 +98,7 @@ jest.mock('~/cache', () => ({ logViolation: jest.fn(), })); -const { initializeClient } = require('./initialize'); +const { createInitializeClient, initializeClient } = require('./initialize'); const { processAddedConvo } = require('./addedConvo'); const { getSkillDbMethods, getSkillToolDeps } = require('./skillDeps'); const { loadAgentTools } = require('~/server/services/ToolService'); @@ -182,6 +182,53 @@ describe('initializeClient — processAgent ACL gate', () => { }, }); + it('defers host credential resolution during scheduled agent initialization', async () => { + const upstreamTokenProvider = jest.fn(); + const resolveUpstreamTokenProvider = jest.fn().mockResolvedValue(upstreamTokenProvider); + const hostInitializeClient = createInitializeClient({ resolveUpstreamTokenProvider }); + const req = makeReq(); + req._isScheduledFire = true; + const signal = new AbortController().signal; + mockInitializeAgent.mockImplementationOnce(async ({ loadTools, agent }) => { + await loadTools({ + tools: ['run_query_mcp_warehouse'], + model: agent.model, + agentId: agent.id, + provider: agent.provider, + }); + return makePrimaryConfig([]); + }); + + await hostInitializeClient({ + req, + res: {}, + signal, + endpointOption: makeEndpointOption(), + }); + + expect(resolveUpstreamTokenProvider).not.toHaveBeenCalled(); + const toolLoadParams = loadAgentTools.mock.calls[0][0]; + expect(toolLoadParams.upstreamTokenProvider).toBeUndefined(); + const resolver = toolLoadParams.upstreamTokenProviderResolver; + await expect(resolver({ signal })).resolves.toBe(upstreamTokenProvider); + expect(resolveUpstreamTokenProvider).toHaveBeenCalledWith(req.user, { signal }); + }); + + it('keeps interactive agent initialization independent of the host resolver', async () => { + const resolveUpstreamTokenProvider = jest.fn(); + const hostInitializeClient = createInitializeClient({ resolveUpstreamTokenProvider }); + mockInitializeAgent.mockResolvedValue(makePrimaryConfig([])); + + await hostInitializeClient({ + req: makeReq(), + res: {}, + signal: new AbortController().signal, + endpointOption: makeEndpointOption(), + }); + + expect(resolveUpstreamTokenProvider).not.toHaveBeenCalled(); + }); + it('replaces untrusted artifact route metadata with the executing agent context', async () => { mockInitializeAgent.mockResolvedValue(makePrimaryConfig([])); diff --git a/api/server/services/MCP.js b/api/server/services/MCP.js index baf11c268c..b38ba20d6f 100644 --- a/api/server/services/MCP.js +++ b/api/server/services/MCP.js @@ -736,6 +736,7 @@ function resolveToolCallUserId({ effectiveUser, capturedUser, invocationUserId, * @param {Object} params * @param {ServerResponse} params.res - The Express response object for sending events. * @param {import('@librechat/api').UpstreamTokenProvider} [params.upstreamTokenProvider] - Live upstream-token closure for OBO, built at the request boundary so this layer never receives the raw Express request. + * @param {import('@librechat/api').UpstreamTokenProviderResolver} [params.upstreamTokenProviderResolver] * @param {import('@librechat/api').AuthIdentityContext} [params.oboIdentityContext] - Non-template-visible OBO identity context built from the real request user. * @param {IUser} params.user - The user from the request object. * @param {string} params.serverName @@ -761,6 +762,7 @@ async function reconnectServer({ requestBody, requestScopedConnections, upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, streamId = null, jobCreatedAt, @@ -829,6 +831,7 @@ async function reconnectServer({ requestBody, requestScopedConnections, upstreamTokenProvider, + upstreamTokenProviderResolver, recoveryPolicy, oboIdentityContext, forceNew: true, @@ -859,6 +862,7 @@ async function reconnectServer({ * @param {import('@librechat/api').RequestScopedMCPConnectionStore} [params.requestScopedConnections] * @param {Record>} [params.userMCPAuthMap] * @param {import('@librechat/api').UpstreamTokenProvider} [params.upstreamTokenProvider] - Live upstream-token closure for OBO, built at the request boundary. + * @param {import('@librechat/api').UpstreamTokenProviderResolver} [params.upstreamTokenProviderResolver] * @param {import('@librechat/api').AuthIdentityContext} [params.oboIdentityContext] - Non-template-visible OBO identity context built from the real request user. * @returns { Promise unknown}>> } An object with `_call` method to execute the tool input. */ @@ -876,6 +880,7 @@ async function createMCPTools({ requestBody, requestScopedConnections, upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, streamId = null, jobCreatedAt, @@ -920,6 +925,7 @@ async function createMCPTools({ requestBody, requestScopedConnections, upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, streamId, jobCreatedAt, @@ -959,6 +965,7 @@ async function createMCPTools({ requestBody, requestScopedConnections, upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, config: serverConfig, }); @@ -988,6 +995,7 @@ async function createMCPTools({ * @param {Record>} [params.userMCPAuthMap] * @param {import('@librechat/api').ParsedServerConfig} [params.config] * @param {import('@librechat/api').UpstreamTokenProvider} [params.upstreamTokenProvider] - Live upstream-token closure for OBO, built at the request boundary. + * @param {import('@librechat/api').UpstreamTokenProviderResolver} [params.upstreamTokenProviderResolver] * @param {import('@librechat/api').AuthIdentityContext} [params.oboIdentityContext] - Non-template-visible OBO identity context built from the real request user. * @param {string} [params.serverName] - Resolved raw MCP server name from tool loading. * @param {(availableTools: LCAvailableTools) => void} [params.onAvailableTools] @@ -1009,6 +1017,7 @@ async function createMCPTool({ config, configServers, upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, serverName: resolvedServerName, onAvailableTools, @@ -1141,6 +1150,7 @@ async function createMCPTool({ requestBody, requestScopedConnections, upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, streamId, jobCreatedAt, @@ -1186,6 +1196,7 @@ async function createMCPTool({ serverConfig, toolDefinition: toolEntry['function'], upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, streamId, jobCreatedAt, @@ -1207,6 +1218,7 @@ function createToolInstance({ toolDefinition, provider: capturedProvider, upstreamTokenProvider: capturedUpstreamTokenProvider = null, + upstreamTokenProviderResolver: capturedUpstreamTokenProviderResolver = null, oboIdentityContext: capturedOboIdentityContext = null, streamId = null, jobCreatedAt, @@ -1337,6 +1349,7 @@ function createToolInstance({ oboTokenResolver: exchangeOboToken, oboTrustChecker: createOboTrustChecker(), upstreamTokenProvider: capturedUpstreamTokenProvider, + upstreamTokenProviderResolver: capturedUpstreamTokenProviderResolver, oboIdentityContext: capturedOboIdentityContext, }); @@ -1350,10 +1363,14 @@ function createToolInstance({ * alerts; the wrapping below still reports it to the turn. The error has * to look like an abort as well: a permission, OAuth, or upstream failure * can reject in the same tick as the Stop and must stay visible. */ - if (config?.signal?.aborted === true && isAbortError(error)) { + if ( + config?.signal?.aborted === true && + (isAbortError(error) || error === config.signal.reason) + ) { logger.debug( `[MCP][${serverName}][${toolName}][User: ${userId}] Tool call cancelled by user abort`, ); + throw error; } else { logger.error( `[MCP][${serverName}][${toolName}][User: ${userId}] Error calling MCP tool:`, diff --git a/api/server/services/MCP.spec.js b/api/server/services/MCP.spec.js index 8df6e52594..b55ecb9a4c 100644 --- a/api/server/services/MCP.spec.js +++ b/api/server/services/MCP.spec.js @@ -1684,6 +1684,51 @@ describe('User parameter passing tests', () => { ); }); + it('preserves AbortError from the MCP tool implementation', async () => { + const mockUser = { id: 'cancel-user', role: 'USER' }; + const abortController = new AbortController(); + const abort = Object.assign(new Error('Stopped'), { name: 'AbortError' }); + const { getRoleByName } = require('~/models'); + getRoleByName.mockResolvedValue({ + permissions: { + [PermissionTypes.MCP_SERVERS]: { + [Permissions.USE]: true, + }, + }, + }); + mockGetMCPManager.mockReturnValue({ + callTool: jest.fn(async () => { + abortController.abort(abort); + throw abort; + }), + }); + + const mcpTool = await createMCPTool({ + user: mockUser, + config: { url: 'https://cancel.example.com/mcp' }, + toolKey: `test-tool${D}test-server`, + provider: 'openai', + userMCPAuthMap: {}, + availableTools: { + [`test-tool${D}test-server`]: { + function: { + description: 'Cached tool', + parameters: { type: 'object', properties: {} }, + }, + }, + }, + }); + + await expect( + mcpTool.func({}, undefined, { + signal: abortController.signal, + configurable: { user: mockUser }, + metadata: { provider: 'openai', thread_id: 'thread-1', run_id: 'run-1' }, + toolCall: {}, + }), + ).rejects.toBe(abort); + }); + it('keeps a real failure racing the Stop at error level', async () => { const mockUser = { id: 'race-user', role: 'USER' }; const mockRes = { write: jest.fn(), flush: jest.fn() }; diff --git a/api/server/services/Schedules/mcp.js b/api/server/services/Schedules/mcp.js index c699936159..e87a8909f9 100644 --- a/api/server/services/Schedules/mcp.js +++ b/api/server/services/Schedules/mcp.js @@ -9,30 +9,44 @@ const { createOboTrustChecker } = require('~/server/services/OboPolicyService'); const { getLogStores } = require('~/cache'); const methods = require('~/models'); -module.exports = createScheduleMCPPreflight({ - getRoleByName: methods.getRoleByName, - resolveAgentGraphAccess: methods.resolveAgentGraphAccess, - getAgentGraphNodes: methods.getAgentGraphNodes, - getModelsConfig: (user) => getModelsConfig({ user }), - getUser: (id) => methods.findUser({ _id: id }), - getAppConfig, - ensureConfigServers: (config, limit) => - getMCPServersRegistry().ensureConfigServers(config, limit), - getServerConfigs: (userId, config, role) => - getMCPServersRegistry().getAllServerConfigs(userId, config, role), - findPluginAuthsByKeys: methods.findPluginAuthsByKeys, - connect: (options) => - getMCPManager().getConnection({ - ...options, - flowManager: getFlowStateManager(getLogStores(CacheKeys.FLOWS)), - tokenMethods: { - findToken: methods.findToken, - updateToken: methods.updateToken, - createToken: methods.createToken, - deleteTokens: methods.deleteTokens, - }, - graphTokenResolver: getGraphApiToken, - oboTokenResolver: exchangeOboToken, - oboTrustChecker: createOboTrustChecker(), - }), -}); +/** + * Builds the schedule MCP preflight with an optional host-owned renewable + * credential resolver. The default application has no durable upstream + * credential source and therefore remains fail-closed for unattended OBO. + * + * @param {object} [options] + * @param {(user: import('@librechat/data-schemas').IUser, options: { signal?: AbortSignal }) => import('@librechat/api').UpstreamTokenProvider | undefined | Promise} [options.resolveUpstreamTokenProvider] + */ +function createMCPPreflight(options = {}) { + return createScheduleMCPPreflight({ + getRoleByName: methods.getRoleByName, + resolveAgentGraphAccess: methods.resolveAgentGraphAccess, + getAgentGraphNodes: methods.getAgentGraphNodes, + getModelsConfig: (user) => getModelsConfig({ user }), + getUser: (id) => methods.findUser({ _id: id }), + getAppConfig, + ensureConfigServers: (config, limit) => + getMCPServersRegistry().ensureConfigServers(config, limit), + getServerConfigs: (userId, config, role) => + getMCPServersRegistry().getAllServerConfigs(userId, config, role), + findPluginAuthsByKeys: methods.findPluginAuthsByKeys, + resolveUpstreamTokenProvider: options.resolveUpstreamTokenProvider, + connect: (connectionOptions) => + getMCPManager().getConnection({ + ...connectionOptions, + flowManager: getFlowStateManager(getLogStores(CacheKeys.FLOWS)), + tokenMethods: { + findToken: methods.findToken, + updateToken: methods.updateToken, + createToken: methods.createToken, + deleteTokens: methods.deleteTokens, + }, + graphTokenResolver: getGraphApiToken, + oboTokenResolver: exchangeOboToken, + oboTrustChecker: createOboTrustChecker(), + }), + }); +} + +module.exports = createMCPPreflight(); +module.exports.createMCPPreflight = createMCPPreflight; diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.js index 4952ee774c..6994c106d3 100644 --- a/api/server/services/ToolService.js +++ b/api/server/services/ToolService.js @@ -12,6 +12,7 @@ const { getToolkitKey, getUserMCPAuthMap, createAuthIdentityContext, + selectMCPUpstreamTokenProvider, loadToolDefinitions, GenerationJobManager, isActionDomainAllowed, @@ -783,6 +784,8 @@ const isBuiltInTool = (toolName) => * @param {string|null} [params.streamId] - Stream ID for resumable mode * @param {number} [params.jobCreatedAt] - The generation epoch that owns emitted tool events * @param {AbortSignal} [params.signal] - Effective run cancellation signal + * @param {import('@librechat/api').UpstreamTokenProvider} [params.upstreamTokenProvider] + * @param {import('@librechat/api').UpstreamTokenProviderResolver} [params.upstreamTokenProviderResolver] * @returns {Promise<{ * toolDefinitions?: import('@librechat/api').LCTool[]; * toolRegistry?: Map; @@ -803,6 +806,8 @@ async function loadToolDefinitionsWrapper({ codeExecutionContext, accessibleMcpServerNames, signal, + upstreamTokenProvider: suppliedUpstreamTokenProvider, + upstreamTokenProviderResolver, }) { if (!agent.tools || agent.tools.length === 0) { return { toolDefinitions: [] }; @@ -988,22 +993,21 @@ async function loadToolDefinitionsWrapper({ /** @type {Record} */ const mcpAvailableTools = {}; const requestScopedConnections = getMCPRequestContext(req, res); - /** - * Build the OBO upstream-token closure once at this request boundary and pass - * the function into MCP handling, so `reinitMCPServer` never receives the raw - * Express request. `res` is forwarded so a rotated refresh token can be - * mirrored to the `refreshToken` cookie when the response is still writable. - */ const oboIdentityContext = createAuthIdentityContext({ user: req.user, tenantId: getTenantId(), }); - const upstreamTokenProvider = createOpenIDSessionTokenProvider({ - req, - res, - user: req.user, - identityContext: oboIdentityContext, - tokenPreference: 'access_token', + const upstreamTokenProvider = selectMCPUpstreamTokenProvider({ + upstreamTokenProvider: suppliedUpstreamTokenProvider, + upstreamTokenProviderResolver, + createSessionProvider: () => + createOpenIDSessionTokenProvider({ + req, + res, + user: req.user, + identityContext: oboIdentityContext, + tokenPreference: 'access_token', + }), }); const rememberMCPAvailableTools = (serverName, availableTools) => { if (!availableTools || Object.keys(availableTools).length === 0) { @@ -1182,6 +1186,7 @@ async function loadToolDefinitionsWrapper({ }; const result = await reinitMCPServer({ + signal, user: req.user, oauthStart, flowManager, @@ -1191,6 +1196,7 @@ async function loadToolDefinitionsWrapper({ requestBody: runtimeRequestBody, requestScopedConnections, upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, recoveryPolicy: appConfig?.mcpSettings?.catalogRecovery, }); @@ -1211,6 +1217,7 @@ async function loadToolDefinitionsWrapper({ } }; const result = await reinitMCPServer({ + signal, user: req.user, forceNew: true, oauthStart, @@ -1221,6 +1228,7 @@ async function loadToolDefinitionsWrapper({ requestBody: runtimeRequestBody, requestScopedConnections, upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, recoveryPolicy: appConfig?.mcpSettings?.catalogRecovery, }); @@ -1361,6 +1369,7 @@ async function loadToolDefinitionsWrapper({ } const result = await reinitMCPServer({ + signal, user: req.user, serverName, configServers, @@ -1372,6 +1381,7 @@ async function loadToolDefinitionsWrapper({ oauthEnd: createOAuthEndEmitter(serverName), connectionTimeout: Time.TWO_MINUTES, upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, recoveryPolicy: appConfig?.mcpSettings?.catalogRecovery, }); @@ -1382,16 +1392,19 @@ async function loadToolDefinitionsWrapper({ return { serverName, success: true }; } return { serverName, success: false }; - } catch { + } catch (error) { + if (isFatalAgentInitializationError(error, { signal })) { + throw error; + } logger.debug('[Tool Definitions] MCP OAuth wait failed for one server'); return { serverName, success: false }; } }); - const results = await Promise.allSettled(oauthWaitPromises); + const results = await Promise.all(oauthWaitPromises); const successfulServers = results - .filter((r) => r.status === 'fulfilled' && r.value.success) - .map((r) => r.value.serverName); + .filter((result) => result.success) + .map((result) => result.serverName); if (successfulServers.length > 0) { logger.info( @@ -1477,7 +1490,7 @@ async function loadToolDefinitionsWrapper({ primedCodeFiles = files; } } catch (error) { - if (isFatalAgentInitializationError(error)) { + if (isFatalAgentInitializationError(error, { signal })) { throw error; } logger.error( @@ -1567,6 +1580,8 @@ async function loadToolDefinitionsWrapper({ * @param {boolean} [params.definitionsOnly=true] - When true, returns only serializable * tool definitions without creating full tool instances. Use for event-driven mode * where tools are loaded on-demand during execution. + * @param {import('@librechat/api').UpstreamTokenProvider} [params.upstreamTokenProvider] + * @param {import('@librechat/api').UpstreamTokenProviderResolver} [params.upstreamTokenProviderResolver] */ async function loadAgentTools({ req, @@ -1582,6 +1597,8 @@ async function loadAgentTools({ definitionsOnly = true, codeExecutionContext: providedCodeExecutionContext, accessibleMcpServerNames, + upstreamTokenProvider, + upstreamTokenProviderResolver, }) { if (definitionsOnly) { try { @@ -1597,10 +1614,12 @@ async function loadAgentTools({ codeExecutionContext: providedCodeExecutionContext, accessibleMcpServerNames, signal, + upstreamTokenProvider, + upstreamTokenProviderResolver, }); } catch (error) { if ( - isFatalAgentInitializationError(error) || + isFatalAgentInitializationError(error, { signal }) || isContentFilterError(error) || !agent.tools?.some(isExpectedMCPTool) ) { @@ -1759,6 +1778,8 @@ async function loadAgentTools({ returnMetadata: true, mcpPermissionContext, requestScopedConnections: getMCPRequestContext(req, res), + upstreamTokenProvider, + upstreamTokenProviderResolver, codeExecutionContext, [Tools.web_search]: webSearchCallbacks, }, @@ -2023,6 +2044,8 @@ async function loadAgentTools({ * @param {Record} [params.mcpAvailableTools] - Run-scoped MCP tool definitions * @param {import('@librechat/api').RequestScopedMCPConnectionStore} [params.requestScopedConnections] - Run-scoped MCP connections * @param {Record>} [params.userMCPAuthMap] - User MCP auth map + * @param {import('@librechat/api').UpstreamTokenProvider} [params.upstreamTokenProvider] + * @param {import('@librechat/api').UpstreamTokenProviderResolver} [params.upstreamTokenProviderResolver] * @param {Object} [params.tool_resources] - Tool resources * @param {string|null} [params.streamId] - Stream ID for web search callbacks * @param {number} [params.jobCreatedAt] - The generation epoch that owns emitted tool events @@ -2046,6 +2069,8 @@ async function loadToolsForExecution({ mcpAvailableTools, requestScopedConnections, userMCPAuthMap, + upstreamTokenProvider, + upstreamTokenProviderResolver, tool_resources, streamId = null, jobCreatedAt, @@ -2355,6 +2380,8 @@ async function loadToolsForExecution({ * turn already advertised. */ accessibleMcpServerNames, requestScopedConnections: mcpRequestScopedConnections, + upstreamTokenProvider, + upstreamTokenProviderResolver, [Tools.web_search]: webSearchCallbacks, }, webSearch: appConfig?.webSearch, diff --git a/api/server/services/Tools/mcp.js b/api/server/services/Tools/mcp.js index 52dd1a362b..aea58c8728 100644 --- a/api/server/services/Tools/mcp.js +++ b/api/server/services/Tools/mcp.js @@ -7,9 +7,7 @@ const { resolveMCPReinitializeConfig, requiresEphemeralUserConnection, getMissingRuntimeBodyPlaceholderFields, - MCPAuthenticationRejectedError, - MCPAuthenticationRefreshError, - OpenIDReauthRequiredError, + isMCPInitializationError, prepareMCPAuthorizationMutation, } = require('@librechat/api'); const { CacheKeys, Constants } = require('librechat-data-provider'); @@ -43,16 +41,12 @@ const MCP_REINITIALIZE_FAILURE_REASONS = { INITIALIZATION_FAILED: 'initialization_failed', }; -const isMCPReauthenticationError = (error) => - error instanceof MCPAuthenticationRejectedError || - error instanceof MCPAuthenticationRefreshError || - error instanceof OpenIDReauthRequiredError; - /** Wires application dependencies into the passive, request-local catalog recovery service. * @param {Object} params * @param {IUser} params.user * @param {Array<{ serverName: string, serverConfig: object }>} params.servers * @param {import('@librechat/api').UpstreamTokenProvider} [params.upstreamTokenProvider] - Live upstream-token closure for OBO discovery, built at the request boundary so this layer never receives the raw Express request. + * @param {import('@librechat/api').UpstreamTokenProviderResolver} [params.upstreamTokenProviderResolver] * @param {import('@librechat/api').AuthIdentityContext} [params.oboIdentityContext] - Non-template-visible OBO identity context built from the real request user. * @param {AbortSignal} [params.signal] - Cancels queued and in-flight catalog reads when the request ends. * @param {import('@librechat/api').MCPServerCatalogRecoveryPolicy} [params.recoveryPolicy] @@ -61,6 +55,7 @@ async function loadMCPServerCatalogs({ user, servers, upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, signal, recoveryPolicy, @@ -96,6 +91,7 @@ async function loadMCPServerCatalogs({ oboTokenResolver: exchangeOboToken, oboTrustChecker: createOboTrustChecker(), upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, }), onOAuthCredentialsChanging, @@ -117,6 +113,7 @@ async function loadMCPServerCatalogs({ * @param {Object} params * @param {IUser} params.user - The user from the request object. * @param {import('@librechat/api').UpstreamTokenProvider} [params.upstreamTokenProvider] - Live upstream-token closure for OBO connection establishment, built at the request boundary so this layer never receives the raw Express request. + * @param {import('@librechat/api').UpstreamTokenProviderResolver} [params.upstreamTokenProviderResolver] * @param {import('@librechat/api').AuthIdentityContext} [params.oboIdentityContext] - Non-template-visible OBO identity context built from the real request user. * @param {string} params.serverName - The name of the MCP server * @param {boolean} params.returnOnOAuth - Whether to initiate OAuth and return, or wait for OAuth flow to finish @@ -146,6 +143,7 @@ async function reinitMCPServer({ requestBody, requestScopedConnections, upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, oauthEnd, recoveryPolicy, @@ -281,12 +279,13 @@ async function reinitMCPServer({ oboTokenResolver: exchangeOboToken, oboTrustChecker: createOboTrustChecker(), upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, }); logger.info('[MCP Reinitialize] Successfully established connection'); } catch (err) { - if (isMCPReauthenticationError(err)) { + if (isMCPInitializationError(err, signal)) { throw err; } logger.info('[MCP Reinitialize] Connection attempt failed'); @@ -322,6 +321,7 @@ async function reinitMCPServer({ oboTokenResolver: exchangeOboToken, oboTrustChecker: createOboTrustChecker(), upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, }); @@ -332,7 +332,7 @@ async function reinitMCPServer({ ); } } catch (error) { - if (isMCPReauthenticationError(error)) { + if (isMCPInitializationError(error, signal)) { throw error; } logger.debug('[MCP Reinitialize] Tool discovery failed'); @@ -347,9 +347,9 @@ async function reinitMCPServer({ mcpManager.getToolPublicationGeneration(connection) ?? publicationGeneration; let snapshot; if (typeof connection.fetchOrderedToolsSnapshot === 'function') { - snapshot = await connection.fetchOrderedToolsSnapshot(); + snapshot = await connection.fetchOrderedToolsSnapshot(undefined, signal); } else if (typeof connection.fetchToolsSnapshot === 'function') { - snapshot = await connection.fetchToolsSnapshot(); + snapshot = await connection.fetchToolsSnapshot(undefined, signal); } else { snapshot = { tools: await connection.fetchTools(), complete: true }; } @@ -451,7 +451,7 @@ async function reinitMCPServer({ return result; } catch (error) { - if (isMCPReauthenticationError(error)) { + if (isMCPInitializationError(error, signal)) { throw error; } logger.error('[MCP Reinitialize] Error loading MCP tools; servers may still be initializing'); diff --git a/api/server/services/Tools/mcp.spec.js b/api/server/services/Tools/mcp.spec.js index dfd222f25f..bc50be0fcc 100644 --- a/api/server/services/Tools/mcp.spec.js +++ b/api/server/services/Tools/mcp.spec.js @@ -314,6 +314,7 @@ describe('reinitMCPServer — customUserVars gating (issue #10969)', () => { }); it('preserves cached tools when live recovery returns an incomplete snapshot', async () => { + const signal = new AbortController().signal; const fetchOrderedToolsSnapshot = jest.fn().mockResolvedValue({ tools: [{ name: 'partial', inputSchema: { type: 'object' } }], complete: false, @@ -325,11 +326,13 @@ describe('reinitMCPServer — customUserVars gating (issue #10969)', () => { const result = await reinitMCPServer({ user, serverName, + signal, serverConfig: { type: 'streamable-http', url: 'https://thingy.example.com/mcp' }, }); expect(result.tools).toBeNull(); expect(fetchOrderedToolsSnapshot).toHaveBeenCalledTimes(1); + expect(fetchOrderedToolsSnapshot).toHaveBeenCalledWith(undefined, signal); expect(mockUpdateMCPServerTools).not.toHaveBeenCalled(); }); @@ -552,6 +555,47 @@ describe('reinitMCPServer — direct bearer authentication outcomes', () => { }), ).rejects.toBe(rejection); }); + + it('propagates cancellation instead of hiding the server tool', async () => { + const abort = new DOMException('Stopped', 'AbortError'); + const controller = new AbortController(); + controller.abort(abort); + mockGetConnection.mockRejectedValue(abort); + await expect( + reinitMCPServer({ + user: { id: 'user-123' }, + serverName: 'example-mcp', + signal: controller.signal, + serverConfig: { type: 'streamable-http', url: 'https://mcp.example.com', source: 'yaml' }, + }), + ).rejects.toBe(abort); + }); + + it.each([false, true])( + 'preserves a typed OBO resolution failure (retryable=%s)', + async (retryable) => { + const { OboTokenResolutionError } = require('@librechat/api'); + const rejection = new OboTokenResolutionError( + 'session_refresh_failed', + 'Sign-in expired.', + retryable, + ); + mockGetConnection.mockRejectedValue(rejection); + + await expect( + reinitMCPServer({ + user: { id: 'user-123' }, + serverName: 'private-mcp', + serverConfig: { + type: 'streamable-http', + url: 'https://mcp.example.com', + source: 'yaml', + obo: { scopes: 'api://mcp/.default' }, + }, + }), + ).rejects.toBe(rejection); + }, + ); }); describe('reinitMCPServer — runtime BODY placeholder pre-check (issue #14074)', () => { diff --git a/api/server/services/__tests__/ToolService.spec.js b/api/server/services/__tests__/ToolService.spec.js index 1c15094b4e..280e7d7572 100644 --- a/api/server/services/__tests__/ToolService.spec.js +++ b/api/server/services/__tests__/ToolService.spec.js @@ -94,8 +94,17 @@ const mockGetUserMCPAuthMap = jest.fn(); jest.mock('@librechat/api', () => ({ ...jest.requireActual('@librechat/api'), AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE: 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', - isFatalAgentInitializationError: (error) => + isFatalAgentInitializationError: (error, { signal } = {}) => + (signal?.aborted === true && (error === signal.reason || error?.name === 'AbortError')) || ['AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', 'resource_recovery_required'].includes(error?.code), + selectMCPUpstreamTokenProvider: ({ + upstreamTokenProvider, + upstreamTokenProviderResolver, + createSessionProvider, + }) => + upstreamTokenProviderResolver + ? upstreamTokenProvider + : (upstreamTokenProvider ?? createSessionProvider()), loadToolDefinitions: (...args) => mockLoadToolDefinitions(...args), getUserMCPAuthMap: (...args) => mockGetUserMCPAuthMap(...args), createAuthIdentityContext: ({ user, tenantId }) => ({ @@ -191,7 +200,6 @@ jest.mock('~/server/services/MCP', () => ({ jest.mock('~/cache', () => ({ getLogStores: jest.fn(() => ({})), })); - const { loadAgentTools, loadToolsForExecution, @@ -1922,6 +1930,55 @@ describe('ToolService - Action Capability Gating', () => { ); }); + it('propagates owning-run cancellation from the pending OAuth fan-out', async () => { + const serverName = 'Google-Workspace'; + const authorizationUrl = 'https://auth.example.com/Google-Workspace'; + const mcpTool = `${Constants.mcp_all}${Constants.mcp_delimiter}${serverName}`; + const capabilities = [AgentCapabilities.tools]; + const req = createMockReq(capabilities); + const res = { writableEnded: false }; + const controller = new AbortController(); + const reason = new Error('generation stopped'); + mockGetEndpointsConfig.mockResolvedValue(createEndpointsConfig(capabilities)); + mockGetServerConfig.mockResolvedValue({ + type: 'streamable-http', + url: 'https://demo.librechat.ai/mcp', + requiresOAuth: true, + }); + mockGetMCPServerTools.mockResolvedValue(null); + mockFlowManager.getFlowState.mockResolvedValue({ + status: 'PENDING', + createdAt: Date.now(), + metadata: { authorizationUrl }, + }); + mockLoadToolDefinitions.mockImplementation(async (params, deps) => { + await deps.getOrFetchMCPServerTools(params.userId, serverName); + return { + toolDefinitions: [], + toolRegistry: new Map(), + hasDeferredTools: false, + }; + }); + reinitMCPServer.mockImplementation( + ({ signal }) => + new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(signal.reason), { once: true }); + }), + ); + + const loading = loadAgentTools({ + req, + res, + signal: controller.signal, + agent: { id: 'agent_123', tools: [mcpTool] }, + definitionsOnly: true, + }); + await new Promise((resolve) => setImmediate(resolve)); + controller.abort(reason); + + await expect(loading).rejects.toBe(reason); + }); + it('should re-emit pending MCP OAuth prompts when selected MCP tools are already concrete', async () => { const serverName = `Google${Constants.mcp_delimiter}Workspace`; const authorizationUrl = 'https://auth.example.com/Google-Workspace'; @@ -2175,6 +2232,96 @@ describe('ToolService - Action Capability Gating', () => { ); }); + it('uses host-supplied renewable upstream credentials when loading tools', async () => { + const serverName = 'Scheduled-OBO'; + const mcpTool = `search${Constants.mcp_delimiter}${serverName}`; + const capabilities = [AgentCapabilities.tools]; + const req = createMockReq(capabilities); + req.body = { conversationId: 'conv-123', messageId: 'msg-123' }; + req.user = { + id: 'user_123', + provider: 'openid', + openidId: 'oidc-sub-123', + tenantId: 'tenant-1', + openidIssuer: 'https://issuer.example.com', + }; + const scheduledProvider = jest.fn().mockResolvedValue({ access_token: 'renewed-token' }); + + mockGetEndpointsConfig.mockResolvedValue(createEndpointsConfig(capabilities)); + mockGetServerConfig.mockResolvedValue({ + type: 'streamable-http', + url: 'https://mcp.example.com/obo', + obo: { scopes: 'api://obo/Mcp.Tools.ReadWrite' }, + }); + mockLoadToolDefinitions.mockImplementation(async (params, dependencies) => { + await dependencies.refreshMCPServerTools(params.userId, serverName); + return { + toolDefinitions: [], + toolRegistry: new Map(), + hasDeferredTools: false, + }; + }); + reinitMCPServer.mockResolvedValue({ availableTools: {} }); + const signal = new AbortController().signal; + + await loadAgentTools({ + req, + agent: { id: 'agent_123', tools: [mcpTool] }, + definitionsOnly: true, + signal, + upstreamTokenProvider: scheduledProvider, + }); + + expect(reinitMCPServer).toHaveBeenCalledWith( + expect.objectContaining({ + serverName, + forceNew: true, + upstreamTokenProvider: scheduledProvider, + }), + ); + }); + + it('keeps a deferred OBO resolver separate from direct-bearer credentials', async () => { + const serverName = 'Scheduled-OBO'; + const mcpTool = `search${Constants.mcp_delimiter}${serverName}`; + const capabilities = [AgentCapabilities.tools]; + const req = createMockReq(capabilities); + const upstreamTokenProviderResolver = jest.fn(); + + mockGetEndpointsConfig.mockResolvedValue(createEndpointsConfig(capabilities)); + mockGetServerConfig.mockResolvedValue({ + type: 'streamable-http', + url: 'https://mcp.example.com/obo', + obo: { scopes: 'api://obo/Mcp.Tools.ReadWrite' }, + }); + mockLoadToolDefinitions.mockImplementation(async (params, dependencies) => { + await dependencies.refreshMCPServerTools(params.userId, serverName); + return { + toolDefinitions: [], + toolRegistry: new Map(), + hasDeferredTools: false, + }; + }); + reinitMCPServer.mockResolvedValue({ availableTools: {} }); + + await loadAgentTools({ + req, + agent: { id: 'agent_123', tools: [mcpTool] }, + definitionsOnly: true, + upstreamTokenProviderResolver, + }); + + expect(reinitMCPServer).toHaveBeenCalledWith( + expect.objectContaining({ + serverName, + forceNew: true, + upstreamTokenProvider: undefined, + upstreamTokenProviderResolver, + }), + ); + expect(upstreamTokenProviderResolver).not.toHaveBeenCalled(); + }); + it('returns run-scoped MCP tool definitions for request-scoped servers', async () => { const serverName = 'ClickHouse'; const mcpTool = `list_tables${Constants.mcp_delimiter}${serverName}`; diff --git a/packages/api/src/agents/__tests__/initialize.test.ts b/packages/api/src/agents/__tests__/initialize.test.ts index c17a5076be..7ee63434de 100644 --- a/packages/api/src/agents/__tests__/initialize.test.ts +++ b/packages/api/src/agents/__tests__/initialize.test.ts @@ -2236,6 +2236,49 @@ describe('initializeAgent — skill `allowed-tools` union (Phase 6)', () => { expect(definedNames).not.toContain('mcp__broken__tool'); }); + it.each([ + ['Error', new Error('run cancelled')], + ['string', 'run cancelled'], + ])( + 'does not retry skill-added tools when the owning signal aborts with an %s reason', + async (_, reason) => { + const { agent, req, res, loadTools, db } = createMocks(); + agent.tools = ['web_search']; + const { Types } = await import('mongoose'); + const skillId = new Types.ObjectId(); + const controller = new AbortController(); + controller.abort(reason); + loadTools.mockRejectedValue(reason); + + const getSkillByName = buildGetSkillByName( + 'cancelled-tool-skill', + ['mcp__warehouse__query'], + skillId, + req.user!.id, + ); + + await expect( + initializeAgent( + { + req, + res, + agent, + loadTools, + signal: controller.signal, + endpointOption: { endpoint: EModelEndpoint.agents }, + allowedProviders: new Set([Providers.OPENAI]), + isInitialAgent: true, + accessibleSkillIds: [skillId], + manualSkills: ['cancelled-tool-skill'], + }, + { ...db, listSkillsByAccess: emptyListSkillsByAccess, getSkillByName }, + ), + ).rejects.toBe(reason); + + expect(loadTools).toHaveBeenCalledTimes(1); + }, + ); + it('does not retry a resource recovery failure when execute_code is skill-added', async () => { const { agent, req, res, loadTools, db } = createMocks(); agent.tools = ['web_search']; diff --git a/packages/api/src/agents/discovery.spec.ts b/packages/api/src/agents/discovery.spec.ts index 1c7c0684ed..16a19f5f4a 100644 --- a/packages/api/src/agents/discovery.spec.ts +++ b/packages/api/src/agents/discovery.spec.ts @@ -1205,6 +1205,90 @@ describe('discoverConnectedAgents', () => { }, ); + it.each([ + ['handoff', [{ from: 'A', to: 'B', edgeType: 'handoff' as const }], undefined], + ['legacy chain', undefined, ['B']], + ])('propagates owning-run cancellation during %s discovery', async (_case, edges, agentIds) => { + const controller = new AbortController(); + controller.abort(); + + await expect( + discoverConnectedAgents( + { + req: makeReq(), + res: makeRes(), + signal: controller.signal, + primaryConfig: makeConfig('A', edges), + agent_ids: agentIds, + allowedProviders: new Set(), + modelsConfig: { openai: ['gpt-4o'] }, + loadTools: jest.fn(), + }, + { + getAgent: jest.fn(async () => makeAgent('B', [])), + checkPermission: jest.fn().mockResolvedValue(true), + logViolation: jest.fn(), + db: {} as never, + }, + ), + ).rejects.toBe(controller.signal.reason); + }); + + it.each(['getAgent', 'checkPermission', 'validateAgentModel', 'initializeAgent'] as const)( + 'detaches promptly when cancellation occurs during %s', + async (stage) => { + const controller = new AbortController(); + const reason = new Error(`stopped during ${stage}`); + let settlePending!: (value: unknown) => void; + const pending = new Promise((resolve) => { + settlePending = resolve; + }); + const getAgent = jest.fn(async () => makeAgent('B', [])); + const checkPermission = jest.fn().mockResolvedValue(true); + + if (stage === 'getAgent') { + getAgent.mockReturnValueOnce(pending as Promise); + } else if (stage === 'checkPermission') { + checkPermission.mockReturnValueOnce(pending as Promise); + } else if (stage === 'validateAgentModel') { + mockValidateAgentModel.mockReturnValueOnce(pending as Promise<{ isValid: boolean }>); + } else { + mockInitializeAgent.mockReturnValueOnce(pending as Promise); + } + + const discovery = discoverConnectedAgents( + { + req: makeReq(), + res: makeRes(), + signal: controller.signal, + primaryConfig: makeConfig('A', [{ from: 'A', to: 'B', edgeType: 'handoff' as const }]), + allowedProviders: new Set(), + modelsConfig: { openai: ['gpt-4o'] }, + loadTools: jest.fn(), + }, + { + getAgent, + checkPermission, + logViolation: jest.fn(), + db: {} as never, + }, + ); + await new Promise((resolve) => setImmediate(resolve)); + controller.abort(reason); + + await expect(discovery).rejects.toBe(reason); + if (stage === 'getAgent') { + settlePending(makeAgent('B', [])); + } else if (stage === 'checkPermission') { + settlePending(true); + } else if (stage === 'validateAgentModel') { + settlePending({ isValid: true }); + } else { + settlePending(makeConfig('B')); + } + }, + ); + it('skips when request has no authenticated user', async () => { const primaryConfig = makeConfig('A', [{ from: 'A', to: 'B', edgeType: 'handoff' }]); @@ -1310,6 +1394,48 @@ describe('resolveSubagentGraphs', () => { }); }); + it('propagates owning-run cancellation while resolving a graph member', async () => { + const controller = new AbortController(); + controller.abort(); + const primaryConfig = makeConfig('A') as GraphSubagentHostConfig; + primaryConfig.subagents = { + enabled: true, + graphs: [ + { + type: 'team', + name: 'Team', + description: 'A remote graph team', + agent_ids: ['A', 'B'], + edges: [{ from: 'A', to: 'B', edgeType: 'direct' }], + entry_agent_id: 'A', + result_agent_id: 'B', + }, + ], + }; + + await expect( + resolveSubagentGraphs( + { + req: makeReq(), + res: makeRes(), + signal: controller.signal, + primaryConfig, + rootConfigs: [primaryConfig], + allowedProviders: new Set(), + modelsConfig: { openai: ['gpt-4o'] }, + loadTools: jest.fn(), + resourceType: 'remote_agent', + }, + { + getAgent: jest.fn(async ({ id }: { id: string }) => makeAgent(id)), + checkPermission: jest.fn().mockResolvedValue(true), + logViolation: jest.fn(), + db: {} as never, + }, + ), + ).rejects.toBe(controller.signal.reason); + }); + it('does not charge initialized root members against the graph load budget', async () => { const rootConfigs = Array.from({ length: MAX_SUBAGENT_GRAPH_NODES }, (_, index) => { const config = makeConfig(`root_${index}`) as GraphSubagentHostConfig; diff --git a/packages/api/src/agents/discovery.ts b/packages/api/src/agents/discovery.ts index a931ac192c..5ef00a003f 100644 --- a/packages/api/src/agents/discovery.ts +++ b/packages/api/src/agents/discovery.ts @@ -26,6 +26,7 @@ import { createEdgeCollector, resolveReachableGraph } from './edges'; import { isFatalAgentInitializationError } from './errors'; import { createConcurrencyLimiter } from '~/utils/promise'; import { createSequentialChainEdges } from './chain'; +import { detachOnAbort } from '~/utils/promises'; const SUBAGENT_GRAPH_LOAD_CONCURRENCY = 4; @@ -55,6 +56,8 @@ export type CheckAgentPermission = (params: { export interface DiscoverConnectedAgentsParams { req: ServerRequest; res: ServerResponse; + /** Owning run signal used to distinguish cancellation from dependency aborts. */ + signal?: AbortSignal; /** The already-initialized primary agent config (starting point for BFS). */ primaryConfig: InitializedAgent; /** @@ -195,7 +198,7 @@ async function initializeReferencedAgent( params: DiscoverConnectedAgentsParams, deps: DiscoverConnectedAgentsDeps, ): Promise<{ agent: Agent; config: InitializedAgent } | null> { - const agent = await deps.getAgent({ id: agentId }); + const agent = await detachOnAbort(deps.getAgent({ id: agentId }), params.signal); if (!agent) { logger.warn(`[initializeReferencedAgent] Agent ${agentId} not found, skipping`); deps.onAgentSkipped?.(agentId); @@ -209,13 +212,16 @@ async function initializeReferencedAgent( return null; } - const hasAccess = await deps.checkPermission({ - userId, - role: params.req.user?.role, - resourceType: params.resourceType ?? ResourceType.AGENT, - resourceId: agent._id, - requiredPermission: PermissionBits.VIEW, - }); + const hasAccess = await detachOnAbort( + deps.checkPermission({ + userId, + role: params.req.user?.role, + resourceType: params.resourceType ?? ResourceType.AGENT, + resourceId: agent._id, + requiredPermission: PermissionBits.VIEW, + }), + params.signal, + ); if (!hasAccess) { logger.warn(`[initializeReferencedAgent] User ${userId} lacks VIEW access to agent ${agentId}`); deps.onAgentSkipped?.(agentId); @@ -223,48 +229,55 @@ async function initializeReferencedAgent( } const validateAgentModel = deps.validateAgentModel ?? defaultValidateAgentModel; - const validation = await validateAgentModel({ - req: params.req, - res: params.res, - agent, - modelsConfig: params.modelsConfig, - logViolation: deps.logViolation, - }); + const validation = await detachOnAbort( + validateAgentModel({ + req: params.req, + res: params.res, + agent, + modelsConfig: params.modelsConfig, + logViolation: deps.logViolation, + }), + params.signal, + ); if (!validation.isValid) { throw new Error(validation.error?.message); } const scopedSkillIds = params.computeAccessibleSkillIds?.(agent); const initializeAgent = deps.initializeAgent ?? defaultInitializeAgent; - const config = await initializeAgent( - { - req: params.req, - res: params.res, - agent, - loadTools: params.loadTools, - requestFiles: params.requestFiles, - conversationId: params.conversationId, - parentMessageId: params.parentMessageId, - requestBody: params.requestBody, - endpointOption: { - ...(params.endpointOption ?? {}), - endpoint: EModelEndpoint.agents, + const config = await detachOnAbort( + initializeAgent( + { + req: params.req, + res: params.res, + agent, + loadTools: params.loadTools, + requestFiles: params.requestFiles, + conversationId: params.conversationId, + parentMessageId: params.parentMessageId, + requestBody: params.requestBody, + endpointOption: { + ...(params.endpointOption ?? {}), + endpoint: EModelEndpoint.agents, + }, + allowedProviders: params.allowedProviders, + accessibleSkillIds: scopedSkillIds, + skillAuthoringAvailable: params.computeSkillAuthoringAvailable?.(agent, scopedSkillIds), + skillStates: params.skillStates, + defaultActiveOnShare: params.defaultActiveOnShare, + codeEnvAvailable: params.codeEnvAvailable, + fileSearchAvailable: params.fileSearchAvailable, + resolveWebSearchGrant: params.resolveWebSearchGrant, + backgroundToolsAvailable: params.backgroundToolsAvailable, + toolIntentsAvailable: params.toolIntentsAvailable, + statefulSessionsAvailable: params.statefulSessionsAvailable, + allowedStatefulCodeEnvironments: params.allowedStatefulCodeEnvironments, + memoryAvailable: params.memoryAvailable, + signal: params.signal, }, - allowedProviders: params.allowedProviders, - accessibleSkillIds: scopedSkillIds, - skillAuthoringAvailable: params.computeSkillAuthoringAvailable?.(agent, scopedSkillIds), - skillStates: params.skillStates, - defaultActiveOnShare: params.defaultActiveOnShare, - codeEnvAvailable: params.codeEnvAvailable, - fileSearchAvailable: params.fileSearchAvailable, - resolveWebSearchGrant: params.resolveWebSearchGrant, - backgroundToolsAvailable: params.backgroundToolsAvailable, - toolIntentsAvailable: params.toolIntentsAvailable, - statefulSessionsAvailable: params.statefulSessionsAvailable, - allowedStatefulCodeEnvironments: params.allowedStatefulCodeEnvironments, - memoryAvailable: params.memoryAvailable, - }, - deps.db, + deps.db, + ), + params.signal, ); deps.onAgentInitialized?.(agentId, agent, config); return { agent, config }; @@ -329,7 +342,7 @@ export async function resolveSubagentGraphs( } return resolved; } catch (error) { - if (isFatalAgentInitializationError(error)) { + if (isFatalAgentInitializationError(error, { signal: params.signal })) { throw error; } failedMemberIds.add(memberId); @@ -458,7 +471,7 @@ export async function discoverConnectedAgents( collectEdges(agent.edges); } } catch (err) { - if (isFatalAgentInitializationError(err)) { + if (isFatalAgentInitializationError(err, { signal: params.signal })) { throw err; } logger.error(`[discoverConnectedAgents] Error processing agent ${agentId}:`, err); @@ -475,7 +488,7 @@ export async function discoverConnectedAgents( try { await processAgent(agentId); } catch (err) { - if (isFatalAgentInitializationError(err)) { + if (isFatalAgentInitializationError(err, { signal: params.signal })) { throw err; } logger.error(`[discoverConnectedAgents] Error processing chain agent ${agentId}:`, err); diff --git a/packages/api/src/agents/errors.spec.ts b/packages/api/src/agents/errors.spec.ts index caa3dd660b..a1d9817b4e 100644 --- a/packages/api/src/agents/errors.spec.ts +++ b/packages/api/src/agents/errors.spec.ts @@ -12,13 +12,27 @@ import { } from './errors'; describe('isFatalAgentInitializationError', () => { - it.each([ - ErrorTypes.RESOURCE_RECOVERY_REQUIRED, - ErrorTypes.STATEFUL_CODE_ENVIRONMENT_NOT_ALLOWED, - ErrorTypes.CODE_WORKSPACE_UNAVAILABLE, - AGENT_ATTACHMENT_LIMIT_EXCEEDED, - AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE, - ])('classifies %s as fatal', (code) => { + it('propagates cancellation even when optional MCP fallback is allowed', () => { + const abort = new DOMException('Stopped', 'AbortError'); + const controller = new AbortController(); + expect(isFatalAgentInitializationError(abort, { signal: controller.signal })).toBe(false); + controller.abort(abort); + expect( + isFatalAgentInitializationError(abort, { + allowExpectedMCPFallback: true, + signal: controller.signal, + }), + ).toBe(true); + }); + it.each( + [ + ErrorTypes.RESOURCE_RECOVERY_REQUIRED, + ErrorTypes.STATEFUL_CODE_ENVIRONMENT_NOT_ALLOWED, + ErrorTypes.CODE_WORKSPACE_UNAVAILABLE, + AGENT_ATTACHMENT_LIMIT_EXCEEDED, + AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE, + ].filter((code): code is string => typeof code === 'string'), + )('classifies %s as fatal', (code) => { expect(isFatalAgentInitializationError({ code })).toBe(true); }); @@ -38,6 +52,10 @@ describe('isFatalAgentInitializationError', () => { expect(isFatalAgentInitializationError(error)).toBe(false); }, ); + + it('does not classify a missing code as fatal when an enum member is unavailable', () => { + expect(isFatalAgentInitializationError(new Error('ordinary failure'))).toBe(false); + }); }); describe('LangChain provider error text', () => { diff --git a/packages/api/src/agents/errors.ts b/packages/api/src/agents/errors.ts index 0207f7c332..81a8ff14b8 100644 --- a/packages/api/src/agents/errors.ts +++ b/packages/api/src/agents/errors.ts @@ -3,10 +3,20 @@ import { parseLangChainErrorCode, stripLangChainTroubleshootingUrl, } from 'librechat-data-provider'; +import { isOwnedAbortError } from '~/utils/errors'; export const AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE = 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE'; export const AGENT_ATTACHMENT_LIMIT_EXCEEDED = 'AGENT_ATTACHMENT_LIMIT_EXCEEDED'; +const FATAL_AGENT_INITIALIZATION_CODES = new Set( + [ + AGENT_ATTACHMENT_LIMIT_EXCEEDED, + ErrorTypes.RESOURCE_RECOVERY_REQUIRED, + ErrorTypes.STATEFUL_CODE_ENVIRONMENT_NOT_ALLOWED, + ErrorTypes.CODE_WORKSPACE_UNAVAILABLE, + ].filter((code): code is string => typeof code === 'string'), +); + export function createStatefulCodeEnvironmentPolicyError(environment: string): Error { return Object.assign( new Error(`Stateful code environment is not allowed by this deployment: ${environment}`), @@ -19,6 +29,7 @@ export function createStatefulCodeEnvironmentPolicyError(environment: string): E } export interface FatalAgentInitializationOptions { + signal?: AbortSignal; /** * Skill `allowed-tools` may add an MCP tool beyond the agent's configured * baseline. That union load is allowed to retry without the skill extras; @@ -46,10 +57,8 @@ export function isFatalAgentInitializationError( ): boolean { const code = getErrorCode(error); return ( - code === AGENT_ATTACHMENT_LIMIT_EXCEEDED || - code === ErrorTypes.RESOURCE_RECOVERY_REQUIRED || - code === ErrorTypes.STATEFUL_CODE_ENVIRONMENT_NOT_ALLOWED || - code === ErrorTypes.CODE_WORKSPACE_UNAVAILABLE || + isOwnedAbortError(error, options.signal) || + FATAL_AGENT_INITIALIZATION_CODES.has(code as string) || (code === AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE && options.allowExpectedMCPFallback !== true) ); } diff --git a/packages/api/src/agents/failures/terminal.ts b/packages/api/src/agents/failures/terminal.ts index 08cb0c17f4..c03c993fef 100644 --- a/packages/api/src/agents/failures/terminal.ts +++ b/packages/api/src/agents/failures/terminal.ts @@ -1,7 +1,7 @@ import { ErrorTypes } from 'librechat-data-provider'; import type { SafeErrorMetadata } from '../../utils/errors'; import type { ModelErrorTrackerCallback } from './tracker'; -import { getSafeErrorMetadata, isAbortError } from '../../utils/errors'; +import { getSafeErrorMetadata, isOwnedAbortError } from '../../utils/errors'; import { traceIdForMessage } from '../../langfuse/trace'; import { createModelErrorTracker } from './tracker'; import { resolveLangChainError } from '../errors'; @@ -36,9 +36,9 @@ export interface TerminalRunErrorObserver { readonly getUserFacingError: (error: unknown, fallback: () => string) => string; } -/** A run cancellation requires both host-owned abort state and an abort-shaped rejection. */ +/** A run cancellation requires host-owned abort state plus its own reason or an abort shape. */ export function isAgentRunCancellation(error: unknown, signal?: AbortSignal): boolean { - return signal?.aborted === true && isAbortError(error); + return isOwnedAbortError(error, signal); } export function getUpstreamModelErrorMetadata( diff --git a/packages/api/src/agents/initialize.ts b/packages/api/src/agents/initialize.ts index 92f95e2ee5..c1e720392e 100644 --- a/packages/api/src/agents/initialize.ts +++ b/packages/api/src/agents/initialize.ts @@ -790,6 +790,8 @@ export function optsOutOfAttachedCodeEnvironment( * Matches the CJS signature from api/server/services/Endpoints/agents/agent.js */ export interface InitializeAgentParams { + /** Cancellation signal owned by the run performing initialization. */ + signal?: AbortSignal; /** Explicit transport-free execution state. */ runtime?: AgentExecutionContext; /** Request-backed compatibility adapter for callers not yet migrated. */ @@ -1815,7 +1817,12 @@ export async function initializeAgent( try { loadToolsResult = await callLoadTools(requestedToolNames); } catch (err) { - if (isFatalAgentInitializationError(err, { allowExpectedMCPFallback: true })) { + if ( + isFatalAgentInitializationError(err, { + signal: params.signal, + allowExpectedMCPFallback: true, + }) + ) { throw err; } if (extraAllowedToolNames.length > 0) { diff --git a/packages/api/src/agents/openai/service.ts b/packages/api/src/agents/openai/service.ts index 42b0927f07..e3c2f7c97e 100644 --- a/packages/api/src/agents/openai/service.ts +++ b/packages/api/src/agents/openai/service.ts @@ -187,6 +187,8 @@ interface InitializedAgent { interface InitializeAgentParams { req: Request; res: ServerResponse; + /** Cancellation signal owned by this chat-completion request. */ + signal?: CoreInitializeAgentParams['signal']; agent: Agent; conversationId?: string | null; parentMessageId?: string | null; @@ -777,6 +779,7 @@ export async function createAgentChatCompletion( allowedStatefulCodeEnvironments, backgroundToolsAvailable, toolIntentsAvailable, + signal: abortController.signal, }); const modelBoundAgents = collectReachableAgents([initializedAgent]); diff --git a/packages/api/src/mcp/MCPConnectionFactory.ts b/packages/api/src/mcp/MCPConnectionFactory.ts index 95fcb8ec05..66b3368ea1 100644 --- a/packages/api/src/mcp/MCPConnectionFactory.ts +++ b/packages/api/src/mcp/MCPConnectionFactory.ts @@ -11,7 +11,12 @@ import type { OAuthStoredClientMetadata, OAuthClientSource, } from '~/mcp/oauth'; -import type { OboTokenResolver, OboTrustChecker, UpstreamTokenProvider } from '~/mcp/oauth/obo'; +import type { + OboTokenResolver, + OboTrustChecker, + UpstreamTokenProvider, + UpstreamTokenProviderResolver, +} from '~/mcp/oauth/obo'; import type { AuthIdentityContext } from '~/utils/identity'; import type { FlowStateManager } from '~/flow/manager'; import type * as t from './types'; @@ -43,7 +48,9 @@ import { createDeadlineAbortSignal, } from './utils'; import { PENDING_STALE_MS, FlowStateNotFoundError, normalizeExpiresAt } from '~/flow/manager'; +import { createLazyOboUpstreamTokenProvider, awaitOboOperation } from '~/mcp/oauth/obo'; import { preProcessGraphTokens } from '~/utils/graph'; +import { isAbortError } from '~/utils/errors'; import { MCPConnection } from './connection'; import { processMCPEnv } from '~/utils'; import { mcpConfig } from './mcpConfig'; @@ -103,6 +110,7 @@ export class MCPConnectionFactory { protected readonly oboTokenResolver?: OboTokenResolver; protected readonly oboTrustChecker?: OboTrustChecker; protected upstreamTokenProvider?: UpstreamTokenProvider; + protected upstreamTokenProviderResolver?: UpstreamTokenProviderResolver; protected readonly oboIdentityContext?: AuthIdentityContext; /** Why the OBO re-exchange failed, when that is more actionable than the server's 401. */ private oboRefreshError?: Error; @@ -612,6 +620,7 @@ export class MCPConnectionFactory { this.user = options?.user; this.upstreamTokenProvider = options?.upstreamTokenProvider; + this.upstreamTokenProviderResolver = options?.upstreamTokenProviderResolver; if (options != null && 'useOAuth' in options) { this.useOAuth = true; @@ -641,6 +650,12 @@ export class MCPConnectionFactory { return null; } + if (!this.upstreamTokenProvider && this.upstreamTokenProviderResolver) { + this.upstreamTokenProvider = createLazyOboUpstreamTokenProvider( + this.upstreamTokenProviderResolver, + this.signal, + ); + } if (!this.upstreamTokenProvider) { throw new Error( `${this.logPrefix} Internal: upstreamTokenProvider not plumbed for OBO connection. ` + @@ -651,11 +666,14 @@ export class MCPConnectionFactory { if (this.oboTrustChecker) { const config = this.serverConfig as t.ParsedServerConfig; - const trusted = await this.oboTrustChecker({ - source: config.source, - author: config.author, - dbId: config.dbId, - }); + const trusted = await awaitOboOperation( + this.oboTrustChecker({ + source: config.source, + author: config.author, + dbId: config.dbId, + }), + this.signal, + ); if (!trusted) { logger.warn( `${this.logPrefix} OBO config not trusted (author lacks CONFIGURE_OBO permission); skipping OBO token exchange`, @@ -665,13 +683,16 @@ export class MCPConnectionFactory { } logger.info(`${this.logPrefix} Resolving OBO token for scopes: ${oboConfig.scopes}`); - return resolveOboToken( - this.user, - oboConfig, - this.oboTokenResolver, - this.upstreamTokenProvider, - this.oboIdentityContext, - forceRefresh, + return awaitOboOperation( + resolveOboToken( + this.user, + oboConfig, + this.oboTokenResolver, + this.upstreamTokenProvider, + this.oboIdentityContext, + forceRefresh, + ), + this.signal, ); } @@ -680,7 +701,7 @@ export class MCPConnectionFactory { return !!this.serverConfig.obo && !!this.oboTokenResolver && !!this.user; } - protected createOboConnectionError(error: OboTokenResolutionError): Error { + protected createOboConnectionError(error: OboTokenResolutionError): OboTokenResolutionError { let recoveryHint = 'Re-authenticate the user and retry.'; if (error.retryable) { @@ -689,8 +710,11 @@ export class MCPConnectionFactory { recoveryHint = 'Re-authenticate the user or verify the configured OBO scopes and retry.'; } - return new Error( + return new OboTokenResolutionError( + error.reason, `${error.userMessage} Unable to connect to OBO server "${this.serverName}". ${recoveryHint}`, + error.retryable, + error, ); } @@ -784,6 +808,7 @@ export class MCPConnectionFactory { this.oauthEnd = undefined; this.returnOnOAuth = false; this.upstreamTokenProvider = undefined; + this.upstreamTokenProviderResolver = undefined; } private getServerUrl(): string | undefined { @@ -1438,7 +1463,11 @@ export class MCPConnectionFactory { logger.info(`${this.logPrefix} OBO token re-exchanged; retrying connection`); connection.emit('oauthHandled'); } catch (error) { - logger.error(`${this.logPrefix} OBO token re-exchange failed`, error); + if (isAbortError(error)) { + logger.debug(`${this.logPrefix} OBO token re-exchange cancelled`); + } else { + logger.error(`${this.logPrefix} OBO token re-exchange failed`, error); + } /** * `connectClient` rejects its handling promise with this error and then * rethrows the server's original 401, so a diagnosis like an unrefreshable @@ -1465,6 +1494,11 @@ export class MCPConnectionFactory { if (error instanceof Error) { return error; } + if (isAbortError(error)) { + return Object.assign(new Error('The operation was aborted.', { cause: error }), { + name: 'AbortError', + }); + } return new Error(`OBO token re-exchange failed for "${this.serverName}".`); } diff --git a/packages/api/src/mcp/MCPManager.ts b/packages/api/src/mcp/MCPManager.ts index 7e7d273aa1..5e9a7825cf 100644 --- a/packages/api/src/mcp/MCPManager.ts +++ b/packages/api/src/mcp/MCPManager.ts @@ -4,7 +4,12 @@ import { Permissions, PermissionTypes } from 'librechat-data-provider'; import { CallToolResultSchema, ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js'; import type { RequestOptions } from '@modelcontextprotocol/sdk/shared/protocol.js'; import type { TokenMethods, IUser } from '@librechat/data-schemas'; -import type { OboTokenResolver, OboTrustChecker, UpstreamTokenProvider } from '~/mcp/oauth/obo'; +import type { + OboTokenResolver, + OboTrustChecker, + UpstreamTokenProvider, + UpstreamTokenProviderResolver, +} from '~/mcp/oauth/obo'; import type { AuthIdentityContext } from '~/utils/identity'; import type { GraphTokenResolver } from '~/utils/graph'; import type { FlowStateManager } from '~/flow/manager'; @@ -25,6 +30,7 @@ import { import { getMCPAppToolsPublicationGeneration, getMCPToolsChangedGeneration } from './toolsChanged'; import { MCPAuthenticationRejectedError, isMCPTransportAuthenticationError } from './errors'; import { resolveDirectOpenIDBearerConfig, usesDirectOpenIDBearerRecovery } from './openid'; +import { createLazyOboUpstreamTokenProvider, awaitOboOperation } from '~/mcp/oauth/obo'; import { MCPServersInitializer } from './registry/MCPServersInitializer'; import { OboTokenResolutionError, resolveOboToken } from '~/mcp/oauth'; import { MCPServerCatalogRecoveryTracker } from './catalog/recovery'; @@ -36,7 +42,7 @@ import { MCPConnectionFactory } from './MCPConnectionFactory'; import { processMCPEnv, isPluginSourced } from '~/utils/env'; import { OAuthLifecycleRelay } from './oauth/pending'; import { preProcessGraphTokens } from '~/utils/graph'; -import { isAbortError } from '~/utils/errors'; +import { isOwnedAbortError } from '~/utils/errors'; import { formatToolContent } from './parsers'; import { MCPConnection } from './connection'; import { mcpConfig } from './mcpConfig'; @@ -500,6 +506,7 @@ export class MCPManager extends UserConnectionManager { requestBody: args.requestBody, graphTokenResolver: args.graphTokenResolver, upstreamTokenProvider: args.upstreamTokenProvider, + upstreamTokenProviderResolver: args.upstreamTokenProviderResolver, connectionTimeout: args.connectionTimeout, deadlineMs: args.deadlineMs, signal: args.signal, @@ -535,6 +542,7 @@ export class MCPManager extends UserConnectionManager { oboTokenResolver: args.oboTokenResolver, oboTrustChecker: args.oboTrustChecker, upstreamTokenProvider: args.upstreamTokenProvider, + upstreamTokenProviderResolver: args.upstreamTokenProviderResolver, oboIdentityContext: args.oboIdentityContext, }); @@ -841,6 +849,7 @@ Please follow these instructions when using tools from the respective MCP server requestScopedConnections, graphTokenResolver, upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, onOAuthCredentialsChanged, onOAuthCredentialsChanging, @@ -860,6 +869,7 @@ Please follow these instructions when using tools from the respective MCP server requestScopedConnections?: t.RequestScopedMCPConnectionStore; graphTokenResolver?: GraphTokenResolver; upstreamTokenProvider?: UpstreamTokenProvider; + upstreamTokenProviderResolver?: UpstreamTokenProviderResolver; oboIdentityContext?: AuthIdentityContext; onOAuthCredentialsChanged?: t.UserConnectionContext['onOAuthCredentialsChanged']; onOAuthCredentialsChanging?: t.UserConnectionContext['onOAuthCredentialsChanging']; @@ -919,6 +929,7 @@ Please follow these instructions when using tools from the respective MCP server requestScopedConnections, graphTokenResolver, upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, onOAuthCredentialsChanged, onOAuthCredentialsChanging, @@ -1093,6 +1104,7 @@ Please follow these instructions when using tools from the respective MCP server oboTokenResolver, oboTrustChecker, upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, onOAuthCredentialsChanged, onOAuthCredentialsChanging, @@ -1116,6 +1128,7 @@ Please follow these instructions when using tools from the respective MCP server oboTokenResolver?: OboTokenResolver; oboTrustChecker?: OboTrustChecker; upstreamTokenProvider?: UpstreamTokenProvider; + upstreamTokenProviderResolver?: UpstreamTokenProviderResolver; oboIdentityContext?: AuthIdentityContext; onOAuthCredentialsChanged?: t.UserConnectionContext['onOAuthCredentialsChanged']; onOAuthCredentialsChanging?: t.UserConnectionContext['onOAuthCredentialsChanging']; @@ -1180,6 +1193,7 @@ Please follow these instructions when using tools from the respective MCP server oboTokenResolver, oboTrustChecker, upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, onOAuthCredentialsChanged, onOAuthCredentialsChanging, @@ -1272,6 +1286,7 @@ Please follow these instructions when using tools from the respective MCP server const oboConfig = rawConfig.obo; const usesObo = Boolean(oboConfig && oboTokenResolver && user); + let oboUpstreamTokenProvider = upstreamTokenProvider; /** * Resolves the downstream token for this call and installs it as the request @@ -1283,7 +1298,13 @@ Please follow these instructions when using tools from the respective MCP server if (!oboConfig || !oboTokenResolver || !user) { return; } - if (!upstreamTokenProvider) { + if (!oboUpstreamTokenProvider && upstreamTokenProviderResolver) { + oboUpstreamTokenProvider = createLazyOboUpstreamTokenProvider( + upstreamTokenProviderResolver, + options?.signal, + ); + } + if (!oboUpstreamTokenProvider) { throw new McpError( ErrorCode.InternalError, `${logPrefix} Internal: upstreamTokenProvider not plumbed for OBO tool call. ` + @@ -1309,13 +1330,16 @@ Please follow these instructions when using tools from the respective MCP server } let oboTokens: MCPOAuthTokens; try { - oboTokens = await resolveOboToken( - user, - oboConfig, - oboTokenResolver, - upstreamTokenProvider, - oboIdentityContext, - forceRefresh, + oboTokens = await awaitOboOperation( + resolveOboToken( + user, + oboConfig, + oboTokenResolver, + oboUpstreamTokenProvider, + oboIdentityContext, + forceRefresh, + ), + options?.signal, ); } catch (error) { if (error instanceof OboTokenResolutionError) { @@ -1425,6 +1449,7 @@ Please follow these instructions when using tools from the respective MCP server requestScopedConnections, graphTokenResolver, upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, onOAuthCredentialsChanged, onOAuthCredentialsChanging, @@ -1515,6 +1540,7 @@ Please follow these instructions when using tools from the respective MCP server requestScopedConnections, graphTokenResolver, upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, onOAuthCredentialsChanged, onOAuthCredentialsChanging, @@ -1594,7 +1620,7 @@ Please follow these instructions when using tools from the respective MCP server * cancellation working, not a fault, so it stays out of the error log. * The error must look like an abort too — a real failure can reject in * the same tick as the Stop and has to stay visible. */ - if (options?.signal?.aborted === true && isAbortError(error)) { + if (isOwnedAbortError(error, options?.signal)) { logger.debug(`${logPrefix}[${toolName}] Tool call cancelled by user abort`); throw error; } diff --git a/packages/api/src/mcp/UserConnectionManager.ts b/packages/api/src/mcp/UserConnectionManager.ts index d02c179bc5..d264f039ca 100644 --- a/packages/api/src/mcp/UserConnectionManager.ts +++ b/packages/api/src/mcp/UserConnectionManager.ts @@ -583,6 +583,7 @@ export abstract class UserConnectionManager { oboTokenResolver, oboTrustChecker, upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, onOAuthCredentialsChanged, onOAuthCredentialsChanging, @@ -858,6 +859,7 @@ export abstract class UserConnectionManager { oboTokenResolver: oboTokenResolver, oboTrustChecker: oboTrustChecker, upstreamTokenProvider: upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, graphTokenResolver, returnOnOAuth: returnOnOAuth, @@ -875,6 +877,7 @@ export abstract class UserConnectionManager { requestBody, graphTokenResolver, upstreamTokenProvider, + upstreamTokenProviderResolver, connectionTimeout, signal, }; @@ -948,6 +951,7 @@ export abstract class UserConnectionManager { oboTokenResolver, oboTrustChecker, upstreamTokenProvider, + upstreamTokenProviderResolver, oboIdentityContext, signal, returnOnOAuth, diff --git a/packages/api/src/mcp/__tests__/MCPConnectionFactory.test.ts b/packages/api/src/mcp/__tests__/MCPConnectionFactory.test.ts index 2e888a7a65..f370a8d12a 100644 --- a/packages/api/src/mcp/__tests__/MCPConnectionFactory.test.ts +++ b/packages/api/src/mcp/__tests__/MCPConnectionFactory.test.ts @@ -5551,6 +5551,165 @@ describe('MCPConnectionFactory', () => { ); }); + it('resolves a provider lazily after the server is confirmed to use OBO', async () => { + const { resolveOboToken } = jest.requireMock('~/mcp/oauth') as { + resolveOboToken: jest.Mock; + }; + resolveOboToken.mockImplementationOnce(async (_user, _config, _exchange, provider) => { + await provider(); + return { + access_token: 'connection-time-obo-token', + token_type: 'Bearer', + obtained_at: Date.now(), + expires_at: Date.now() + 3600_000, + }; + }); + const upstreamTokenProvider = jest.fn(); + const upstreamTokenProviderResolver = jest.fn().mockResolvedValue(upstreamTokenProvider); + const oboTokenResolver = jest.fn(); + + await MCPConnectionFactory.create( + { serverName: 'obo-srv', serverConfig: oboServerConfig }, + { + useOAuth: true, + user: mockUser, + flowManager: mockFlowManager, + tokenMethods: { + findToken: jest.fn(), + createToken: jest.fn(), + updateToken: jest.fn(), + deleteTokens: jest.fn(), + }, + oboTokenResolver, + upstreamTokenProviderResolver, + }, + ); + + expect(upstreamTokenProviderResolver).toHaveBeenCalledTimes(1); + expect(resolveOboToken).toHaveBeenCalledWith( + mockUser, + oboServerConfig.obo, + oboTokenResolver, + expect.any(Function), + undefined, + false, + ); + }); + + it.each([undefined, new Error('owner stopped'), 'stopped'])( + 'detaches connection initialization from an uncooperative OBO exchange (%s)', + async (reason) => { + const controller = new AbortController(); + const { resolveOboToken } = jest.requireMock('~/mcp/oauth'); + resolveOboToken.mockImplementationOnce(() => { + queueMicrotask(() => controller.abort(reason)); + return new Promise(() => {}); + }); + await expect( + MCPConnectionFactory.create( + { serverName: 'obo-srv', serverConfig: oboServerConfig }, + { + useOAuth: true, + user: mockUser, + flowManager: mockFlowManager, + tokenMethods: { + findToken: jest.fn(), + createToken: jest.fn(), + updateToken: jest.fn(), + deleteTokens: jest.fn(), + }, + signal: controller.signal, + oboTokenResolver: jest.fn(), + upstreamTokenProvider: jest.fn(), + }, + ), + ).rejects.toMatchObject({ name: 'AbortError' }); + }, + ); + + it.each([undefined, new Error('owner stopped during trust check'), 'stopped'])( + 'detaches connection initialization from a pending OBO trust check (%s)', + async (reason) => { + const controller = new AbortController(); + const oboTrustChecker = jest.fn().mockImplementation( + () => + new Promise(() => { + queueMicrotask(() => controller.abort(reason)); + }), + ); + const { resolveOboToken } = jest.requireMock('~/mcp/oauth'); + resolveOboToken.mockClear(); + + await expect( + MCPConnectionFactory.create( + { serverName: 'obo-srv', serverConfig: oboServerConfig }, + { + useOAuth: true, + user: mockUser, + flowManager: mockFlowManager, + tokenMethods: { + findToken: jest.fn(), + createToken: jest.fn(), + updateToken: jest.fn(), + deleteTokens: jest.fn(), + }, + signal: controller.signal, + oboTokenResolver: jest.fn(), + upstreamTokenProvider: jest.fn(), + oboTrustChecker, + }, + ), + ).rejects.toMatchObject({ name: 'AbortError' }); + expect(resolveOboToken).not.toHaveBeenCalled(); + }, + ); + + it('does not look up credentials for an untrusted OBO configuration', async () => { + const upstreamTokenProviderResolver = jest.fn(); + const oboTrustChecker = jest.fn().mockResolvedValue(false); + const { resolveOboToken } = jest.requireMock('~/mcp/oauth'); + resolveOboToken.mockClear(); + await expect( + MCPConnectionFactory.create( + { serverName: 'obo-srv', serverConfig: oboServerConfig }, + { + useOAuth: true, + user: mockUser, + flowManager: mockFlowManager, + tokenMethods: { + findToken: jest.fn(), + createToken: jest.fn(), + updateToken: jest.fn(), + deleteTokens: jest.fn(), + }, + oboTokenResolver: jest.fn(), + oboTrustChecker, + upstreamTokenProviderResolver, + }, + ), + ).rejects.toThrow('OBO token exchange failed'); + expect(oboTrustChecker).toHaveBeenCalled(); + expect(upstreamTokenProviderResolver).not.toHaveBeenCalled(); + expect(resolveOboToken).not.toHaveBeenCalled(); + }); + + it('does not invoke the resolver for a non-OBO direct-bearer server', async () => { + const directBearerConfig = { + type: 'streamable-http' as const, + url: 'https://direct.example.com', + headers: { Authorization: 'Bearer static-token' }, + } as unknown as t.MCPOptions; + mockProcessMCPEnv.mockReturnValue(directBearerConfig); + const upstreamTokenProviderResolver = jest.fn(); + + await MCPConnectionFactory.create( + { serverName: 'direct-srv', serverConfig: directBearerConfig }, + { user: mockUser, upstreamTokenProviderResolver }, + ); + + expect(upstreamTokenProviderResolver).not.toHaveBeenCalled(); + }); + it('throws an internal error when upstreamTokenProvider is omitted on an OBO connection', async () => { const { resolveOboToken } = jest.requireMock('~/mcp/oauth') as { resolveOboToken: jest.Mock; @@ -5578,6 +5737,52 @@ describe('MCPConnectionFactory', () => { ).rejects.toThrow(/upstreamTokenProvider not plumbed/); expect(resolveOboToken).not.toHaveBeenCalled(); }); + + it('preserves OBO resolution failure metadata at connection establishment', async () => { + const { resolveOboToken } = jest.requireMock('~/mcp/oauth') as { + resolveOboToken: jest.Mock; + }; + const OboError = OboTokenResolutionError as unknown as jest.Mock; + OboError.mockImplementation( + (reason: string, userMessage: string, retryable = false, cause?: unknown) => { + const error = new Error(userMessage); + Object.setPrototypeOf(error, OboError.prototype); + return Object.assign(error, { + name: 'OboTokenResolutionError', + reason, + retryable, + userMessage, + cause, + }); + }, + ); + resolveOboToken.mockRejectedValueOnce( + new OboTokenResolutionError('session_refresh_failed', 'Sign-in expired.'), + ); + + await expect( + MCPConnectionFactory.create( + { serverName: 'obo-srv', serverConfig: oboServerConfig }, + { + useOAuth: true, + user: mockUser, + flowManager: mockFlowManager, + tokenMethods: { + findToken: jest.fn(), + createToken: jest.fn(), + updateToken: jest.fn(), + deleteTokens: jest.fn(), + }, + oboTokenResolver: jest.fn(), + upstreamTokenProvider: jest.fn(), + }, + ), + ).rejects.toMatchObject({ + name: 'OboTokenResolutionError', + reason: 'session_refresh_failed', + retryable: false, + }); + }); }); describe('OBO authentication failures', () => { @@ -5674,6 +5879,18 @@ describe('MCPConnectionFactory', () => { expect(mockConnectionInstance.stopReconnecting).not.toHaveBeenCalled(); }); + it('preserves a cancelled re-exchange without logging a sign-in failure', async () => { + const abort = new DOMException('Stopped', 'AbortError'); + resolveOboToken.mockResolvedValueOnce(connectionTokens).mockRejectedValueOnce(abort); + mockConnectionInstance.connect.mockImplementationOnce(async () => { + await getAuthHandler()(); + throw new Error('HTTP 401 Unauthorized'); + }); + await expect(createOboConnection()).rejects.toMatchObject({ name: 'AbortError' }); + expect(mockLogger.error).not.toHaveBeenCalled(); + expect(mockConnectionInstance.stopReconnecting).toHaveBeenCalled(); + }); + it('retires the connection when the re-exchange fails', async () => { resolveOboToken .mockResolvedValueOnce(connectionTokens) diff --git a/packages/api/src/mcp/__tests__/MCPManager.test.ts b/packages/api/src/mcp/__tests__/MCPManager.test.ts index c6e94ed501..ac8c638cd1 100644 --- a/packages/api/src/mcp/__tests__/MCPManager.test.ts +++ b/packages/api/src/mcp/__tests__/MCPManager.test.ts @@ -2855,6 +2855,102 @@ describe('MCPManager', () => { expect(mockConnection.client.request).toHaveBeenCalled(); }); + it.each([undefined, new Error('owner stopped'), 'stopped'])( + 'detaches a cancelled tool from an uncooperative OBO exchange (%s)', + async (reason) => { + const controller = new AbortController(); + mockResolveOboToken.mockImplementationOnce(() => { + queueMicrotask(() => controller.abort(reason)); + return new Promise(() => {}); + }); + mockAppConnections({ get: jest.fn().mockResolvedValue(mockConnection) }); + (mockRegistryInstance.getServerConfig as jest.Mock).mockResolvedValue(serverConfig); + const manager = await MCPManager.createInstance(newMCPServersConfig()); + jest.spyOn(manager, 'getUserConnection').mockResolvedValue(mockConnection); + await expect( + manager.callTool({ + user: mockUser as IUser, + serverName, + toolName: 'test_tool', + provider: 'openai', + options: { signal: controller.signal }, + oboTokenResolver: mockOboTokenResolver, + upstreamTokenProvider: mockUpstreamTokenProvider, + flowManager: mockFlowManager as unknown as Parameters< + typeof manager.callTool + >[0]['flowManager'], + }), + ).rejects.toMatchObject({ name: 'AbortError' }); + expect(mockConnection.client.request).not.toHaveBeenCalled(); + }, + ); + + it('lazily resolves the upstream provider for an OBO tool call', async () => { + mockResolveOboToken.mockImplementationOnce(async (_user, _config, _exchange, provider) => { + await provider(); + return { + access_token: 'fresh-obo-token', + token_type: 'Bearer', + obtained_at: Date.now(), + expires_at: Date.now() + 3600_000, + }; + }); + mockAppConnections({ get: jest.fn().mockResolvedValue(mockConnection) }); + (mockRegistryInstance.getServerConfig as jest.Mock).mockResolvedValue(serverConfig); + const manager = await MCPManager.createInstance(newMCPServersConfig()); + jest.spyOn(manager, 'getUserConnection').mockResolvedValue(mockConnection); + const upstreamTokenProviderResolver = jest.fn().mockResolvedValue(mockUpstreamTokenProvider); + + await manager.callTool({ + user: mockUser as IUser, + serverName, + toolName: 'test_tool', + provider: 'openai', + flowManager: mockFlowManager as unknown as Parameters< + typeof manager.callTool + >[0]['flowManager'], + oboTokenResolver: mockOboTokenResolver, + upstreamTokenProviderResolver, + }); + + expect(upstreamTokenProviderResolver).toHaveBeenCalledTimes(1); + expect(mockResolveOboToken).toHaveBeenCalledWith( + mockUser, + serverConfig.obo, + mockOboTokenResolver, + expect.any(Function), + undefined, + false, + ); + }); + + it('does not resolve an OBO provider for a non-OBO tool call', async () => { + const directConfig = { + type: 'streamable-http', + url: 'https://direct.example.com', + headers: { Authorization: 'Bearer static-token' }, + } as t.ParsedServerConfig; + mockAppConnections({ get: jest.fn().mockResolvedValue(mockConnection) }); + (mockRegistryInstance.getServerConfig as jest.Mock).mockResolvedValue(directConfig); + const manager = await MCPManager.createInstance(newMCPServersConfig()); + jest.spyOn(manager, 'getUserConnection').mockResolvedValue(mockConnection); + const upstreamTokenProviderResolver = jest.fn(); + + await manager.callTool({ + user: mockUser as IUser, + serverName, + serverConfig: directConfig, + toolName: 'test_tool', + provider: 'openai', + flowManager: mockFlowManager as unknown as Parameters< + typeof manager.callTool + >[0]['flowManager'], + upstreamTokenProviderResolver, + }); + + expect(upstreamTokenProviderResolver).not.toHaveBeenCalled(); + }); + it('passes the OBO-cleaned config into runtime header processing', async () => { const mixedConfig = { ...serverConfig, diff --git a/packages/api/src/mcp/errors.spec.ts b/packages/api/src/mcp/errors.spec.ts index 6c9bb13661..175d001061 100644 --- a/packages/api/src/mcp/errors.spec.ts +++ b/packages/api/src/mcp/errors.spec.ts @@ -3,7 +3,35 @@ import { SseError } from '@modelcontextprotocol/sdk/client/sse.js'; import { McpError, ErrorCode } from '@modelcontextprotocol/sdk/types.js'; import { UnauthorizedError } from '@modelcontextprotocol/sdk/client/auth.js'; import { StreamableHTTPError } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; -import { isMCPTransportAuthenticationError, MCPTransportAuthenticationError } from './errors'; +import { + isMCPTransportAuthenticationError, + MCPTransportAuthenticationError, + isMCPInitializationError, +} from './errors'; +import { OboTokenResolutionError } from './oauth/obo'; + +describe('MCP initialization propagation', () => { + it.each([ + new DOMException('Stopped', 'AbortError'), + new OboTokenResolutionError('session_refresh_failed', 'Retry later', true), + new OboTokenResolutionError('session_refresh_failed', 'Sign in', false), + ])('propagates cancellation and typed credential failures: %s', (error) => { + const controller = new AbortController(); + controller.abort(error); + expect(isMCPInitializationError(error, controller.signal)).toBe(true); + }); + it('does not confuse a dependency abort with cancellation of a live run', () => { + expect( + isMCPInitializationError( + new DOMException('timeout', 'AbortError'), + new AbortController().signal, + ), + ).toBe(false); + }); + it('keeps unrelated optional-tool failures eligible for fallback', () => { + expect(isMCPInitializationError(new Error('optional tool unavailable'))).toBe(false); + }); +}); describe('direct bearer transport rejection classification', () => { it.each([ diff --git a/packages/api/src/mcp/errors.ts b/packages/api/src/mcp/errors.ts index 0399770db4..50c652de77 100644 --- a/packages/api/src/mcp/errors.ts +++ b/packages/api/src/mcp/errors.ts @@ -5,6 +5,20 @@ import { McpError } from '@modelcontextprotocol/sdk/types.js'; import { SseError } from '@modelcontextprotocol/sdk/client/sse.js'; import { UnauthorizedError } from '@modelcontextprotocol/sdk/client/auth.js'; import { StreamableHTTPError } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; +import { OpenIDReauthRequiredError } from '~/utils/oidc'; +import { OboTokenResolutionError } from './oauth/obo'; +import { isOwnedAbortError } from '~/utils/errors'; + +/** These outcomes must reach the owning run instead of hiding a tool during discovery. */ +export function isMCPInitializationError(error: unknown, signal?: AbortSignal): boolean { + return ( + isOwnedAbortError(error, signal) || + error instanceof MCPAuthenticationRejectedError || + error instanceof MCPAuthenticationRefreshError || + error instanceof OboTokenResolutionError || + error instanceof OpenIDReauthRequiredError + ); +} export const MCPErrorCodes = { DOMAIN_NOT_ALLOWED: 'MCP_DOMAIN_NOT_ALLOWED', diff --git a/packages/api/src/mcp/oauth/obo.spec.ts b/packages/api/src/mcp/oauth/obo.spec.ts index 6f90b2c4aa..daa1572641 100644 --- a/packages/api/src/mcp/oauth/obo.spec.ts +++ b/packages/api/src/mcp/oauth/obo.spec.ts @@ -1,7 +1,12 @@ import { Permissions, PermissionTypes } from 'librechat-data-provider'; import type { IUser } from '@librechat/data-schemas'; import type { OboTokenResolver, UpstreamTokenProvider } from './obo'; -import { isOboConfigStillTrusted, resolveOboToken } from './obo'; +import { + createLazyOboUpstreamTokenProvider, + isOboConfigStillTrusted, + resolveOboToken, + selectMCPUpstreamTokenProvider, +} from './obo'; jest.mock('@librechat/data-schemas', () => ({ logger: { @@ -33,6 +38,36 @@ const liveTokens = { const liveProvider: UpstreamTokenProvider = jest.fn().mockResolvedValue(liveTokens); const nullProvider: UpstreamTokenProvider = jest.fn().mockResolvedValue(null); +describe('selectMCPUpstreamTokenProvider', () => { + it.each([false, true])( + 'keeps explicit credentials regardless of deferred OBO lookup (%s)', + (deferred) => { + const createSessionProvider = jest.fn(); + expect( + selectMCPUpstreamTokenProvider({ + upstreamTokenProvider: liveProvider, + upstreamTokenProviderResolver: deferred ? jest.fn() : undefined, + createSessionProvider, + }), + ).toBe(liveProvider); + expect(createSessionProvider).not.toHaveBeenCalled(); + }, + ); + + it('creates browser credentials only when no deferred OBO source is present', () => { + const createSessionProvider = jest.fn().mockReturnValue(liveProvider); + expect( + selectMCPUpstreamTokenProvider({ + upstreamTokenProviderResolver: jest.fn(), + createSessionProvider, + }), + ).toBeUndefined(); + expect(createSessionProvider).not.toHaveBeenCalled(); + expect(selectMCPUpstreamTokenProvider({ createSessionProvider })).toBe(liveProvider); + expect(createSessionProvider).toHaveBeenCalledTimes(1); + }); +}); + describe('resolveOboToken', () => { const mockUser: Partial = { id: 'user-123', @@ -61,6 +96,123 @@ describe('resolveOboToken', () => { }); }); + it.each([undefined, new Error('stopped by owner')])( + 'preserves pending lookup cancellation (%s)', + async (reason) => { + const controller = new AbortController(); + const lookup = jest.fn(() => new Promise(() => {})); + const result = resolveOboToken( + mockUser as IUser, + oboConfig, + mockResolver, + createLazyOboUpstreamTokenProvider(lookup, controller.signal), + ); + await Promise.resolve(); + controller.abort(reason); + await expect(result).rejects.toMatchObject({ name: 'AbortError' }); + expect(mockResolver).not.toHaveBeenCalled(); + expect(jest.requireMock('@librechat/data-schemas').logger.error).not.toHaveBeenCalled(); + }, + ); + + it('stops waiting for token retrieval that ignores cancellation', async () => { + const controller = new AbortController(); + let started!: () => void; + const retrieving = new Promise((resolve) => { + started = resolve; + }); + const provider = jest.fn(() => { + started(); + return new Promise(() => {}); + }); + const result = resolveOboToken( + mockUser as IUser, + oboConfig, + mockResolver, + createLazyOboUpstreamTokenProvider(async () => provider, controller.signal), + ); + await retrieving; + controller.abort(); + await expect(result).rejects.toMatchObject({ name: 'AbortError' }); + expect(mockResolver).not.toHaveBeenCalled(); + }); + + it('preserves cancellation during token retrieval and exchange', async () => { + const abort = new DOMException('Stopped', 'AbortError'); + const provider = jest.fn().mockRejectedValue(abort); + await expect( + resolveOboToken(mockUser as IUser, oboConfig, mockResolver, provider), + ).rejects.toBe(abort); + (mockResolver as jest.Mock).mockRejectedValueOnce(abort); + await expect( + resolveOboToken(mockUser as IUser, oboConfig, mockResolver, liveProvider), + ).rejects.toBe(abort); + expect(jest.requireMock('@librechat/data-schemas').logger.error).not.toHaveBeenCalled(); + }); + + it('retries failed lookups but shares each in-flight attempt', async () => { + const lookup = jest + .fn() + .mockRejectedValueOnce(Object.assign(new Error('unavailable'), { status: 503 })) + .mockResolvedValue(liveProvider); + const provider = createLazyOboUpstreamTokenProvider(lookup); + const failed = await Promise.allSettled([provider(), provider()]); + expect(failed.map((result) => result.status)).toEqual(['rejected', 'rejected']); + expect(lookup).toHaveBeenCalledTimes(1); + await expect( + resolveOboToken(mockUser as IUser, oboConfig, mockResolver, provider), + ).resolves.toMatchObject({ access_token: 'exchanged-mcp-token' }); + expect(lookup).toHaveBeenCalledTimes(2); + }); + + it.each([false, true])('preserves lookup failure retryability (%s)', async (retryable) => { + const lookup = jest + .fn() + .mockRejectedValue( + Object.assign(new Error('credential lookup failed'), { status: retryable ? 503 : 401 }), + ); + await expect( + resolveOboToken( + mockUser as IUser, + oboConfig, + mockResolver, + createLazyOboUpstreamTokenProvider(lookup), + ), + ).rejects.toMatchObject({ reason: 'session_refresh_failed', retryable }); + expect(mockResolver).not.toHaveBeenCalled(); + }); + + it('does not fall back to a snapshot when durable credential lookup returns no provider', async () => { + mockExtractOpenIDTokenInfo.mockReturnValue({ accessToken: 'stale-snapshot' }); + const lookup = jest.fn().mockResolvedValue(undefined); + await expect( + resolveOboToken( + mockUser as IUser, + oboConfig, + mockResolver, + createLazyOboUpstreamTokenProvider(lookup), + ), + ).rejects.toMatchObject({ reason: 'session_refresh_failed', retryable: false }); + expect(mockExtractOpenIDTokenInfo).not.toHaveBeenCalled(); + expect(mockResolver).not.toHaveBeenCalled(); + }); + + it('does not look up credentials after cancellation', async () => { + const controller = new AbortController(); + controller.abort(); + const lookup = jest.fn(); + await expect(createLazyOboUpstreamTokenProvider(lookup, controller.signal)()).rejects.toThrow(); + expect(lookup).not.toHaveBeenCalled(); + }); + + it('shares lookup while obtaining fresh tokens on every invocation', async () => { + const lookup = jest.fn().mockResolvedValue(liveProvider); + const provider = createLazyOboUpstreamTokenProvider(lookup); + await Promise.all([provider(), provider()]); + expect(lookup).toHaveBeenCalledTimes(1); + expect(liveProvider).toHaveBeenCalledTimes(2); + }); + it('throws missing_upstream_token when provider returns null and no federated fallback', async () => { await expect( resolveOboToken(mockUser as IUser, oboConfig, mockResolver, nullProvider), diff --git a/packages/api/src/mcp/oauth/obo.ts b/packages/api/src/mcp/oauth/obo.ts index 41b9b5baac..878dfee4ac 100644 --- a/packages/api/src/mcp/oauth/obo.ts +++ b/packages/api/src/mcp/oauth/obo.ts @@ -7,6 +7,8 @@ import type { OpenIDTokenInfo } from '~/utils/oidc'; import type { MCPOAuthTokens } from './types'; import { getSkewedTokenExpiresAtMs, getTokenExpiresAtMs } from '~/oauth/expiry'; import { extractOpenIDTokenInfo, isOpenIDTokenValid } from '~/utils/oidc'; +import { detachOnAbort } from '~/utils/promises'; +import { isAbortError } from '~/utils/errors'; export interface OboConfig { scopes: string; @@ -43,6 +45,82 @@ export type UpstreamTokenProvider = (options?: { signal?: AbortSignal; }) => Promise; +/** Lazily supplies a renewable upstream-token provider when an OBO server actually needs one. */ +export type UpstreamTokenProviderResolver = (options?: { + signal?: AbortSignal; +}) => UpstreamTokenProvider | undefined | Promise; + +/** Scheduled OBO credentials must not replace the browser's direct-bearer source. */ +export function selectMCPUpstreamTokenProvider({ + upstreamTokenProvider, + upstreamTokenProviderResolver, + createSessionProvider, +}: { + upstreamTokenProvider?: UpstreamTokenProvider | null; + upstreamTokenProviderResolver?: UpstreamTokenProviderResolver | null; + createSessionProvider: () => UpstreamTokenProvider; +}): UpstreamTokenProvider | null | undefined { + return upstreamTokenProviderResolver + ? upstreamTokenProvider + : (upstreamTokenProvider ?? createSessionProvider()); +} + +function normalizeOboCancellation(error: unknown, signal?: AbortSignal): unknown { + if (signal?.aborted && error === signal.reason && !isAbortError(error)) { + return Object.assign(new Error('The operation was aborted.', { cause: error }), { + name: 'AbortError', + }); + } + return error; +} + +/** Detach cancelled callers and preserve cancellation for arbitrary AbortController reasons. */ +export async function awaitOboOperation( + operation: Promise, + signal?: AbortSignal, +): Promise { + try { + return await detachOnAbort(operation, signal); + } catch (error) { + throw normalizeOboCancellation(error, signal); + } +} + +/** Keep lookup failures inside resolveOboToken's typed failure boundary. */ +export function createLazyOboUpstreamTokenProvider( + resolver: UpstreamTokenProviderResolver, + signal?: AbortSignal, +): UpstreamTokenProvider { + let pending: Promise | undefined; + return async (options) => { + const effectiveSignal = options?.signal ?? signal; + try { + effectiveSignal?.throwIfAborted(); + pending ??= Promise.resolve() + .then(() => { + effectiveSignal?.throwIfAborted(); + return resolver({ signal: effectiveSignal }); + }) + .catch((error) => { + pending = undefined; + throw error; + }); + const provider = await detachOnAbort(pending, effectiveSignal); + effectiveSignal?.throwIfAborted(); + if (!provider) { + pending = undefined; + throw new Error('Renewable upstream credentials are unavailable.'); + } + return await detachOnAbort( + provider({ ...options, signal: effectiveSignal }), + effectiveSignal, + ); + } catch (error) { + throw normalizeOboCancellation(error, effectiveSignal); + } + }; +} + export type OboTokenResolutionReason = | 'missing_upstream_token' | 'missing_upstream_access_token' @@ -212,6 +290,7 @@ export async function resolveOboToken( try { liveTokens = await upstreamTokenProvider(); } catch (error) { + if (isAbortError(error)) throw error; logger.error('[OBO] Upstream session refresh failed:', error); const retryable = isRetryableOboExchangeError(error); throw new OboTokenResolutionError( @@ -292,6 +371,7 @@ export async function resolveOboToken( expires_at: skewedExpiresAt, }; } catch (error) { + if (isAbortError(error)) throw error; if (error instanceof OboTokenResolutionError) { throw error; } diff --git a/packages/api/src/mcp/types/index.ts b/packages/api/src/mcp/types/index.ts index 68c7759743..ee6d02767e 100644 --- a/packages/api/src/mcp/types/index.ts +++ b/packages/api/src/mcp/types/index.ts @@ -20,7 +20,12 @@ import type { import type { SearchResultData, UIResource, TPlugin } from 'librechat-data-provider'; import type { TokenMethods, IUser } from '@librechat/data-schemas'; import type { LCTool } from '@librechat/agents'; -import type { OboTokenResolver, OboTrustChecker, UpstreamTokenProvider } from '~/mcp/oauth/obo'; +import type { + OboTokenResolver, + OboTrustChecker, + UpstreamTokenProvider, + UpstreamTokenProviderResolver, +} from '~/mcp/oauth/obo'; import type { AuthIdentityContext } from '~/utils/identity'; import type { GraphTokenResolver } from '~/utils/graph'; import type { FlowStateManager } from '~/flow/manager'; @@ -253,6 +258,8 @@ export interface UserConnectionContext { graphTokenResolver?: GraphTokenResolver; /** Live OpenID session credential source for trusted direct bearer and OBO configurations. */ upstreamTokenProvider?: UpstreamTokenProvider; + /** Deferred credential source used only after a server is confirmed to require OBO. */ + upstreamTokenProviderResolver?: UpstreamTokenProviderResolver; connectionTimeout?: number; /** Cancels the connection's SDK requests when the caller itself is cancelled; previously only * OAuth connections could carry a signal, leaving non-OAuth discovery uncancellable. */ @@ -362,6 +369,7 @@ export interface ToolDiscoveryOptions { oboTokenResolver?: OboTokenResolver; oboTrustChecker?: OboTrustChecker; upstreamTokenProvider?: UpstreamTokenProvider; + upstreamTokenProviderResolver?: UpstreamTokenProviderResolver; oboIdentityContext?: AuthIdentityContext; } diff --git a/packages/api/src/schedules/mcp.spec.ts b/packages/api/src/schedules/mcp.spec.ts index 803450d906..4def3a3d6c 100644 --- a/packages/api/src/schedules/mcp.spec.ts +++ b/packages/api/src/schedules/mcp.spec.ts @@ -1,11 +1,54 @@ import { AgentCapabilities, Permissions, PermissionTypes } from 'librechat-data-provider'; import type { IUser, IRole, AppConfig, AgentGraphNode } from '@librechat/data-schemas'; +import type { UpstreamTokenProvider } from '../mcp/oauth/obo'; import type { ParsedServerConfig } from '../mcp/types'; -import { createScheduleMCPPreflight, ScheduleMCPError } from './mcp'; +import { + bindUpstreamTokenProviderResolver, + createScheduleMCPPreflight, + ScheduleMCPError, +} from './mcp'; +import { OboTokenResolutionError, createLazyOboUpstreamTokenProvider } from '../mcp/oauth/obo'; const principal = { id: 'owner', role: 'USER' }; const server: ParsedServerConfig = { type: 'streamable-http', url: 'https://mcp.example.test/mcp' }; +it('retries through both run-bound and consumer lookup caches after failure', async () => { + const tokenProvider = jest.fn().mockResolvedValue({ access_token: 'fresh' }); + const lookup = jest + .fn() + .mockRejectedValueOnce(new Error('temporary unavailable')) + .mockResolvedValue(tokenProvider); + const bound = bindUpstreamTokenProviderResolver(principal as IUser, lookup)!; + const first = createLazyOboUpstreamTokenProvider(bound); + const second = createLazyOboUpstreamTokenProvider(bound); + const results = await Promise.allSettled([first(), second()]); + expect(results.map((r) => r.status)).toEqual(['rejected', 'rejected']); + expect(lookup).toHaveBeenCalledTimes(1); + await expect(Promise.all([first(), second()])).resolves.toEqual([ + { access_token: 'fresh' }, + { access_token: 'fresh' }, + ]); + expect(lookup).toHaveBeenCalledTimes(2); +}); + +it('shares credential lookup across sibling consumers without adopting child cancellation', async () => { + const owner = new AbortController(); + const child = new AbortController(); + child.abort(); + const provider = jest.fn(); + const lookup = jest.fn().mockResolvedValue(provider); + const resolve = bindUpstreamTokenProviderResolver(principal as IUser, lookup, owner.signal)!; + expect(lookup).not.toHaveBeenCalled(); + await expect(Promise.all([resolve({ signal: child.signal }), resolve()])).resolves.toEqual([ + provider, + provider, + ]); + expect(lookup).toHaveBeenCalledTimes(1); + expect(lookup).toHaveBeenCalledWith(principal, { signal: owner.signal }); + owner.abort(); + expect(() => resolve()).toThrow(); +}); + function graphNode(id: string, fields: Partial = {}): AgentGraphNode { return { id, provider: 'openAI', model: 'gpt-test', ...fields }; } @@ -78,6 +121,69 @@ it('uses persisted identity with isolated connections and disposes them after di expect(disconnect).toHaveBeenCalledTimes(1); }); +it('lazily resolves an upstream token provider for an OBO preflight', async () => { + const { check, deps } = setup(); + deps.getServerConfigs = jest.fn(async () => ({ + docs: { ...server, obo: { scopes: 'api://mcp/.default' } }, + })); + const upstreamTokenProvider: UpstreamTokenProvider = jest.fn(async () => ({ + access_token: 'current-token', + })); + deps.resolveUpstreamTokenProvider = jest.fn(async () => upstreamTokenProvider); + const connect = deps.connect; + deps.connect = jest.fn(async (options) => { + await options.upstreamTokenProviderResolver?.({ signal: options.signal }); + return connect(options); + }); + + await check('agent', principal); + + expect(deps.resolveUpstreamTokenProvider).toHaveBeenCalledWith( + expect.objectContaining({ id: 'owner' }), + { signal: undefined }, + ); + expect(deps.connect).toHaveBeenCalledWith( + expect.objectContaining({ upstreamTokenProviderResolver: expect.any(Function) }), + ); +}); + +it('does not resolve upstream credentials for non-OBO servers', async () => { + const { check, deps } = setup(); + deps.resolveUpstreamTokenProvider = jest.fn(async () => { + throw new Error('credential service unavailable'); + }); + + await expect(check('agent', principal)).resolves.toEqual([{ server: 'docs', status: 'ready' }]); + expect(deps.resolveUpstreamTokenProvider).not.toHaveBeenCalled(); +}); + +it('does not expose an OBO provider to a sibling direct-bearer server', async () => { + const { check, deps } = setup(['search_mcp_obo', 'search_mcp_direct']); + deps.getServerConfigs = jest.fn(async () => ({ + obo: { ...server, obo: { scopes: 'api://mcp/.default' } }, + direct: { ...server, headers: { Authorization: 'Bearer {{LIBRECHAT_OPENID_ACCESS_TOKEN}}' } }, + })); + const upstreamTokenProvider: UpstreamTokenProvider = jest.fn(async () => ({ + access_token: 'current-token', + })); + deps.resolveUpstreamTokenProvider = jest.fn(async () => upstreamTokenProvider); + const connect = deps.connect; + deps.connect = jest.fn(async (options) => { + if (options.serverConfig?.obo) { + await options.upstreamTokenProviderResolver?.({ signal: options.signal }); + } + return connect(options); + }); + + await check('agent', principal); + + expect(deps.resolveUpstreamTokenProvider).toHaveBeenCalledTimes(1); + const directOptions = (deps.connect as jest.Mock).mock.calls.find( + ([options]) => options.serverName === 'direct', + )?.[0]; + expect(directOptions.upstreamTokenProvider).toBeUndefined(); +}); + it('rejects partial readiness and reports each server without exception details', async () => { const { check, deps, disconnect } = setup(['search_mcp_docs', 'read_mcp_private']); deps.getServerConfigs = async () => ({ docs: server, private: server }); @@ -130,6 +236,28 @@ it('classifies a transport outage as retryable', async () => { }); }); +it('classifies a permanent OBO credential failure as requiring reauthentication', async () => { + const { check, deps } = setup(); + deps.connect = async () => { + throw new OboTokenResolutionError('session_refresh_failed', 'Sign-in expired.'); + }; + + await expect(check('agent', principal)).rejects.toMatchObject({ + code: 'mcp_reauth_required', + }); +}); + +it('classifies a retryable OBO credential failure as unavailable', async () => { + const { check, deps } = setup(); + deps.connect = async () => { + throw new OboTokenResolutionError('session_refresh_failed', 'Refresh unavailable.', true); + }; + + await expect(check('agent', principal)).rejects.toMatchObject({ + code: 'mcp_unavailable', + }); +}); + it('checks graph agents once even when edges cycle', async () => { const { check, deps } = setup(); deps.getAgentGraphNodes = jest.fn(async (ids) => diff --git a/packages/api/src/schedules/mcp.ts b/packages/api/src/schedules/mcp.ts index 7bbb1a9de0..63a7e4f8db 100644 --- a/packages/api/src/schedules/mcp.ts +++ b/packages/api/src/schedules/mcp.ts @@ -22,6 +22,7 @@ import type { AgentGraphAccessContext, } from '@librechat/data-schemas'; import type { TModelsConfig, ScheduleMCPStatus, ScheduleMCPOutcome } from 'librechat-data-provider'; +import type { UpstreamTokenProvider, UpstreamTokenProviderResolver } from '../mcp/oauth/obo'; import type { ParsedServerConfig, UserMCPConnectionOptions } from '../mcp/types'; import type { CheckAccessParams } from '../middleware/access'; import type { MCPToolsSnapshot } from '../mcp/connection'; @@ -42,12 +43,56 @@ import { MCPConfigInitializationCanceledError } from '../mcp/registry/MCPServers import { createMCPRequestContext, cleanupMCPRequestContext } from '../mcp/request'; import { getAppConfigOptionsFromUser } from '../app/service'; import { createConcurrencyLimiter } from '../utils/promise'; +import { OboTokenResolutionError } from '../mcp/oauth/obo'; import { OpenIDReauthRequiredError } from '../utils/oidc'; import { resolveReachableGraph } from '../agents/edges'; import { formatMCPServerTools } from '../mcp/tools'; import { checkAccess } from '../middleware/access'; import { detachOnAbort } from '../utils/promises'; import { getPluginAuthMap } from '../agents/auth'; +import { isScheduleFireRequest } from './trigger'; + +type HostUpstreamTokenProviderResolver = ( + user: IUser, + options: { signal?: AbortSignal }, +) => ReturnType; + +/** Bind a credential lookup to the trusted principal and owning run's cancellation. */ +export function bindUpstreamTokenProviderResolver( + user: IUser, + resolve: HostUpstreamTokenProviderResolver | undefined, + signal?: AbortSignal, +): UpstreamTokenProviderResolver | undefined { + if (!resolve) return undefined; + let pending: Promise | undefined; + return () => { + signal?.throwIfAborted(); + pending ??= Promise.resolve() + .then(() => { + signal?.throwIfAborted(); + return resolve(user, { signal }); + }) + .then((provider) => { + if (!provider) pending = undefined; + return provider; + }) + .catch((error) => { + pending = undefined; + throw error; + }); + return pending; + }; +} + +export function createScheduleUpstreamTokenProviderResolver( + req: Parameters[0] & { user: IUser }, + resolve: HostUpstreamTokenProviderResolver | undefined, + signal?: AbortSignal, +): UpstreamTokenProviderResolver | undefined { + return isScheduleFireRequest(req) + ? bindUpstreamTokenProviderResolver(req.user, resolve, signal) + : undefined; +} // The public schedule schema caps mcpPreflightConcurrency at 10. Keep the same // ceiling across every preflight owned by this process so concurrent schedules @@ -100,12 +145,16 @@ interface ScheduleMCPDeps { role?: string, ) => Promise>; findPluginAuthsByKeys: PluginAuthMethods['findPluginAuthsByKeys']; + resolveUpstreamTokenProvider?: ( + user: IUser, + options: { signal?: AbortSignal }, + ) => UpstreamTokenProvider | undefined | Promise; connect: (options: UserMCPConnectionOptions) => Promise<{ fetchToolsSnapshot: (deadlineMs?: number, signal?: AbortSignal) => Promise; }>; } -/** Probes only persisted identity and credentials, with isolated user connections and no OAuth wait. */ +/** Probes MCP readiness with isolated user connections and no interactive OAuth wait. */ export function createScheduleMCPPreflight(deps: ScheduleMCPDeps): ScheduleMCPPreflight { const sharedProbeLimit = createConcurrencyLimiter(MAX_SHARED_MCP_PREFLIGHT_CONCURRENCY); const runPreflight: ScheduleMCPPreflight = async (agentId, principal, options) => { @@ -571,6 +620,12 @@ export function createScheduleMCPPreflight(deps: ScheduleMCPDeps): ScheduleMCPPr throwError: true, findPluginAuthsByKeys: deps.findPluginAuthsByKeys, }); + const upstreamTokenProviderResolver = bindUpstreamTokenProviderResolver( + user, + deps.resolveUpstreamTokenProvider, + options.signal, + ); + throwIfAborted(); const requestBody = { messageId: randomUUID(), conversationId: randomUUID(), @@ -602,6 +657,7 @@ export function createScheduleMCPPreflight(deps: ScheduleMCPDeps): ScheduleMCPPr customUserVars, requestBody, requestScopedConnections: context, + upstreamTokenProviderResolver, ephemeralConnection: true, returnOnOAuth: true, oauthStart: async () => { @@ -651,6 +707,7 @@ export function createScheduleMCPPreflight(deps: ScheduleMCPDeps): ScheduleMCPPr reauth || error instanceof MCPAuthenticationRejectedError || error instanceof OpenIDReauthRequiredError || + (error instanceof OboTokenResolutionError && !error.retryable) || error instanceof MCPOAuthSecretReentryRequiredError || isOAuthAuthenticationError(error) ? 'mcp_reauth_required' diff --git a/packages/api/src/utils/errors.spec.ts b/packages/api/src/utils/errors.spec.ts index fde1e223c2..181eabf73f 100644 --- a/packages/api/src/utils/errors.spec.ts +++ b/packages/api/src/utils/errors.spec.ts @@ -1,4 +1,4 @@ -import { getSafeErrorMetadata, getSafeErrorText, isAbortError } from './errors'; +import { getSafeErrorMetadata, getSafeErrorText, isAbortError, isOwnedAbortError } from './errors'; describe('getSafeErrorMetadata', () => { it('keeps bounded diagnostic fields without serializing raw provider data', () => { @@ -182,3 +182,22 @@ describe('isAbortError', () => { expect(isAbortError('aborted')).toBe(false); }); }); + +describe('isOwnedAbortError', () => { + it.each([new Error('stopped'), 'stopped'])( + 'recognizes the owning signal reason (%s)', + (reason) => { + const controller = new AbortController(); + controller.abort(reason); + + expect(isOwnedAbortError(reason, controller.signal)).toBe(true); + }, + ); + + it('does not hide an unrelated failure that races cancellation', () => { + const controller = new AbortController(); + controller.abort(new Error('stopped')); + + expect(isOwnedAbortError(new Error('upstream failed'), controller.signal)).toBe(false); + }); +}); diff --git a/packages/api/src/utils/errors.ts b/packages/api/src/utils/errors.ts index 56eaae7813..e8dcc95c72 100644 --- a/packages/api/src/utils/errors.ts +++ b/packages/api/src/utils/errors.ts @@ -133,3 +133,8 @@ export function isAbortError(error: unknown): boolean { return false; } + +/** Whether an error represents cancellation by this specific owning signal. */ +export function isOwnedAbortError(error: unknown, signal?: AbortSignal | null): boolean { + return signal?.aborted === true && (isAbortError(error) || error === signal.reason); +} diff --git a/packages/api/src/utils/promises.spec.ts b/packages/api/src/utils/promises.spec.ts index cc3ddf2481..ee9ba11232 100644 --- a/packages/api/src/utils/promises.spec.ts +++ b/packages/api/src/utils/promises.spec.ts @@ -5,6 +5,10 @@ describe('detachOnAbort', () => { await expect(detachOnAbort(Promise.resolve('value'))).resolves.toBe('value'); }); + it('accepts an already-resolved operation value', async () => { + await expect(detachOnAbort('value', new AbortController().signal)).resolves.toBe('value'); + }); + it('resolves normally when the signal never aborts', async () => { const controller = new AbortController(); await expect(detachOnAbort(Promise.resolve('value'), controller.signal)).resolves.toBe('value'); diff --git a/packages/api/src/utils/promises.ts b/packages/api/src/utils/promises.ts index b51a8a9d24..e243d61ca1 100644 --- a/packages/api/src/utils/promises.ts +++ b/packages/api/src/utils/promises.ts @@ -12,7 +12,11 @@ * The detached promise's later settlement is swallowed rather than surfacing as * an unhandled rejection. */ -export function detachOnAbort(promise: Promise, signal?: AbortSignal | null): Promise { +export function detachOnAbort( + operation: T | PromiseLike, + signal?: AbortSignal | null, +): Promise { + const promise = Promise.resolve(operation); if (signal == null) { return promise; }