diff --git a/api/server/services/Tools/mcp.js b/api/server/services/Tools/mcp.js index 0070b65c0a..ab67338d31 100644 --- a/api/server/services/Tools/mcp.js +++ b/api/server/services/Tools/mcp.js @@ -4,6 +4,7 @@ const { getUserMCPAuthMap, getMissingCustomUserVars, loadMCPServerCatalogs: loadCatalogs, + resolveMCPReinitializeConfig, requiresEphemeralUserConnection, getMissingRuntimeBodyPlaceholderFields, MCPAuthenticationRejectedError, @@ -37,7 +38,6 @@ const { } = require('~/server/services/MCPAuthorizationFenceRetry'); const MCP_REINITIALIZE_FAILURE_REASONS = { - UNREACHABLE: 'unreachable', MISSING_CUSTOM_USER_VARS: 'missing_custom_user_vars', OAUTH_REQUIRED: 'oauth_required', INITIALIZATION_FAILED: 'initialization_failed', @@ -166,45 +166,17 @@ async function reinitMCPServer({ try { const registry = getMCPServersRegistry(); - serverConfig = - serverConfig ?? (await registry.getServerConfig(serverName, user?.id, configServers)); - ephemeralServer = serverConfig ? requiresEphemeralUserConnection(serverConfig) : false; - if (serverConfig?.inspectionFailed) { - if (serverConfig.source === 'config') { - logger.info( - '[MCP Reinitialize] Config-source server inspection failed; retry handled by config cache', - ); - return { - availableTools: null, - success: false, - message: `MCP server '${serverName}' is still unreachable`, - failureReason: MCP_REINITIALIZE_FAILURE_REASONS.UNREACHABLE, - oauthRequired: false, - serverName, - oauthUrl: null, - tools: null, - }; - } else { - logger.info('[MCP Reinitialize] Server inspection failed; attempting reinspection'); - try { - const storageLocation = serverConfig.source === 'user' ? 'DB' : 'CACHE'; - await registry.reinspectServer(serverName, storageLocation, user?.id); - logger.info('[MCP Reinitialize] Server reinspection succeeded'); - } catch { - logger.error('[MCP Reinitialize] Server reinspection failed'); - return { - availableTools: null, - success: false, - message: `MCP server '${serverName}' is still unreachable`, - failureReason: MCP_REINITIALIZE_FAILURE_REASONS.UNREACHABLE, - oauthRequired: false, - serverName, - oauthUrl: null, - tools: null, - }; - } - } + const resolution = await resolveMCPReinitializeConfig( + registry, + serverName, + serverConfig ?? (await registry.getServerConfig(serverName, user?.id, configServers)), + user?.id, + ); + if (resolution.result) { + return resolution.result; } + serverConfig = resolution.serverConfig; + ephemeralServer = serverConfig ? requiresEphemeralUserConnection(serverConfig) : false; const customUserVars = userMCPAuthMap?.[`${Constants.mcp_prefix}${serverName}`]; diff --git a/api/server/services/Tools/mcp.spec.js b/api/server/services/Tools/mcp.spec.js index e0f54659d7..9e6a745b97 100644 --- a/api/server/services/Tools/mcp.spec.js +++ b/api/server/services/Tools/mcp.spec.js @@ -445,6 +445,58 @@ describe('reinitMCPServer — customUserVars gating (issue #10969)', () => { }); }); +describe('reinitMCPServer — recovery of a server that failed inspection', () => { + const user = { id: 'user-123' }; + const serverName = 'Recovering'; + const stub = { + type: 'streamable-http', + url: 'https://recovering.example.com/mcp', + source: 'yaml', + inspectionFailed: true, + }; + const { getMCPServersRegistry } = require('~/config'); + + beforeEach(() => { + mockUpdateMCPServerTools.mockResolvedValue({}); + }); + + it('connects with the recovered config instead of the stub it read', async () => { + const recovered = { + type: 'streamable-http', + url: 'https://recovering.example.com/mcp', + source: 'yaml', + requiresOAuth: false, + }; + const recoverServerConfig = jest.fn().mockResolvedValue(recovered); + getMCPServersRegistry.mockReturnValueOnce({ recoverServerConfig }); + mockGetConnection.mockResolvedValue({ fetchTools: jest.fn().mockResolvedValue([]) }); + + const result = await reinitMCPServer({ user, serverName, serverConfig: stub }); + + expect(recoverServerConfig).toHaveBeenCalledWith(serverName, stub, user.id); + expect(mockGetConnection).toHaveBeenCalledWith( + expect.objectContaining({ serverName, serverConfig: recovered }), + ); + expect(result).toMatchObject({ success: true, serverName }); + }); + + it('reports the server unreachable without connecting while it cannot be recovered', async () => { + const recoverServerConfig = jest.fn().mockResolvedValue(undefined); + getMCPServersRegistry.mockReturnValueOnce({ recoverServerConfig }); + + const result = await reinitMCPServer({ user, serverName, serverConfig: stub }); + + expect(mockGetConnection).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + availableTools: null, + success: false, + message: `MCP server '${serverName}' is still unreachable`, + failureReason: 'unreachable', + tools: null, + }); + }); +}); + describe('reinitMCPServer — direct bearer authentication outcomes', () => { it('preserves a typed rejection instead of reducing it to a generic result', async () => { const { MCPAuthenticationRejectedError } = require('@librechat/api'); diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 440b1d375a..782597bb8b 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -34,6 +34,7 @@ export * from './mcp/authorization'; export * from './mcp/authorizationRetry'; export * from './mcp/assistants'; export * from './mcp/request'; +export * from './mcp/reinitialize'; export * from './mcp/icons'; /* Utilities */ export * from './mcp/utils'; diff --git a/packages/api/src/mcp/registry/MCPServersRegistry.ts b/packages/api/src/mcp/registry/MCPServersRegistry.ts index 1e6700d484..9e50ae8e7b 100644 --- a/packages/api/src/mcp/registry/MCPServersRegistry.ts +++ b/packages/api/src/mcp/registry/MCPServersRegistry.ts @@ -10,8 +10,8 @@ import { CONFIG_CACHE_NAMESPACE, } from './cache/ServerConfigsCacheFactory'; import { MCPInspectionFailedError, isMCPDomainNotAllowedError } from '~/mcp/errors'; +import { canBackfillSharedServerInstructions, isUserSourced } from '~/mcp/utils'; import { ReadThroughAllCache } from './cache/ReadThroughAllCache'; -import { canBackfillSharedServerInstructions } from '~/mcp/utils'; import { isPluginSourced, MCP_PLUGIN_SOURCE } from '~/utils/env'; import { ReadThroughCache } from './cache/ReadThroughCache'; import { MCPServerInspector } from './MCPServerInspector'; @@ -197,6 +197,14 @@ interface ResolvedMCPAllowlists { allowedAddresses?: string[] | null; } +/** The stored entry a reinspection reads and writes back to. */ +interface ReinspectionTarget { + configRepo: IServerConfigsRepositoryInterface; + serverName: string; + storageLocation: 'CACHE' | 'DB'; + userId?: string; +} + /** * Central registry for managing MCP server configurations. * Authoritative source of truth for all MCP servers provided by LibreChat. @@ -235,6 +243,13 @@ export class MCPServersRegistry { Promise >(); + /** In-flight reinspections, shared by callers that would inspect the same stored entry + * under the same allowlists. */ + private readonly pendingReinspections = new Map>(); + + /** The in-flight reinspection each settling flight waits on, by key, so waits never form a cycle. */ + private readonly reinspectionWaits = new Map(); + /** Memoized YAML server names — set once after boot-time init, never changes. */ private yamlServerNames: Set | null = null; private yamlServerNamesPromise: Promise> | null = null; @@ -652,48 +667,249 @@ export class MCPServersRegistry { } /** - * Re-inspects a server that previously failed initialization. - * Uses the stored stub config to attempt a full inspection and replaces the stub on success. + * Resolves a config that failed inspection to the config a connection should be made with. + * + * Once the server is reachable this is the stored, inspected config — whether this call + * recovered it or a concurrent request did, on this replica or another — so a caller that + * read the stub connects with what inspection found rather than the stub. Resolves undefined + * while the server is still unreachable. Config-tier stubs are left to `ensureConfigServers`, + * which retries them on its own schedule. + */ + public async recoverServerConfig( + serverName: string, + config: t.ParsedServerConfig, + userId?: string, + ): Promise { + if (!config.inspectionFailed) { + return config; + } + if (config.source === 'config') { + logger.info( + '[MCPServersRegistry] Config-source server inspection failed; retry handled by config cache', + ); + return undefined; + } + try { + const result = await this.reinspectServer( + serverName, + isUserSourced(config) ? 'DB' : 'CACHE', + userId, + ); + return result.config; + } catch { + logger.info('[MCPServersRegistry] Server is still unreachable after reinspection'); + return undefined; + } + } + + /** + * Re-inspects a server whose stored config failed inspection and replaces that stub with the + * inspected config. + * + * Inspection is a network round trip, and every caller that read the stub before a recovery + * was written would otherwise inspect and write again. Each write bumps `updatedAt`, which + * marks connections made after the previous write stale, so a caller already holding one + * reads an empty tool list. Callers in this process that read the same stub and are judged + * against the same allowlists therefore share one inspection and one write, and the write + * replaces only the stub that was inspected, so a replica that lost the race writes nothing. + * + * Every decision reads the entry past the store's process-local snapshot. An entry that is no + * longer failed was already recovered and resolves to the stored config without another + * inspection, including when that recovery is found only after this call's own inspection + * failed. A stub that a registry re-initialization replaced while it was inspected resolves + * through an inspection of the newer stub. A server that is still unreachable rejects with + * `MCPInspectionFailedError`; an allowlist rejection is rethrown as is. */ public async reinspectServer( serverName: string, storageLocation: 'CACHE' | 'DB', userId?: string, ): Promise { - const configRepo = this.getConfigRepository(storageLocation); - const existing = await configRepo.get(serverName, userId); - if (!existing) { - throw new Error(`Server "${serverName}" not found in ${storageLocation} for reinspection.`); - } - if (!existing.inspectionFailed) { - throw new Error( - `Server "${serverName}" is not in a failed state. Use updateServer() instead.`, - ); - } - - const { inspectionFailed: _, ...configForInspection } = existing; + const target: ReinspectionTarget = { + configRepo: this.getConfigRepository(storageLocation), + serverName, + storageLocation, + userId, + }; const { allowedDomains, allowedAddresses } = await this.resolveAllowlists({ userId }); + const entry = await this.getReinspectionEntry(target); + if (!entry.inspectionFailed) { + return { serverName, config: entry }; + } + return this.joinReinspection(target, { allowedDomains, allowedAddresses }, entry); + } + + /** + * Shares one inspection and write of `stub` among callers judged against the same allowlists. + * A flight settling against another stub passes its own key as `waiter` and joins that stub's + * flight, unless the flight already waits on the waiter; then the waiter inspects it itself. + */ + private async joinReinspection( + target: ReinspectionTarget, + allowlists: ResolvedMCPAllowlists, + stub: t.ParsedServerConfig, + waiter?: string, + ): Promise { + const key = this.reinspectionKey(target, allowlists, stub); + if (waiter != null && this.waitsOn(key, waiter)) { + return this.reinspectStub(target, allowlists, stub, waiter); + } + const pending = this.pendingReinspections.get(key); + const reinspection = pending ?? this.reinspectStub(target, allowlists, stub, key); + if (!pending) { + this.pendingReinspections.set(key, reinspection); + } + if (waiter != null) { + this.reinspectionWaits.set(waiter, key); + } + try { + return await reinspection; + } finally { + if (waiter != null && this.reinspectionWaits.get(waiter) === key) { + this.reinspectionWaits.delete(waiter); + } + if (!pending && this.pendingReinspections.get(key) === reinspection) { + this.pendingReinspections.delete(key); + } + } + } + + /** Whether the flight for `key` is `waiter` or waits on it, directly or through other flights. */ + private waitsOn(key: string, waiter: string): boolean { + for ( + let next: string | undefined = key; + next != null; + next = this.reinspectionWaits.get(next) + ) { + if (next === waiter) { + return true; + } + } + return false; + } + + private async reinspectStub( + target: ReinspectionTarget, + allowlists: ResolvedMCPAllowlists, + stub: t.ParsedServerConfig, + flightKey: string, + ): Promise { + const { serverName, storageLocation, userId } = target; + const { inspectionFailed: _, ...configForInspection } = stub; let parsedConfig: t.ParsedServerConfig; try { parsedConfig = await MCPServerInspector.inspect( serverName, configForInspection, undefined, - allowedDomains, - allowedAddresses, + allowlists.allowedDomains, + allowlists.allowedAddresses, ); } catch (error) { logger.error('[MCPServersRegistry] Server reinspection failed'); if (isMCPDomainNotAllowedError(error)) { throw error; } - throw new MCPInspectionFailedError(serverName, error as Error); + return this.resolveStoredEntry( + target, + allowlists, + stub, + flightKey, + new MCPInspectionFailedError(serverName, error as Error), + ); } + const stored = await this.replaceStub(target, stub, parsedConfig); + if (stored) { + await this.invalidateServerReadCaches(serverName, userId, storageLocation); + return { serverName, config: stored }; + } + return this.resolveStoredEntry( + target, + allowlists, + stub, + flightKey, + new MCPInspectionFailedError( + serverName, + new Error('Storage did not replace the inspected stub'), + ), + ); + } + + /** + * Settles a reinspection whose own inspection did not replace `stub` against the entry stored + * now. A recovery another writer stored is the outcome, and this replica's read caches drop + * the stub they may still memoize from before it. A different stub from a registry + * re-initialization is settled through its own flight, which this one joins when another + * request already started it. A stub still in place rejects with `failure`. + */ + private async resolveStoredEntry( + target: ReinspectionTarget, + allowlists: ResolvedMCPAllowlists, + stub: t.ParsedServerConfig, + flightKey: string, + failure: MCPInspectionFailedError, + ): Promise { + const { serverName, storageLocation, userId } = target; + const current = await this.getReinspectionEntry(target); + if (!current.inspectionFailed) { + await this.invalidateServerReadCaches(serverName, userId, storageLocation); + return { serverName, config: current }; + } + if (current.updatedAt === stub.updatedAt) { + throw failure; + } + return this.joinReinspection(target, allowlists, current, flightKey); + } + + private async getReinspectionEntry({ + configRepo, + serverName, + storageLocation, + userId, + }: ReinspectionTarget): Promise { + const entry = configRepo.getCurrent + ? await configRepo.getCurrent(serverName, userId) + : await configRepo.get(serverName, userId); + if (!entry) { + throw new Error(`Server "${serverName}" not found in ${storageLocation} for reinspection.`); + } + return entry; + } + + /** Writes an inspected config over the stub it came from; undefined when another writer + * replaced the stub first. DB storage holds no startup stubs, so it keeps a plain update. */ + private async replaceStub( + { configRepo, serverName, userId }: ReinspectionTarget, + stub: t.ParsedServerConfig, + parsedConfig: t.ParsedServerConfig, + ): Promise { + if (configRepo.replaceStub) { + return configRepo.replaceStub(serverName, parsedConfig, stub.updatedAt); + } const updatedConfig = { ...parsedConfig, updatedAt: Date.now() }; await configRepo.update(serverName, updatedConfig, userId); - await this.invalidateServerReadCaches(serverName, userId, storageLocation); - return { serverName, config: updatedConfig }; + return updatedConfig; + } + + /** + * Identity of a reinspection: the stored stub it inspects — a DB entry is visible per user and + * tenant — and the allowlists it is judged against, so neither a user's DB visibility nor a + * tenant's allowlist decision reaches another caller through a shared inspection, and a newer + * stub is never answered by an inspection of the one it replaced. + */ + private reinspectionKey( + { serverName, storageLocation, userId }: ReinspectionTarget, + allowlists: ResolvedMCPAllowlists, + stub: t.ParsedServerConfig, + ): string { + return JSON.stringify([ + storageLocation, + storageLocation === 'DB' ? this.getReadThroughCacheKey(serverName, userId) : serverName, + stub.updatedAt ?? null, + allowlists.allowedDomains ?? null, + allowlists.allowedAddresses ?? null, + ]); } /** diff --git a/packages/api/src/mcp/registry/ServerConfigsRepositoryInterface.ts b/packages/api/src/mcp/registry/ServerConfigsRepositoryInterface.ts index 3a855ecf80..42d78c8fed 100644 --- a/packages/api/src/mcp/registry/ServerConfigsRepositoryInterface.ts +++ b/packages/api/src/mcp/registry/ServerConfigsRepositoryInterface.ts @@ -28,12 +28,34 @@ export interface IServerConfigsRepositoryInterface { expectedUpdatedAt?: number, ): Promise; + /** + * Replaces a failed-inspection stub with the config inspected from it, stamping a + * new `updatedAt` as `update` does. The write lands only while the stored entry is + * still that stub — `inspectionFailed` and carrying `stubUpdatedAt` — so an + * inspection that raced another replica's recovery or a registry re-initialization + * never overwrites the newer entry, and never bumps `updatedAt` past connections + * made from it. Returns the stored config, or undefined when the entry is no longer + * that stub. Optional: the in-memory and Redis aggregate-key stores behind the YAML + * tier, the only tier that holds startup stubs, implement it. + */ + replaceStub?( + serverName: string, + config: ParsedServerConfig, + stubUpdatedAt: number | undefined, + ): Promise; + //ACL Entry check if remove is possible remove(serverName: string, userId?: string): Promise; //ACL Entry check if read is possible get(serverName: string, userId?: string): Promise; + /** + * Reads an entry past any process-local snapshot, for a decision that must see a write another + * replica just made. Optional: stores whose `get` already reads current state omit it. + */ + getCurrent?(serverName: string, userId?: string): Promise; + //ACL Entry get all accessible mcp config definitions + any mcp configured with agents getAll(userId?: string, role?: string): Promise>; diff --git a/packages/api/src/mcp/registry/__tests__/MCPReinitRecovery.integration.test.ts b/packages/api/src/mcp/registry/__tests__/MCPReinitRecovery.integration.test.ts index 6c7cca6393..c248b7f25e 100644 --- a/packages/api/src/mcp/registry/__tests__/MCPReinitRecovery.integration.test.ts +++ b/packages/api/src/mcp/registry/__tests__/MCPReinitRecovery.integration.test.ts @@ -28,6 +28,7 @@ import type { Socket } from 'net'; import type * as t from '~/mcp/types'; import { registryStatusCache } from '~/mcp/registry/cache/RegistryStatusCache'; import { MCPServersInitializer } from '~/mcp/registry/MCPServersInitializer'; +import { MCPServerInspector } from '~/mcp/registry/MCPServerInspector'; import { MCPServersRegistry } from '~/mcp/registry/MCPServersRegistry'; import { ConnectionsRepository } from '~/mcp/ConnectionsRepository'; import { MCPInspectionFailedError } from '~/mcp/errors'; @@ -154,6 +155,37 @@ interface TestServer { close: () => Promise; } +/** + * Lets the first inspection run and holds every later one until released, so a test can connect + * to the first recovery before a second inspection would write. + */ +function holdLaterInspections(): { + firstStarted: Promise; + release: () => void; + calls: () => number; +} { + const inspect = MCPServerInspector.inspect.bind(MCPServerInspector); + let release!: () => void; + const released = new Promise((resolve) => { + release = resolve; + }); + let markFirstStarted!: () => void; + const firstStarted = new Promise((resolve) => { + markFirstStarted = resolve; + }); + const spy = jest + .spyOn(MCPServerInspector, 'inspect') + .mockImplementation(async (...args: Parameters) => { + if (spy.mock.calls.length === 1) { + markFirstStarted(); + } else { + await released; + } + return inspect(...args); + }); + return { firstStarted, release, calls: () => spy.mock.calls.length }; +} + interface ReinitOutcome { success: boolean; /** Tool count from a complete snapshot; `null` when the snapshot was incomplete */ @@ -358,7 +390,7 @@ describe('MCP reinitialize recovery – integration (issue #12143)', () => { expect(config2!.inspectionFailed).toBe(true); }); - it('concurrent reinspectServer calls should not crash or corrupt state', async () => { + it('concurrent reinspectServer calls share one inspection and all recover', async () => { const deadPort = await getFreePort(); await MCPServersInitializer.initialize({ 'race-server': { @@ -369,34 +401,25 @@ describe('MCP reinitialize recovery – integration (issue #12143)', () => { expect((await registry.getServerConfig('race-server'))!.inspectionFailed).toBe(true); server = await createMCPServerOnPort(deadPort); + const inspectSpy = jest.spyOn(MCPServerInspector, 'inspect'); // Simulate multiple users clicking Reinitialize at the same time. - // reinitMCPServer calls reinspectServer internally — this tests the critical section. - const n = 3 + Math.floor(Math.random() * 8); // 3–10 concurrent calls - const results = await Promise.allSettled( - Array.from({ length: n }, () => registry.reinspectServer('race-server', 'CACHE')), + const results = await Promise.all( + Array.from({ length: 5 }, () => registry.reinspectServer('race-server', 'CACHE')), ); - const successes = results.filter((r) => r.status === 'fulfilled'); - const failures = results.filter((r) => r.status === 'rejected'); - - // At least one must succeed - expect(successes.length).toBeGreaterThanOrEqual(1); - - // Any failure must be the "not in a failed state" guard (the first call already - // replaced the stub), not an unhandled crash or data corruption. - for (const f of failures) { - expect((f as PromiseRejectedResult).reason.message).toMatch(/not in a failed state/); + expect(inspectSpy).toHaveBeenCalledTimes(1); + for (const result of results) { + expect(result.config).toEqual(results[0].config); } - // Final state must be fully recovered regardless of how many succeeded const config = await registry.getServerConfig('race-server'); - expect(config).toBeDefined(); + expect(config).toEqual(results[0].config); expect(config!.inspectionFailed).toBeUndefined(); expect(config!.tools).toContain('echo'); }); - it('concurrent reinitMCPServer-equivalent flows should not crash or corrupt state', async () => { + it('concurrent reinitMCPServer-equivalent flows all connect with the recovered tools', async () => { const deadPort = await getFreePort(); const serverName = 'concurrent-reinit'; const configs: t.MCPServers = { @@ -421,22 +444,16 @@ describe('MCP reinitialize recovery – integration (issue #12143)', () => { const flowManager = new FlowStateManager(new Keyv(), { ttl: 60_000 }); /** - * Replicate reinitMCPServer logic: check inspectionFailed → reinspect → getConnection → - * tools snapshot. Each call uses a distinct user to simulate concurrent requests from - * different users. Concurrent reinspections can each succeed, and a later one replaces the - * app connection an earlier call was handed; that call's snapshot is incomplete, which - * reinitMCPServer answers by keeping the cached tools, reported here as `tools: null`. + * Replicate reinitMCPServer logic: recover a failed config → getConnection → tools snapshot. + * Each call uses a distinct user to simulate concurrent requests from different users. */ async function simulateReinitMCPServer(): Promise { const user = makeUser(); - const config = await registry.getServerConfig(serverName, user.id); + let config = await registry.getServerConfig(serverName, user.id); if (config?.inspectionFailed) { - try { - const storageLocation = config.dbId ? 'DB' : 'CACHE'; - await registry.reinspectServer(serverName, storageLocation, user.id); - } catch { - // Mirrors reinitMCPServer early return on failed reinspection - return { success: false, tools: 0 }; + config = await registry.recoverServerConfig(serverName, config, user.id); + if (!config) { + return { success: false, tools: null }; } } @@ -445,38 +462,17 @@ describe('MCP reinitialize recovery – integration (issue #12143)', () => { user, flowManager, forceNew: true, + serverConfig: config, }); const snapshot = await connection.fetchToolsSnapshot(); return { success: true, tools: snapshot.complete ? snapshot.tools.length : null }; } - const n = 3 + Math.floor(Math.random() * 5); // 3–7 concurrent calls - const results = await Promise.allSettled( - Array.from({ length: n }, () => simulateReinitMCPServer()), - ); + const results = await Promise.all(Array.from({ length: 5 }, () => simulateReinitMCPServer())); - // All promises should resolve (no unhandled throws) - for (const r of results) { - expect(r.status).toBe('fulfilled'); - } - - const values = (results as PromiseFulfilledResult[]).map((r) => r.value); - - // At least one full reinit must succeed with tools - const succeeded = values.filter((v) => v.success); - expect(succeeded.length).toBeGreaterThanOrEqual(1); - - // A connection created after the last reinspection is never replaced, so one snapshot completes - const listed = succeeded.filter((v) => v.tools !== null); - expect(listed.length).toBeGreaterThanOrEqual(1); - for (const s of listed) { - expect(s.tools).toBe(2); - } - - // Any that returned success: false hit the reinspect guard — that's fine - const earlyReturned = values.filter((v) => !v.success); - expect(earlyReturned.every((v) => v.tools === 0)).toBe(true); + // One recovery is written before any flow connects, so no connection is replaced mid-snapshot + expect(results).toEqual(Array.from({ length: 5 }, () => ({ success: true, tools: 2 }))); // Final registry state must be fully recovered const finalConfig = await registry.getServerConfig(serverName); @@ -500,7 +496,7 @@ describe('MCP reinitialize recovery – integration (issue #12143)', () => { await registry.reinspectServer(serverName, 'CACHE', firstUser.id); const held = await mcpManager.getConnection({ serverName, user: firstUser, flowManager }); - // What another replica's reinspection or an admin edit does: store a newer config + // What an admin edit does: store a newer config await registry.updateServer(serverName, { type: 'streamable-http', url: server.url }, 'CACHE'); const replacement = await mcpManager.getConnection({ serverName, @@ -517,6 +513,118 @@ describe('MCP reinitialize recovery – integration (issue #12143)', () => { expect(current.tools.map((tool) => tool.name).sort()).toEqual(['echo', 'greet']); }); + it('keeps a connection made after recovery current when another request reinspects mid-flight', async () => { + const serverName = 'mid-flight-reinspection'; + server = await createMCPServerOnPort(await getFreePort()); + (MCPManager as unknown as { instance: null }).instance = null; + const mcpManager = await MCPManager.createInstance({}); + await registry.addServerStub(serverName, { type: 'streamable-http', url: server.url }, 'CACHE'); + const flowManager = new FlowStateManager(new Keyv(), { ttl: 60_000 }); + const inspections = holdLaterInspections(); + + const first = registry.reinspectServer(serverName, 'CACHE'); + await inspections.firstStarted; + const second = registry.reinspectServer(serverName, 'CACHE'); + const firstResult = await first; + const recoveredConnection = await mcpManager.getConnection({ + serverName, + user: makeUser(), + flowManager, + }); + + inspections.release(); + const secondResult = await second; + const laterConnection = await mcpManager.getConnection({ + serverName, + user: makeUser(), + flowManager, + }); + const stored = await registry.getServerConfig(serverName); + const snapshot = await recoveredConnection.fetchToolsSnapshot(); + + expect(inspections.calls()).toBe(1); + expect(secondResult.config).toEqual(firstResult.config); + expect(stored).toEqual(firstResult.config); + expect(recoveredConnection.isStale(stored!.updatedAt!)).toBe(false); + expect(laterConnection).toBe(recoveredConnection); + expect(snapshot.complete).toBe(true); + expect(snapshot.tools.map((tool) => tool.name).sort()).toEqual(['echo', 'greet']); + }); + + it('keeps the first recovery when a reinspection under other allowlists finishes later', async () => { + const serverName = 'allowlist-reinspection'; + (MCPServersRegistry as unknown as { instance: undefined }).instance = undefined; + registry = MCPServersRegistry.createInstance( + mockMongoose, + ['127.0.0.1'], + undefined, + async (ctx) => ({ + allowedDomains: ctx?.userId === 'user-b' ? ['127.0.0.1', 'localhost'] : ['127.0.0.1'], + allowedAddresses: null, + }), + ); + server = await createMCPServerOnPort(await getFreePort()); + (MCPManager as unknown as { instance: null }).instance = null; + const mcpManager = await MCPManager.createInstance({}); + await registry.addServerStub(serverName, { type: 'streamable-http', url: server.url }, 'CACHE'); + const flowManager = new FlowStateManager(new Keyv(), { ttl: 60_000 }); + const inspections = holdLaterInspections(); + + const first = registry.reinspectServer(serverName, 'CACHE', 'user-a'); + await inspections.firstStarted; + const second = registry.reinspectServer(serverName, 'CACHE', 'user-b'); + const firstResult = await first; + const recoveredConnection = await mcpManager.getConnection({ + serverName, + user: makeUser(), + flowManager, + }); + + inspections.release(); + const secondResult = await second; + const laterConnection = await mcpManager.getConnection({ + serverName, + user: makeUser(), + flowManager, + }); + const snapshot = await recoveredConnection.fetchToolsSnapshot(); + + expect(inspections.calls()).toBe(2); + expect(secondResult.config).toEqual(firstResult.config); + await expect(registry.getServerConfig(serverName)).resolves.toEqual(firstResult.config); + expect(laterConnection).toBe(recoveredConnection); + expect(snapshot.complete).toBe(true); + expect(snapshot.tools.map((tool) => tool.name).sort()).toEqual(['echo', 'greet']); + }); + + it('connects a request that read the stub before another request recovered the server', async () => { + const serverName = 'stale-stub-reinspection'; + server = await createMCPServerOnPort(await getFreePort()); + (MCPManager as unknown as { instance: null }).instance = null; + const mcpManager = await MCPManager.createInstance({}); + await registry.addServerStub(serverName, { type: 'streamable-http', url: server.url }, 'CACHE'); + const flowManager = new FlowStateManager(new Keyv(), { ttl: 60_000 }); + const user = makeUser(); + + const staleStub = await registry.getServerConfig(serverName, user.id); + const { config: recovered } = await registry.reinspectServer(serverName, 'CACHE'); + const inspectSpy = jest.spyOn(MCPServerInspector, 'inspect'); + const serverConfig = await registry.recoverServerConfig(serverName, staleStub!, user.id); + const connection = await mcpManager.getConnection({ + serverName, + user, + flowManager, + serverConfig, + }); + const snapshot = await connection.fetchToolsSnapshot(); + + expect(staleStub!.inspectionFailed).toBe(true); + expect(inspectSpy).not.toHaveBeenCalled(); + expect(serverConfig).toEqual(recovered); + expect(snapshot.complete).toBe(true); + expect(snapshot.tools.map((tool) => tool.name).sort()).toEqual(['echo', 'greet']); + }); + it('reinspectServer should throw MCPInspectionFailedError when the server is still unreachable', async () => { const deadPort = await getFreePort(); const configs: t.MCPServers = { diff --git a/packages/api/src/mcp/registry/__tests__/MCPServersRegistry.cache_integration.spec.ts b/packages/api/src/mcp/registry/__tests__/MCPServersRegistry.cache_integration.spec.ts index 85fd7105ce..4c7bcd2dc5 100644 --- a/packages/api/src/mcp/registry/__tests__/MCPServersRegistry.cache_integration.spec.ts +++ b/packages/api/src/mcp/registry/__tests__/MCPServersRegistry.cache_integration.spec.ts @@ -343,4 +343,102 @@ describe('MCPServersRegistry Redis Integration Tests', () => { } }); }); + + describe('reinspection across replicas', () => { + it('lands one recovery when two replicas reinspect the same stub', async () => { + const replica = new MCPServersRegistry({} as typeof import('mongoose'), [ + 'replica.example.com', + ]); + await registry.addServerStub('recovering_server', testRawConfig, 'CACHE'); + const replicaMemo = await replica.getAllServerConfigs(); + + const inspect = jest.mocked(MCPServerInspector.inspect).getMockImplementation()!; + let releaseReplica!: () => void; + const replicaHeld = new Promise((resolve) => { + releaseReplica = resolve; + }); + let markReplicaInspecting!: () => void; + const replicaInspecting = new Promise((resolve) => { + markReplicaInspecting = resolve; + }); + jest + .spyOn(MCPServerInspector, 'inspect') + .mockImplementation(async (serverName, rawConfig, connection, domains, addresses) => { + if (domains?.includes('replica.example.com')) { + markReplicaInspecting(); + await replicaHeld; + } + return inspect(serverName, rawConfig, connection, domains, addresses); + }); + + const replicaReinspection = replica.reinspectServer('recovering_server', 'CACHE'); + await replicaInspecting; + const { config: recovered } = await registry.reinspectServer('recovering_server', 'CACHE'); + releaseReplica(); + const { config: replicaConfig } = await replicaReinspection; + + expect(recovered.inspectionFailed).toBeUndefined(); + expect(replicaConfig).toEqual(recovered); + expect(await registry['cacheConfigsRepo'].get('recovering_server')).toEqual(recovered); + expect(replicaMemo.recovering_server.inspectionFailed).toBe(true); + expect((await replica.getAllServerConfigs()).recovering_server).toEqual(recovered); + }); + + it('settles a failed inspection against a recovery another replica stored', async () => { + const replica = new MCPServersRegistry({} as typeof import('mongoose'), [ + 'replica.example.com', + ]); + await registry.addServerStub('flaky_server', testRawConfig, 'CACHE'); + + const inspect = jest.mocked(MCPServerInspector.inspect).getMockImplementation()!; + let failReplica!: () => void; + const replicaHeld = new Promise((resolve) => { + failReplica = resolve; + }); + let markReplicaInspecting!: () => void; + const replicaInspecting = new Promise((resolve) => { + markReplicaInspecting = resolve; + }); + jest + .spyOn(MCPServerInspector, 'inspect') + .mockImplementation(async (serverName, rawConfig, connection, domains, addresses) => { + if (domains?.includes('replica.example.com')) { + markReplicaInspecting(); + await replicaHeld; + throw new Error('connect ECONNREFUSED'); + } + return inspect(serverName, rawConfig, connection, domains, addresses); + }); + + const replicaReinspection = replica.reinspectServer('flaky_server', 'CACHE'); + await replicaInspecting; + const { config: recovered } = await registry.reinspectServer('flaky_server', 'CACHE'); + await expect(replica['cacheConfigsRepo'].get('flaky_server')).resolves.toMatchObject({ + inspectionFailed: true, + }); + failReplica(); + + await expect(replicaReinspection).resolves.toEqual({ + serverName: 'flaky_server', + config: recovered, + }); + }); + + it('returns a recovery another replica stored without inspecting while its snapshot holds the stub', async () => { + const replica = new MCPServersRegistry({} as typeof import('mongoose')); + await registry.addServerStub('recovered_server', testRawConfig, 'CACHE'); + await expect(replica['cacheConfigsRepo'].get('recovered_server')).resolves.toMatchObject({ + inspectionFailed: true, + }); + const { config: recovered } = await registry.reinspectServer('recovered_server', 'CACHE'); + const inspectSpy = jest.spyOn(MCPServerInspector, 'inspect'); + inspectSpy.mockClear(); + + await expect(replica.reinspectServer('recovered_server', 'CACHE')).resolves.toEqual({ + serverName: 'recovered_server', + config: recovered, + }); + expect(inspectSpy.mock.calls).toHaveLength(0); + }); + }); }); diff --git a/packages/api/src/mcp/registry/__tests__/MCPServersRegistry.test.ts b/packages/api/src/mcp/registry/__tests__/MCPServersRegistry.test.ts index 37d03e1869..3192792cd2 100644 --- a/packages/api/src/mcp/registry/__tests__/MCPServersRegistry.test.ts +++ b/packages/api/src/mcp/registry/__tests__/MCPServersRegistry.test.ts @@ -1,11 +1,14 @@ import './helpers/setupCredsEnv'; import { logger } from '@librechat/data-schemas'; +import { setImmediate as realSetImmediate } from 'timers'; import type * as t from '~/mcp/types'; import { MCPServersRegistry, MCPConfigInitializationCanceledError, } from '~/mcp/registry/MCPServersRegistry'; +import { ServerConfigsCacheInMemory } from '~/mcp/registry/cache/ServerConfigsCacheInMemory'; import { MCPServerInspector } from '~/mcp/registry/MCPServerInspector'; +import { MCPInspectionFailedError } from '~/mcp/errors'; import { processMCPEnv } from '~/utils/env'; // Mock MCPServerInspector to avoid actual server connections @@ -730,12 +733,30 @@ describe('MCPServersRegistry', () => { }); describe('reinspectServer', () => { - it('should throw when called on a healthy (non-stub) server', async () => { - await registry.addServer('healthy_server', testParsedConfig, 'CACHE'); + const stubOptions: t.MCPOptions = { + type: 'streamable-http', + url: 'https://recovering.example.com/mcp', + }; - await expect(registry.reinspectServer('healthy_server', 'CACHE')).rejects.toThrow( - 'is not in a failed state', - ); + beforeEach(() => { + /** The inspector is a module automock, so its recorded calls outlive each test. */ + jest.mocked(MCPServerInspector.inspect).mockClear(); + }); + + afterEach(() => { + jest.setSystemTime(new Date(FIXED_TIME)); + }); + + it('resolves a server that is no longer failed to its stored config without inspecting', async () => { + const { config } = await registry.addServer('healthy_server', testParsedConfig, 'CACHE'); + const inspectSpy = jest.spyOn(MCPServerInspector, 'inspect'); + inspectSpy.mockClear(); + + await expect(registry.reinspectServer('healthy_server', 'CACHE')).resolves.toEqual({ + serverName: 'healthy_server', + config, + }); + expect(inspectSpy).not.toHaveBeenCalled(); }); it('should throw when the server does not exist', async () => { @@ -743,6 +764,549 @@ describe('MCPServersRegistry', () => { 'not found in CACHE', ); }); + + it('shares one inspection and one write among concurrent callers', async () => { + await registry.addServerStub('stub_server', stubOptions, 'CACHE'); + const inspectSpy = jest.spyOn(MCPServerInspector, 'inspect'); + const replaceSpy = jest.spyOn(ServerConfigsCacheInMemory.prototype, 'replaceStub'); + + const [first, ...joined] = await Promise.all( + Array.from({ length: 3 }, () => registry.reinspectServer('stub_server', 'CACHE')), + ); + + expect(inspectSpy).toHaveBeenCalledTimes(1); + expect(replaceSpy).toHaveBeenCalledTimes(1); + expect(joined).toEqual([first, first]); + expect(first.config.inspectionFailed).toBeUndefined(); + await expect(registry['cacheConfigsRepo'].get('stub_server')).resolves.toEqual(first.config); + }); + + it('inspects separately under different allowlists and keeps the recovery that landed first', async () => { + (MCPServersRegistry as unknown as { instance: undefined }).instance = undefined; + const tenantRegistry = MCPServersRegistry.createInstance( + mockMongoose, + null, + null, + async (ctx) => ({ allowedDomains: [`${ctx?.userId}.example.com`], allowedAddresses: null }), + ); + await tenantRegistry.reset(); + await tenantRegistry.addServerStub('stub_server', stubOptions, 'CACHE'); + + const inspect = jest.mocked(MCPServerInspector.inspect).getMockImplementation()!; + let releaseTenantA!: () => void; + const tenantAHeld = new Promise((resolve) => { + releaseTenantA = resolve; + }); + let markTenantAInspecting!: () => void; + const tenantAInspecting = new Promise((resolve) => { + markTenantAInspecting = resolve; + }); + const inspectSpy = jest + .spyOn(MCPServerInspector, 'inspect') + .mockImplementation(async (serverName, rawConfig, connection, domains, addresses) => { + const tenant = domains?.[0]; + if (tenant === 'tenant-a.example.com') { + markTenantAInspecting(); + await tenantAHeld; + } else { + await tenantAInspecting; + } + const parsed = await inspect(serverName, rawConfig, connection, domains, addresses); + return { ...parsed, description: tenant }; + }); + + const tenantA = tenantRegistry.reinspectServer('stub_server', 'CACHE', 'tenant-a'); + const tenantB = await tenantRegistry.reinspectServer('stub_server', 'CACHE', 'tenant-b'); + releaseTenantA(); + const tenantAResult = await tenantA; + + expect(inspectSpy).toHaveBeenCalledTimes(2); + expect(tenantB.config.description).toBe('tenant-b.example.com'); + expect(tenantAResult.config).toEqual(tenantB.config); + await expect(tenantRegistry['cacheConfigsRepo'].get('stub_server')).resolves.toEqual( + tenantB.config, + ); + }); + + it('does not share a DB reinspection across users', async () => { + jest.spyOn(registry['dbConfigsRepo'], 'get').mockResolvedValue({ + ...stubOptions, + source: 'user', + dbId: 'db-server-id', + inspectionFailed: true, + updatedAt: FIXED_TIME, + }); + const updateSpy = jest.spyOn(registry['dbConfigsRepo'], 'update'); + const inspectSpy = jest.spyOn(MCPServerInspector, 'inspect'); + + const [userOne, userOneAgain, userTwo] = await Promise.all([ + registry.reinspectServer('db_server', 'DB', 'user-1'), + registry.reinspectServer('db_server', 'DB', 'user-1'), + registry.reinspectServer('db_server', 'DB', 'user-2'), + ]); + + expect(inspectSpy).toHaveBeenCalledTimes(2); + expect(userOneAgain).toBe(userOne); + expect(userTwo).not.toBe(userOne); + expect(updateSpy.mock.calls.map(([, , userId]) => userId).sort()).toEqual([ + 'user-1', + 'user-2', + ]); + }); + + it('writes nothing when another replica recovered the stub during inspection', async () => { + await registry.addServerStub('stub_server', stubOptions, 'CACHE'); + const memoized = await registry.getAllServerConfigs(); + const recoveredElsewhere: t.ParsedServerConfig = { + ...stubOptions, + description: 'recovered elsewhere', + }; + const inspect = jest.mocked(MCPServerInspector.inspect).getMockImplementation()!; + jest.spyOn(MCPServerInspector, 'inspect').mockImplementationOnce(async (...args) => { + await registry['cacheConfigsRepo'].update('stub_server', recoveredElsewhere); + return inspect(...args); + }); + const replaceSpy = jest.spyOn(ServerConfigsCacheInMemory.prototype, 'replaceStub'); + + const result = await registry.reinspectServer('stub_server', 'CACHE'); + + await expect(replaceSpy.mock.results[0].value).resolves.toBeUndefined(); + expect(result.config).toMatchObject(recoveredElsewhere); + await expect(registry['cacheConfigsRepo'].get('stub_server')).resolves.toEqual(result.config); + expect(memoized.stub_server.inspectionFailed).toBe(true); + await expect(registry.getAllServerConfigs()).resolves.toMatchObject({ + stub_server: { description: 'recovered elsewhere' }, + }); + }); + + it('resolves to a recovery stored elsewhere when its own inspection fails', async () => { + await registry.addServerStub('stub_server', stubOptions, 'CACHE'); + const memoized = await registry.getAllServerConfigs(); + const recoveredElsewhere: t.ParsedServerConfig = { + ...stubOptions, + description: 'recovered elsewhere', + }; + jest.spyOn(MCPServerInspector, 'inspect').mockImplementationOnce(async () => { + await registry['cacheConfigsRepo'].update('stub_server', recoveredElsewhere); + throw new Error('connect ECONNREFUSED'); + }); + + const result = await registry.reinspectServer('stub_server', 'CACHE'); + + expect(result.config).toMatchObject(recoveredElsewhere); + expect(result.config.inspectionFailed).toBeUndefined(); + expect(memoized.stub_server.inspectionFailed).toBe(true); + await expect(registry.getAllServerConfigs()).resolves.toMatchObject({ + stub_server: { description: 'recovered elsewhere' }, + }); + }); + + it('does not inspect again when a recovery lands while its allowlists resolve', async () => { + let releaseResolver!: () => void; + const resolverHeld = new Promise((resolve) => { + releaseResolver = resolve; + }); + let markResolving!: () => void; + const resolving = new Promise((resolve) => { + markResolving = resolve; + }); + (MCPServersRegistry as unknown as { instance: undefined }).instance = undefined; + const slowRegistry = MCPServersRegistry.createInstance( + mockMongoose, + null, + null, + async (ctx) => { + if (ctx?.userId === 'slow-user') { + markResolving(); + await resolverHeld; + } + return { allowedDomains: null, allowedAddresses: null }; + }, + ); + await slowRegistry.reset(); + await slowRegistry.addServerStub('stub_server', stubOptions, 'CACHE'); + const inspectSpy = jest.spyOn(MCPServerInspector, 'inspect'); + + const slow = slowRegistry.reinspectServer('stub_server', 'CACHE', 'slow-user'); + await resolving; + const recovered = await slowRegistry.reinspectServer('stub_server', 'CACHE', 'fast-user'); + releaseResolver(); + + await expect(slow).resolves.toEqual(recovered); + expect(inspectSpy).toHaveBeenCalledTimes(1); + }); + + it('inspects the newer stub when a registry re-initialization replaced the inspected one', async () => { + await registry.addServerStub('stub_server', stubOptions, 'CACHE'); + const movedOptions: t.MCPOptions = { ...stubOptions, url: 'https://moved.example.com/mcp' }; + const inspect = jest.mocked(MCPServerInspector.inspect).getMockImplementation()!; + const inspectSpy = jest + .spyOn(MCPServerInspector, 'inspect') + .mockImplementationOnce(async (...args) => { + jest.setSystemTime(new Date(FIXED_TIME + 1000)); + await registry['cacheConfigsRepo'].update('stub_server', { + ...movedOptions, + source: 'yaml', + inspectionFailed: true, + }); + return inspect(...args); + }); + + const result = await registry.reinspectServer('stub_server', 'CACHE'); + + expect(inspectSpy).toHaveBeenCalledTimes(2); + expect(inspectSpy.mock.calls[1][1]).toMatchObject(movedOptions); + expect(result.config).toMatchObject(movedOptions); + expect(result.config.inspectionFailed).toBeUndefined(); + await expect(registry['cacheConfigsRepo'].get('stub_server')).resolves.toEqual(result.config); + }); + + it('does not answer a caller that read a newer stub with the inspection of the one it replaced', async () => { + await registry.addServerStub('stub_server', stubOptions, 'CACHE'); + const movedOptions: t.MCPOptions = { ...stubOptions, url: 'https://moved.example.com/mcp' }; + const inspect = jest.mocked(MCPServerInspector.inspect).getMockImplementation()!; + let releaseReplaced!: () => void; + const replacedHeld = new Promise((resolve) => { + releaseReplaced = resolve; + }); + let markReplacedInspecting!: () => void; + const replacedInspecting = new Promise((resolve) => { + markReplacedInspecting = resolve; + }); + const inspectSpy = jest + .spyOn(MCPServerInspector, 'inspect') + .mockImplementationOnce(async (...args) => { + markReplacedInspecting(); + await replacedHeld; + return inspect(...args); + }); + + const replaced = registry.reinspectServer('stub_server', 'CACHE'); + await replacedInspecting; + jest.setSystemTime(new Date(FIXED_TIME + 1000)); + await registry['cacheConfigsRepo'].update('stub_server', { + ...movedOptions, + source: 'yaml', + inspectionFailed: true, + }); + + const current = await registry.reinspectServer('stub_server', 'CACHE'); + expect(inspectSpy).toHaveBeenCalledTimes(2); + expect(inspectSpy.mock.calls[1][1]).toMatchObject(movedOptions); + expect(current.config).toMatchObject(movedOptions); + + releaseReplaced(); + await expect(replaced).resolves.toEqual(current); + expect(inspectSpy).toHaveBeenCalledTimes(2); + await expect(registry['cacheConfigsRepo'].get('stub_server')).resolves.toEqual( + current.config, + ); + }); + + it('adopts the outcome of the flight another request started for a newer stub', async () => { + await registry.addServerStub('stub_server', stubOptions, 'CACHE'); + const movedOptions: t.MCPOptions = { ...stubOptions, url: 'https://moved.example.com/mcp' }; + const inspect = jest.mocked(MCPServerInspector.inspect).getMockImplementation()!; + let releaseReplaced!: () => void; + const replacedHeld = new Promise((resolve) => { + releaseReplaced = resolve; + }); + let markReplacedInspecting!: () => void; + const replacedInspecting = new Promise((resolve) => { + markReplacedInspecting = resolve; + }); + let releaseNewer!: () => void; + const newerHeld = new Promise((resolve) => { + releaseNewer = resolve; + }); + let markNewerInspecting!: () => void; + const newerInspecting = new Promise((resolve) => { + markNewerInspecting = resolve; + }); + const inspectSpy = jest + .spyOn(MCPServerInspector, 'inspect') + .mockImplementationOnce(async (...args) => { + markReplacedInspecting(); + await replacedHeld; + return inspect(...args); + }) + .mockImplementationOnce(async (...args) => { + markNewerInspecting(); + await newerHeld; + return inspect(...args); + }) + .mockRejectedValue(new Error('connect ECONNREFUSED')); + + const replaced = registry.reinspectServer('stub_server', 'CACHE'); + await replacedInspecting; + jest.setSystemTime(new Date(FIXED_TIME + 1000)); + await registry['cacheConfigsRepo'].update('stub_server', { + ...movedOptions, + source: 'yaml', + inspectionFailed: true, + }); + const newer = registry.reinspectServer('stub_server', 'CACHE'); + await newerInspecting; + + releaseReplaced(); + /** The store is promise-only, so one real macrotask lets the replaced flight settle as far + * as it can while the newer flight is still held. */ + await new Promise((resolve) => realSetImmediate(resolve)); + releaseNewer(); + const [replacedResult, newerResult] = await Promise.all([replaced, newer]); + + expect(inspectSpy).toHaveBeenCalledTimes(2); + expect(newerResult.config).toMatchObject(movedOptions); + expect(newerResult.config.inspectionFailed).toBeUndefined(); + expect(replacedResult).toEqual(newerResult); + }); + + it('adopts the outcome of the flight for a replacement stored with an older timestamp', async () => { + jest.setSystemTime(new Date(FIXED_TIME + 1000)); + await registry.addServerStub('stub_server', stubOptions, 'CACHE'); + const skewedOptions: t.MCPOptions = { ...stubOptions, url: 'https://skewed.example.com/mcp' }; + const inspect = jest.mocked(MCPServerInspector.inspect).getMockImplementation()!; + let releaseReplaced!: () => void; + const replacedHeld = new Promise((resolve) => { + releaseReplaced = resolve; + }); + let markReplacedInspecting!: () => void; + const replacedInspecting = new Promise((resolve) => { + markReplacedInspecting = resolve; + }); + let releaseSkewed!: () => void; + const skewedHeld = new Promise((resolve) => { + releaseSkewed = resolve; + }); + let markSkewedInspecting!: () => void; + const skewedInspecting = new Promise((resolve) => { + markSkewedInspecting = resolve; + }); + const inspectSpy = jest + .spyOn(MCPServerInspector, 'inspect') + .mockImplementationOnce(async (...args) => { + markReplacedInspecting(); + await replacedHeld; + return inspect(...args); + }) + .mockImplementationOnce(async (...args) => { + markSkewedInspecting(); + await skewedHeld; + return inspect(...args); + }) + .mockRejectedValue(new Error('connect ECONNREFUSED')); + + const replaced = registry.reinspectServer('stub_server', 'CACHE'); + await replacedInspecting; + jest.setSystemTime(new Date(FIXED_TIME)); + await registry['cacheConfigsRepo'].update('stub_server', { + ...skewedOptions, + source: 'yaml', + inspectionFailed: true, + }); + const skewed = registry.reinspectServer('stub_server', 'CACHE'); + await skewedInspecting; + + releaseReplaced(); + /** The store is promise-only, so one real macrotask lets the replaced flight settle as far + * as it can while the skewed flight is still held. */ + await new Promise((resolve) => realSetImmediate(resolve)); + releaseSkewed(); + const [replacedResult, skewedResult] = await Promise.all([replaced, skewed]); + + expect(inspectSpy).toHaveBeenCalledTimes(2); + expect(skewedResult.config).toMatchObject(skewedOptions); + expect(skewedResult.config.inspectionFailed).toBeUndefined(); + expect(replacedResult).toEqual(skewedResult); + }); + + it('breaks a mutual wait by inspecting within the flight that would close it', async () => { + await registry.addServerStub('stub_server', stubOptions, 'CACHE'); + const movedOptions: t.MCPOptions = { ...stubOptions, url: 'https://moved.example.com/mcp' }; + const restoredOptions: t.MCPOptions = { + ...stubOptions, + url: 'https://restored.example.com/mcp', + }; + const inspect = jest.mocked(MCPServerInspector.inspect).getMockImplementation()!; + let releaseFirst!: () => void; + const firstHeld = new Promise((resolve) => { + releaseFirst = resolve; + }); + let markFirstInspecting!: () => void; + const firstInspecting = new Promise((resolve) => { + markFirstInspecting = resolve; + }); + let releaseMoved!: () => void; + const movedHeld = new Promise((resolve) => { + releaseMoved = resolve; + }); + let markMovedInspecting!: () => void; + const movedInspecting = new Promise((resolve) => { + markMovedInspecting = resolve; + }); + const inspectSpy = jest + .spyOn(MCPServerInspector, 'inspect') + .mockImplementationOnce(async (...args) => { + markFirstInspecting(); + await firstHeld; + return inspect(...args); + }) + .mockImplementationOnce(async (...args) => { + markMovedInspecting(); + await movedHeld; + return inspect(...args); + }); + + const first = registry.reinspectServer('stub_server', 'CACHE'); + await firstInspecting; + jest.setSystemTime(new Date(FIXED_TIME + 1000)); + await registry['cacheConfigsRepo'].update('stub_server', { + ...movedOptions, + source: 'yaml', + inspectionFailed: true, + }); + const moved = registry.reinspectServer('stub_server', 'CACHE'); + await movedInspecting; + + releaseFirst(); + await new Promise((resolve) => realSetImmediate(resolve)); + /** A stub carrying the first flight's `updatedAt` again, which only clock skew can write, + * makes the flight the first one now waits on settle into the first one's key. */ + jest.setSystemTime(new Date(FIXED_TIME)); + await registry['cacheConfigsRepo'].update('stub_server', { + ...restoredOptions, + source: 'yaml', + inspectionFailed: true, + }); + releaseMoved(); + const [firstResult, movedResult] = await Promise.all([first, moved]); + + expect(inspectSpy).toHaveBeenCalledTimes(3); + expect(inspectSpy.mock.calls[2][1]).toMatchObject(restoredOptions); + expect(movedResult.config).toMatchObject(restoredOptions); + expect(firstResult).toEqual(movedResult); + await expect(registry['cacheConfigsRepo'].get('stub_server')).resolves.toEqual( + movedResult.config, + ); + }); + + it('rejects instead of waiting on itself when storage leaves the inspected stub in place', async () => { + await registry.addServerStub('stub_server', stubOptions, 'CACHE'); + jest + .spyOn(ServerConfigsCacheInMemory.prototype, 'replaceStub') + .mockResolvedValueOnce(undefined); + + await expect(registry.reinspectServer('stub_server', 'CACHE')).rejects.toThrow( + MCPInspectionFailedError, + ); + await expect(registry['cacheConfigsRepo'].get('stub_server')).resolves.toMatchObject({ + inspectionFailed: true, + }); + + const retried = await registry.reinspectServer('stub_server', 'CACHE'); + expect(retried.config.inspectionFailed).toBeUndefined(); + }); + + it('rejects every concurrent caller while the server is unreachable, then retries on the next call', async () => { + await registry.addServerStub('stub_server', stubOptions, 'CACHE'); + const inspectSpy = jest.spyOn(MCPServerInspector, 'inspect'); + inspectSpy.mockRejectedValueOnce(new Error('connect ECONNREFUSED')); + + const outcomes = await Promise.allSettled([ + registry.reinspectServer('stub_server', 'CACHE'), + registry.reinspectServer('stub_server', 'CACHE'), + ]); + + expect(inspectSpy).toHaveBeenCalledTimes(1); + for (const outcome of outcomes) { + expect(outcome).toMatchObject({ + status: 'rejected', + reason: expect.any(MCPInspectionFailedError), + }); + } + await expect(registry['cacheConfigsRepo'].get('stub_server')).resolves.toMatchObject({ + inspectionFailed: true, + }); + + const retried = await registry.reinspectServer('stub_server', 'CACHE'); + expect(inspectSpy).toHaveBeenCalledTimes(2); + expect(retried.config.inspectionFailed).toBeUndefined(); + }); + }); + + describe('recoverServerConfig', () => { + const stubOptions: t.MCPOptions = { + type: 'streamable-http', + url: 'https://recovering.example.com/mcp', + }; + + it('returns a config that did not fail inspection as is', async () => { + const config: t.ParsedServerConfig = { ...stubOptions, source: 'yaml' }; + const reinspectSpy = jest.spyOn(registry, 'reinspectServer'); + + await expect(registry.recoverServerConfig('healthy_server', config)).resolves.toBe(config); + expect(reinspectSpy).not.toHaveBeenCalled(); + }); + + it('leaves a config-tier stub to the config cache retry', async () => { + const reinspectSpy = jest.spyOn(registry, 'reinspectServer'); + + await expect( + registry.recoverServerConfig('config_server', { + ...stubOptions, + source: 'config', + inspectionFailed: true, + }), + ).resolves.toBeUndefined(); + expect(reinspectSpy).not.toHaveBeenCalled(); + }); + + it('returns the recovered config for a stub', async () => { + const { config: stub } = await registry.addServerStub('stub_server', stubOptions, 'CACHE'); + + const recovered = await registry.recoverServerConfig('stub_server', stub, 'user-1'); + + expect(recovered?.inspectionFailed).toBeUndefined(); + await expect(registry['cacheConfigsRepo'].get('stub_server')).resolves.toEqual(recovered); + }); + + it('returns the stored config when another request already recovered the server', async () => { + const { config: stub } = await registry.addServerStub('stub_server', stubOptions, 'CACHE'); + const { config: recoveredElsewhere } = await registry.reinspectServer('stub_server', 'CACHE'); + const inspectSpy = jest.spyOn(MCPServerInspector, 'inspect'); + inspectSpy.mockClear(); + + await expect(registry.recoverServerConfig('stub_server', stub, 'user-1')).resolves.toEqual( + recoveredElsewhere, + ); + expect(inspectSpy).not.toHaveBeenCalled(); + }); + + it('returns undefined while the server is still unreachable', async () => { + const { config: stub } = await registry.addServerStub('stub_server', stubOptions, 'CACHE'); + jest + .spyOn(MCPServerInspector, 'inspect') + .mockRejectedValueOnce(new Error('connect ECONNREFUSED')); + + await expect( + registry.recoverServerConfig('stub_server', stub, 'user-1'), + ).resolves.toBeUndefined(); + await expect(registry['cacheConfigsRepo'].get('stub_server')).resolves.toMatchObject({ + inspectionFailed: true, + }); + }); + + it('reinspects a user-sourced stub in DB storage for that user', async () => { + const reinspectSpy = jest.spyOn(registry, 'reinspectServer'); + + await registry.recoverServerConfig( + 'db_server', + { ...stubOptions, source: 'user', inspectionFailed: true }, + 'user-1', + ); + + expect(reinspectSpy).toHaveBeenCalledWith('db_server', 'DB', 'user-1'); + }); }); describe('Read-through cache', () => { diff --git a/packages/api/src/mcp/registry/cache/ServerConfigsCacheInMemory.ts b/packages/api/src/mcp/registry/cache/ServerConfigsCacheInMemory.ts index 72e648ca45..d058e1e8f5 100644 --- a/packages/api/src/mcp/registry/cache/ServerConfigsCacheInMemory.ts +++ b/packages/api/src/mcp/registry/cache/ServerConfigsCacheInMemory.ts @@ -55,6 +55,21 @@ export class ServerConfigsCacheInMemory { return true; } + /** Replaces a failed-inspection stub only while it is still that stub — see the interface doc. */ + public async replaceStub( + serverName: string, + config: ParsedServerConfig, + stubUpdatedAt: number | undefined, + ): Promise { + const existing = this.cache.get(serverName); + if (existing?.inspectionFailed !== true || existing.updatedAt !== stubUpdatedAt) { + return undefined; + } + const storedConfig = { ...config, updatedAt: Date.now() }; + this.cache.set(serverName, storedConfig); + return storedConfig; + } + public async remove(serverName: string): Promise { if (!this.cache.delete(serverName)) { throw new Error(`Failed to remove server "${serverName}" in cache.`); diff --git a/packages/api/src/mcp/registry/cache/ServerConfigsCacheRedisAggregateKey.ts b/packages/api/src/mcp/registry/cache/ServerConfigsCacheRedisAggregateKey.ts index 94043d1dfe..dce06afa71 100644 --- a/packages/api/src/mcp/registry/cache/ServerConfigsCacheRedisAggregateKey.ts +++ b/packages/api/src/mcp/registry/cache/ServerConfigsCacheRedisAggregateKey.ts @@ -43,6 +43,11 @@ if not envelope.value then envelope.value = {} end local existing = envelope.value[serverName] if operation == 'add' and existing then return -1 end if (operation == 'update' or operation == 'remove') and not existing then return 0 end +if operation == 'replaceStub' then + if not existing or existing.inspectionFailed ~= true then return 0 end + if ARGV[5] == '' and existing.updatedAt ~= nil then return 0 end + if ARGV[5] ~= '' and tonumber(ARGV[5]) ~= existing.updatedAt then return 0 end +end local ttl = redis.call('PTTL', KEYS[1]) if operation == 'remove' then envelope.value[serverName] = nil @@ -131,10 +136,11 @@ export class ServerConfigsCacheRedisAggregateKey } private async mutateRedisEntry( - operation: 'add' | 'update' | 'upsert' | 'remove', + operation: 'add' | 'update' | 'upsert' | 'remove' | 'replaceStub', serverName: string, config?: ParsedServerConfig, updatedAt?: number, + stubUpdatedAt?: number, ): Promise { const result = await evalKeyvRedisScript(MUTATE_AGGREGATE_ENTRY, { keys: [this.aggregateRedisKey()], @@ -143,6 +149,7 @@ export class ServerConfigsCacheRedisAggregateKey serverName, config ? JSON.stringify(config) : '', updatedAt != null ? String(updatedAt) : '', + stubUpdatedAt != null ? String(stubUpdatedAt) : '', ], }); return typeof result === 'number' ? result : 0; @@ -194,6 +201,13 @@ export class ServerConfigsCacheRedisAggregateKey return all[serverName]; } + /** Reads past the local snapshot, which can lag another replica's write by up to its TTL. */ + public async getCurrent(serverName: string): Promise { + this.invalidateLocalSnapshot(); + const all = await this.getAll(); + return all[serverName]; + } + public async add(serverName: string, config: ParsedServerConfig): Promise { if (this.leaderOnly) await this.leaderCheck('add MCP servers'); return this.withWriteLock(async () => { @@ -313,6 +327,39 @@ export class ServerConfigsCacheRedisAggregateKey }); } + /** Replaces a failed-inspection stub only while it is still that stub — see the interface doc. + * The Redis-side check and write are one script, so replicas racing the same recovery cannot + * both land. */ + public async replaceStub( + serverName: string, + config: ParsedServerConfig, + stubUpdatedAt: number | undefined, + ): Promise { + if (this.leaderOnly) await this.leaderCheck('replace MCP server stubs'); + return this.withWriteLock(async () => { + const storedConfig = { ...config, updatedAt: Date.now() }; + if (this.usesRedisStore()) { + const result = await this.mutateRedisEntry( + 'replaceStub', + serverName, + storedConfig, + storedConfig.updatedAt, + stubUpdatedAt, + ); + return result === 1 ? storedConfig : undefined; + } + this.invalidateLocalSnapshot(); // Force fresh Redis read (see add() comment) + const all = await this.getAll(); + const existing = all[serverName]; + if (existing?.inspectionFailed !== true || existing.updatedAt !== stubUpdatedAt) { + return undefined; + } + const success = await this.cache.set(AGGREGATE_KEY, { ...all, [serverName]: storedConfig }); + this.successCheck(`replace ${this.namespace} server stub "${serverName}"`, success); + return storedConfig; + }); + } + public async remove(serverName: string): Promise { if (this.leaderOnly) await this.leaderCheck('remove MCP servers'); return this.withWriteLock(async () => { diff --git a/packages/api/src/mcp/registry/cache/__tests__/ServerConfigsCacheInMemory.test.ts b/packages/api/src/mcp/registry/cache/__tests__/ServerConfigsCacheInMemory.test.ts index b8827a3fe9..a9edbd490e 100644 --- a/packages/api/src/mcp/registry/cache/__tests__/ServerConfigsCacheInMemory.test.ts +++ b/packages/api/src/mcp/registry/cache/__tests__/ServerConfigsCacheInMemory.test.ts @@ -184,6 +184,41 @@ describe('ServerConfigsCacheInMemory Integration Tests', () => { }); }); + describe('replaceStub operation', () => { + const stub: ParsedServerConfig = { ...mockConfig1, inspectionFailed: true }; + + it('replaces the failed stub it was inspected from', async () => { + await cache.add('server1', stub); + + await expect(cache.replaceStub('server1', mockConfig2, FIXED_TIME)).resolves.toEqual( + mockConfig2, + ); + expect(await cache.get('server1')).toEqual(mockConfig2); + }); + + it('leaves an entry another writer already recovered, even within the same millisecond', async () => { + await cache.add('server1', stub); + await cache.update('server1', mockConfig3); + + await expect(cache.replaceStub('server1', mockConfig2, FIXED_TIME)).resolves.toBeUndefined(); + expect(await cache.get('server1')).toEqual(mockConfig3); + }); + + it('leaves a newer stub written after the inspected one', async () => { + await cache.add('server1', stub); + (Date.now as jest.Mock).mockReturnValueOnce(FIXED_TIME + 1); + await cache.update('server1', stub); + + await expect(cache.replaceStub('server1', mockConfig2, FIXED_TIME)).resolves.toBeUndefined(); + expect(await cache.get('server1')).toEqual({ ...stub, updatedAt: FIXED_TIME + 1 }); + }); + + it('does not create a missing entry', async () => { + await expect(cache.replaceStub('server1', mockConfig2, FIXED_TIME)).resolves.toBeUndefined(); + expect(await cache.get('server1')).toBeUndefined(); + }); + }); + describe('credential placeholders in YAML configs', () => { it('should preserve LIBRECHAT_OPENID placeholders (admin configs are trusted)', async () => { const adminConfig: ParsedServerConfig & { headers?: Record } = { diff --git a/packages/api/src/mcp/registry/cache/__tests__/ServerConfigsCacheRedisAggregateKey.cache_integration.spec.ts b/packages/api/src/mcp/registry/cache/__tests__/ServerConfigsCacheRedisAggregateKey.cache_integration.spec.ts index 99cac18319..f472e53d4c 100644 --- a/packages/api/src/mcp/registry/cache/__tests__/ServerConfigsCacheRedisAggregateKey.cache_integration.spec.ts +++ b/packages/api/src/mcp/registry/cache/__tests__/ServerConfigsCacheRedisAggregateKey.cache_integration.spec.ts @@ -256,7 +256,11 @@ describe('ServerConfigsCacheRedisAggregateKey Integration Tests', () => { await replica.add('atomic-server', mockConfig1); await replica.update('atomic-server', mockConfig2); - await replica.upsert('atomic-server', mockConfig3); + await replica.upsert('atomic-server', { ...mockConfig3, inspectionFailed: true }); + const stub = await replica.get('atomic-server'); + await expect( + replica.replaceStub('atomic-server', mockConfig1, stub?.updatedAt), + ).resolves.toBeDefined(); await replica.remove('atomic-server'); expect(cacheSetSpy.mock.calls).toHaveLength(0); @@ -314,6 +318,79 @@ describe('ServerConfigsCacheRedisAggregateKey Integration Tests', () => { }); }); + describe('getCurrent operation', () => { + it('reads a write from another replica that its local snapshot predates', async () => { + const replicaA = new ServerConfigsCacheRedisAggregateKey('agg-test', false); + const replicaB = new ServerConfigsCacheRedisAggregateKey('agg-test', false); + await replicaA.add('server1', mockConfig1); + await replicaA.getAll(); + await replicaB.update('server1', mockConfig2); + + expect(await replicaA.get('server1')).toMatchObject(mockConfig1); + expect(await replicaA.getCurrent('server1')).toMatchObject(mockConfig2); + expect(await replicaA.get('server1')).toMatchObject(mockConfig2); + }); + }); + + describe('replaceStub operation', () => { + const stub = { ...mockConfig1, inspectionFailed: true } as ParsedServerConfig; + + it('replaces the failed stub it was inspected from exactly once', async () => { + const { config: stored } = await cache.add('server1', stub); + + const replaced = await cache.replaceStub('server1', mockConfig2, stored.updatedAt); + + expect(replaced).toMatchObject(mockConfig2); + expect(await cache.get('server1')).toEqual(replaced); + await expect( + cache.replaceStub('server1', mockConfig3, stored.updatedAt), + ).resolves.toBeUndefined(); + expect(await cache.get('server1')).toEqual(replaced); + }); + + it('leaves entries that are not the inspected stub', async () => { + const { config: olderStub } = await cache.add('newer-stub', stub); + const { config: recovered } = await cache.add('recovered', mockConfig2); + + await expect( + cache.replaceStub('newer-stub', mockConfig3, olderStub.updatedAt! - 1), + ).resolves.toBeUndefined(); + await expect( + cache.replaceStub('recovered', mockConfig3, recovered.updatedAt), + ).resolves.toBeUndefined(); + await expect( + cache.replaceStub('missing', mockConfig3, olderStub.updatedAt), + ).resolves.toBeUndefined(); + + expect(await cache.get('newer-stub')).toEqual(olderStub); + expect(await cache.get('recovered')).toEqual(recovered); + expect(await cache.get('missing')).toBeUndefined(); + }); + + it('lands exactly one of two replicas replacing the same stub', async () => { + const replicaA = new ServerConfigsCacheRedisAggregateKey('agg-test', false); + const replicaB = new ServerConfigsCacheRedisAggregateKey('agg-test', false); + const { config: stored } = await cache.add('server1', stub); + + const results = await Promise.all([ + replicaA.replaceStub('server1', mockConfig2, stored.updatedAt), + replicaB.replaceStub('server1', mockConfig3, stored.updatedAt), + ]); + + const landed = results.filter((result) => result != null); + expect(landed).toHaveLength(1); + expect(await cache.get('server1')).toEqual(landed[0]); + }); + + it('preserves empty arrays in the replacement', async () => { + const { config: stored } = await cache.add('empty-arrays', stub); + + await cache.replaceStub('empty-arrays', { ...mockConfig2, args: [] }, stored.updatedAt); + + expect(await cache.get('empty-arrays')).toMatchObject({ args: [] }); + }); + }); + describe('reset operation', () => { it('should clear all configs', async () => { await cache.add('server1', mockConfig1); diff --git a/packages/api/src/mcp/reinitialize.spec.ts b/packages/api/src/mcp/reinitialize.spec.ts new file mode 100644 index 0000000000..64674517d4 --- /dev/null +++ b/packages/api/src/mcp/reinitialize.spec.ts @@ -0,0 +1,53 @@ +import type { ParsedServerConfig } from '~/mcp/types'; +import { resolveMCPReinitializeConfig } from './reinitialize'; + +describe('resolveMCPReinitializeConfig', () => { + const stub: ParsedServerConfig = { + type: 'streamable-http', + url: 'https://recovering.example.com/mcp', + source: 'yaml', + inspectionFailed: true, + }; + + it('passes a config that did not fail inspection through without recovering it', async () => { + const recoverServerConfig = jest.fn(); + const healthy: ParsedServerConfig = { ...stub, inspectionFailed: undefined }; + + await expect( + resolveMCPReinitializeConfig({ recoverServerConfig }, 'recovering', healthy, 'user-1'), + ).resolves.toEqual({ serverConfig: healthy }); + await expect( + resolveMCPReinitializeConfig({ recoverServerConfig }, 'recovering', undefined, 'user-1'), + ).resolves.toEqual({ serverConfig: undefined }); + expect(recoverServerConfig).not.toHaveBeenCalled(); + }); + + it('continues with the recovered config instead of the stub', async () => { + const recovered: ParsedServerConfig = { ...stub, inspectionFailed: undefined, tools: 'echo' }; + const recoverServerConfig = jest.fn().mockResolvedValue(recovered); + + await expect( + resolveMCPReinitializeConfig({ recoverServerConfig }, 'recovering', stub, 'user-1'), + ).resolves.toEqual({ serverConfig: recovered }); + expect(recoverServerConfig).toHaveBeenCalledWith('recovering', stub, 'user-1'); + }); + + it('stops with an unreachable result while the server cannot be recovered', async () => { + const recoverServerConfig = jest.fn().mockResolvedValue(undefined); + + await expect( + resolveMCPReinitializeConfig({ recoverServerConfig }, 'recovering', stub, 'user-1'), + ).resolves.toEqual({ + result: { + availableTools: null, + success: false, + message: "MCP server 'recovering' is still unreachable", + failureReason: 'unreachable', + oauthRequired: false, + serverName: 'recovering', + oauthUrl: null, + tools: null, + }, + }); + }); +}); diff --git a/packages/api/src/mcp/reinitialize.ts b/packages/api/src/mcp/reinitialize.ts new file mode 100644 index 0000000000..58fe7c0f43 --- /dev/null +++ b/packages/api/src/mcp/reinitialize.ts @@ -0,0 +1,53 @@ +import type { MCPReinitializeFailureReason } from 'librechat-data-provider'; +import type { MCPServersRegistry } from '~/mcp/registry/MCPServersRegistry'; +import type { ParsedServerConfig } from '~/mcp/types'; + +/** Result that ends a reinitialization before it connects. */ +export interface MCPReinitializeStopResult { + availableTools: null; + success: false; + message: string; + failureReason: MCPReinitializeFailureReason; + oauthRequired: false; + serverName: string; + oauthUrl: null; + tools: null; +} + +/** The config a reinitialization connects with, or the result that ends it first. */ +export type MCPReinitializeConfigResolution = + | { serverConfig: ParsedServerConfig | undefined; result?: undefined } + | { serverConfig?: undefined; result: MCPReinitializeStopResult }; + +/** + * Resolves the server config a reinitialization connects with. A config that failed inspection + * is recovered through the registry, so a caller holding the stub connects with what inspection + * stored, including a recovery another request made. While the server stays unreachable, the + * reinitialization stops with an `unreachable` result. + */ +export async function resolveMCPReinitializeConfig( + registry: Pick, + serverName: string, + serverConfig: ParsedServerConfig | undefined, + userId?: string, +): Promise { + if (!serverConfig?.inspectionFailed) { + return { serverConfig }; + } + const recovered = await registry.recoverServerConfig(serverName, serverConfig, userId); + if (recovered) { + return { serverConfig: recovered }; + } + return { + result: { + availableTools: null, + success: false, + message: `MCP server '${serverName}' is still unreachable`, + failureReason: 'unreachable', + oauthRequired: false, + serverName, + oauthUrl: null, + tools: null, + }, + }; +}