diff --git a/api/server/controllers/ModelController.spec.js b/api/server/controllers/ModelController.spec.js index b7920487bc..474630c92b 100644 --- a/api/server/controllers/ModelController.spec.js +++ b/api/server/controllers/ModelController.spec.js @@ -12,7 +12,7 @@ jest.mock('~/server/services/Config', () => ({ loadConfigModels: (...args) => mockLoadConfigModels(...args), })); -const { loadModels } = require('./ModelController'); +const { loadModels, getEndpointModelsConfig } = require('./ModelController'); function deferred() { let resolve; @@ -55,3 +55,22 @@ describe('loadModels', () => { }); }); }); + +describe('getEndpointModelsConfig', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('loads only the requested endpoint through the wired loaders', async () => { + const req = { user: { id: 'user-1' }, config: { endpoints: {} }, query: {} }; + mockLoadConfigModels.mockResolvedValue({ Custom: ['custom-model'] }); + + await expect(getEndpointModelsConfig(req, 'Custom')).resolves.toEqual({ + Custom: ['custom-model'], + }); + + expect(mockLoadConfigModels).toHaveBeenCalledTimes(1); + expect(mockLoadConfigModels).toHaveBeenCalledWith(req, 'Custom'); + expect(mockLoadDefaultModels).not.toHaveBeenCalled(); + }); +}); diff --git a/api/server/controllers/agents/__tests__/openai.spec.js b/api/server/controllers/agents/__tests__/openai.spec.js index fb0128641c..561145f855 100644 --- a/api/server/controllers/agents/__tests__/openai.spec.js +++ b/api/server/controllers/agents/__tests__/openai.spec.js @@ -389,7 +389,7 @@ jest.mock('@librechat/api', () => ({ })); jest.mock('~/server/controllers/ModelController', () => ({ - getModelsConfig: jest.fn().mockResolvedValue({}), + getEndpointModelsConfig: jest.fn().mockResolvedValue({}), })); jest.mock('~/server/services/MCP', () => ({ @@ -956,6 +956,27 @@ describe('OpenAIChatCompletionController', () => { ); }); + it('validates handoff agents through a loader that reads only their own provider models', async () => { + const api = require('@librechat/api'); + const { getEndpointModelsConfig } = require('~/server/controllers/ModelController'); + api.initializeAgent.mockResolvedValueOnce({ + id: 'agent-123', + model: 'gpt-4', + model_parameters: {}, + toolRegistry: {}, + edges: [{ source: 'agent-123', target: 'agent-handoff' }], + }); + + await OpenAIChatCompletionController(req, res); + + const [params] = api.discoverConnectedAgents.mock.calls.at(-1); + expect(params).not.toHaveProperty('modelsConfig'); + expect(getEndpointModelsConfig).not.toHaveBeenCalled(); + await params.getEndpointModels('anthropic'); + expect(getEndpointModelsConfig).toHaveBeenCalledTimes(1); + expect(getEndpointModelsConfig).toHaveBeenCalledWith(req, 'anthropic'); + }); + describe('content filtering', () => { it('blocks opaque inline media before text inspection or agent loading', async () => { const api = require('@librechat/api'); diff --git a/api/server/controllers/agents/__tests__/responses.unit.spec.js b/api/server/controllers/agents/__tests__/responses.unit.spec.js index e8da19657f..d68161b18a 100644 --- a/api/server/controllers/agents/__tests__/responses.unit.spec.js +++ b/api/server/controllers/agents/__tests__/responses.unit.spec.js @@ -503,7 +503,7 @@ jest.mock('~/server/services/PermissionService', () => ({ })); jest.mock('~/server/controllers/ModelController', () => ({ - getModelsConfig: jest.fn().mockResolvedValue({}), + getEndpointModelsConfig: jest.fn().mockResolvedValue({}), })); jest.mock('~/server/services/MCP', () => ({ @@ -2647,6 +2647,29 @@ describe('createResponse controller', () => { }); }); + describe('handoff model validation', () => { + it('validates handoff agents through a loader that reads only their own provider models', async () => { + const { initializeAgent, discoverConnectedAgents } = require('@librechat/api'); + const { getEndpointModelsConfig } = require('~/server/controllers/ModelController'); + initializeAgent.mockResolvedValueOnce({ + id: 'agent-123', + model: 'claude-3', + model_parameters: {}, + toolRegistry: new Map(), + edges: [{ source: 'agent-123', target: 'agent-handoff' }], + }); + + await createResponse(req, res); + + const [params] = discoverConnectedAgents.mock.calls.at(-1); + expect(params).not.toHaveProperty('modelsConfig'); + expect(getEndpointModelsConfig).not.toHaveBeenCalled(); + await params.getEndpointModels('anthropic'); + expect(getEndpointModelsConfig).toHaveBeenCalledTimes(1); + expect(getEndpointModelsConfig).toHaveBeenCalledWith(req, 'anthropic'); + }); + }); + describe('sub-agent skill priming', () => { it('passes the sub-agent primed skill IDs into non-streaming tool execution', async () => { const { diff --git a/api/server/controllers/agents/openai.js b/api/server/controllers/agents/openai.js index 18d13e5475..57f7d363ff 100644 --- a/api/server/controllers/agents/openai.js +++ b/api/server/controllers/agents/openai.js @@ -110,7 +110,7 @@ const { checkSessionsAlive, loadCodeApiKey } = require('~/server/services/Files/ const { getLinkedInstructionsResolver, } = require('~/server/services/Endpoints/agents/linkedInstructions'); -const { getModelsConfig } = require('~/server/controllers/ModelController'); +const { getEndpointModelsConfig } = require('~/server/controllers/ModelController'); const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions'); const { resolveConfigServers } = require('~/server/services/MCP'); const { getMCPManager } = require('~/config'); @@ -698,7 +698,6 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { primaryConfig.subagents?.enabled === true && (primaryConfig.subagents.graphs?.length ?? 0) > 0; if (primaryConfig.edges?.length || primaryHasGraphSubagents) { - const modelsConfig = await getModelsConfig(req); const discoveryParams = { req, res, @@ -706,7 +705,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { primaryConfig, endpointOption, allowedProviders, - modelsConfig, + getEndpointModels: (endpoint) => getEndpointModelsConfig(req, endpoint), loadTools, requestFiles: [], conversationId, diff --git a/api/server/controllers/agents/responses.js b/api/server/controllers/agents/responses.js index 2020a491a3..33cc2ee369 100644 --- a/api/server/controllers/agents/responses.js +++ b/api/server/controllers/agents/responses.js @@ -126,7 +126,7 @@ const { checkSessionsAlive, loadCodeApiKey } = require('~/server/services/Files/ const { getLinkedInstructionsResolver, } = require('~/server/services/Endpoints/agents/linkedInstructions'); -const { getModelsConfig } = require('~/server/controllers/ModelController'); +const { getEndpointModelsConfig } = require('~/server/controllers/ModelController'); const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions'); const { resolveConfigServers, getAccessibleMcpServerNames } = require('~/server/services/MCP'); const { resolveConversationTitle } = require('~/server/services/Endpoints/titlePolicy'); @@ -934,7 +934,6 @@ const executeResponse = async (envelope, { req, res }) => { primaryConfig.subagents?.enabled === true && (primaryConfig.subagents.graphs?.length ?? 0) > 0; if (primaryConfig.edges?.length || primaryHasGraphSubagents) { - const modelsConfig = await getModelsConfig(req); const discoveryParams = { req, res, @@ -942,7 +941,7 @@ const executeResponse = async (envelope, { req, res }) => { primaryConfig, endpointOption, allowedProviders, - modelsConfig, + getEndpointModels: (endpoint) => getEndpointModelsConfig(req, endpoint), loadTools, requestFiles: [], conversationId, diff --git a/api/server/services/Config/catalog.spec.js b/api/server/services/Config/catalog.spec.js new file mode 100644 index 0000000000..cbc7bd9334 --- /dev/null +++ b/api/server/services/Config/catalog.spec.js @@ -0,0 +1,231 @@ +jest.mock('@librechat/api', () => ({ + ...jest.requireActual('@librechat/api'), + fetchModels: jest.fn(), +})); +jest.mock('./app'); +jest.mock('~/models', () => ({ getUserKeyValues: jest.fn(), initializeMessageBudget: jest.fn() })); +jest.mock('@librechat/data-schemas', () => ({ + ...jest.requireActual('@librechat/data-schemas'), + logger: { debug: jest.fn(), error: jest.fn(), warn: jest.fn(), info: jest.fn() }, +})); + +const { resolveModelCatalogKey } = require('librechat-data-provider'); +const { createModelsService, fetchModels } = require('@librechat/api'); +const { getUserKeyValues } = require('~/models'); +const loadDefaultModels = require('./loadDefaultModels'); +const loadConfigModels = require('./loadConfigModels'); + +/** + * The per-endpoint catalog agents are validated against, compared with the full catalog through + * the loaders the app wires: the aggregate default loader, `loadConfigModels` and the service. Only + * what leaves the process is replaced: the provider model request (`fetchModels`), the user key + * store and the config service. `getEndpointModelsConfig` must read the entry the full catalog holds. + */ + +const ENV_KEYS = [ + 'OPENAI_API_KEY', + 'OPENAI_MODELS', + 'OPENAI_REVERSE_PROXY', + 'ANTHROPIC_API_KEY', + 'ANTHROPIC_MODELS', + 'ANTHROPIC_REVERSE_PROXY', + 'GOOGLE_MODELS', + 'BEDROCK_AWS_MODELS', + 'AZURE_OPENAI_MODELS', + 'ASSISTANTS_MODELS', + 'ASSISTANTS_BASE_URL', + 'ENDPOINTS', +]; + +const customEndpoints = [ + { + name: 'Ollama', + baseURL: 'http://localhost:11434/v1', + apiKey: 'ollama', + models: { default: ['llama3.2'], fetch: false }, + }, + { + name: 'My Proxy', + baseURL: 'https://proxy.example/v1', + apiKey: 'proxy-key', + models: { default: ['proxy-model'], fetch: false }, + }, + { + name: 'Fetching', + baseURL: 'https://fetch.example/v1', + apiKey: 'fetch-key', + headers: { 'X-Team': 'ml' }, + models: { default: ['fallback'], fetch: true }, + }, + { + name: 'Personal', + baseURL: 'https://personal.example/v1', + apiKey: 'user_provided', + models: { default: ['personal-fallback'], fetch: true }, + }, + { name: 'Unserved', baseURL: 'https://unserved.example/v1', apiKey: 'unserved-key' }, +]; + +/** Every provider an agent can be saved with, in the spellings the catalog lookup resolves. */ +const providers = [ + 'openAI', + 'anthropic', + 'google', + 'vertexai', + 'azureOpenAI', + 'assistants', + 'bedrock', + 'Ollama', + 'ollama', + 'OLLAMA', + 'My Proxy', + 'my proxy', + 'Fetching', + 'Personal', + 'Unserved', + 'Unknown', +]; + +const variants = [ + ['built-in defaults', {}, {}, { Ollama: ['llama3.2'], 'My Proxy': ['proxy-model'] }], + [ + 'environment lists', + {}, + { + OPENAI_MODELS: 'env-gpt', + ANTHROPIC_MODELS: 'env-claude', + GOOGLE_MODELS: 'env-gemini', + BEDROCK_AWS_MODELS: 'env-bedrock', + AZURE_OPENAI_MODELS: 'env-azure', + ASSISTANTS_MODELS: 'env-assistant', + }, + { + openAI: ['env-gpt'], + anthropic: ['env-claude'], + google: ['env-gemini'], + bedrock: ['env-bedrock'], + azureOpenAI: ['env-azure'], + assistants: ['env-assistant'], + }, + ], + [ + 'configured lists', + { + openAI: { models: ['cfg-gpt'] }, + anthropic: { models: ['cfg-claude'] }, + google: { models: ['cfg-gemini'] }, + bedrock: { models: ['cfg-bedrock'] }, + }, + { OPENAI_MODELS: 'env-gpt' }, + { + openAI: ['cfg-gpt'], + anthropic: ['cfg-claude'], + google: ['cfg-gemini'], + bedrock: ['cfg-bedrock'], + }, + ], + [ + 'Vertex model names', + { anthropic: { vertexConfig: { modelNames: ['vertex-claude'] } } }, + {}, + { anthropic: ['vertex-claude'] }, + ], + [ + 'Azure model names', + { azureOpenAI: { modelNames: ['azure-deployment'] } }, + {}, + { azureOpenAI: ['azure-deployment'] }, + ], + [ + 'user provided keys', + {}, + { OPENAI_API_KEY: 'user_provided', ANTHROPIC_API_KEY: 'user_provided' }, + ], + ['built-ins that ENDPOINTS does not enable', {}, { ENDPOINTS: 'agents' }], +]; + +const lookup = (catalog, provider) => catalog[resolveModelCatalogKey(provider, catalog)]; + +describe('per-endpoint model catalog against the full catalog, with the real loaders', () => { + const saved = {}; + const service = createModelsService({ + getAppConfig: async () => { + throw new Error('the config is on the request'); + }, + loadDefaultModels, + loadConfigModels, + loadAsyncEndpoints: async () => ({}), + defaultEndpointsConfig: {}, + }); + const requestFor = (endpoints) => ({ + user: { id: 'user-1' }, + config: { endpoints: { custom: customEndpoints, ...endpoints } }, + query: {}, + body: {}, + }); + + beforeEach(() => { + jest.clearAllMocks(); + for (const key of ENV_KEYS) { + saved[key] = process.env[key]; + delete process.env[key]; + } + fetchModels.mockResolvedValue(['fetched-a', 'fetched-b']); + getUserKeyValues.mockResolvedValue({ apiKey: 'personal-key' }); + }); + + afterEach(() => { + for (const key of ENV_KEYS) { + if (saved[key] == null) { + delete process.env[key]; + } else { + process.env[key] = saved[key]; + } + } + }); + + it.each(variants)( + 'reads the entry the full catalog holds with %s', + async (_label, endpoints, env, anchors = {}) => { + Object.assign(process.env, env); + const full = await service.loadModels(requestFor(endpoints)); + const read = {}; + const expected = {}; + for (const provider of providers) { + read[provider] = lookup( + await service.getEndpointModelsConfig(requestFor(endpoints), provider), + provider, + ); + expected[provider] = lookup(full, provider); + } + + expect(read).toEqual(expected); + expect(read).toMatchObject({ ...anchors, Fetching: ['fetched-a', 'fetched-b'] }); + expect(read.openAI).toEqual(expect.any(Array)); + expect(read.Unknown).toBeUndefined(); + expect(read.Unserved).toBeUndefined(); + }, + ); + + it.each(['Fetching', 'Personal'])( + 'asks the provider for %s exactly as the full catalog does', + async (name) => { + await service.loadModels(requestFor({})); + const full = fetchModels.mock.calls.map(([params]) => params).filter((p) => p.name === name); + fetchModels.mockClear(); + + await service.getEndpointModelsConfig(requestFor({}), name); + const scoped = fetchModels.mock.calls.map(([params]) => params); + + expect(full).toHaveLength(1); + expect(scoped).toEqual(full); + }, + ); + + it('is not reproduced for Azure Assistants without assistantModels, which keeps the Assistants middleware on the full catalog', async () => { + const full = await service.loadModels(requestFor({})); + const entry = await service.getEndpointModelsConfig(requestFor({}), 'azureAssistants'); + + expect(lookup(entry, 'azureAssistants')).not.toEqual(lookup(full, 'azureAssistants')); + }); +}); diff --git a/api/server/services/Endpoints/agents/addedConvo.js b/api/server/services/Endpoints/agents/addedConvo.js index 215ee37b74..7a56f73357 100644 --- a/api/server/services/Endpoints/agents/addedConvo.js +++ b/api/server/services/Endpoints/agents/addedConvo.js @@ -45,7 +45,7 @@ const loadAddedAgent = (params) => * @param {import('express').Request} params.req * @param {import('express').Response} params.res * @param {Object} params.endpointOption - The endpoint option containing addedConvo - * @param {Object} params.modelsConfig - The models configuration + * @param {Function} params.getEndpointModels - Loads the model catalog entry for an endpoint * @param {Function} params.logViolation - Function to log violations * @param {Function} params.loadTools - Function to load agent tools * @param {Array} params.requestFiles - Request files @@ -82,7 +82,7 @@ const processAddedConvo = async ({ req, res, endpointOption, - modelsConfig, + getEndpointModels, logViolation, loadTools, requestFiles, @@ -132,7 +132,7 @@ const processAddedConvo = async ({ const addedValidation = await validateAgentModel({ req, res, - modelsConfig, + getEndpointModels, logViolation, agent: addedAgent, }); diff --git a/api/server/services/Endpoints/agents/addedConvo.spec.js b/api/server/services/Endpoints/agents/addedConvo.spec.js index 8893e9cbad..34326edde1 100644 --- a/api/server/services/Endpoints/agents/addedConvo.spec.js +++ b/api/server/services/Endpoints/agents/addedConvo.spec.js @@ -97,7 +97,7 @@ describe('processAddedConvo', () => { req: makeReq(), res: {}, endpointOption: { addedConvo: { model: 'gpt-4o', agent_id: 'added-agent' } }, - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: jest.fn().mockResolvedValue({ openai: ['gpt-4o'] }), logViolation: jest.fn(), loadTools: jest.fn(), requestFiles: [], @@ -111,6 +111,21 @@ describe('processAddedConvo', () => { ...overrides, }); + it('validates the added agent through the endpoint model loader', async () => { + const params = baseParams(); + + await processAddedConvo(params); + + expect(mockValidateAgentModel).toHaveBeenCalledTimes(1); + expect(mockValidateAgentModel).toHaveBeenCalledWith( + expect.objectContaining({ + agent: { id: 'added-agent', provider: 'openai' }, + getEndpointModels: params.getEndpointModels, + }), + ); + expect(mockValidateAgentModel.mock.calls[0][0]).not.toHaveProperty('modelsConfig'); + }); + it('forwards codeEnvAvailable=true to the added-convo initializeAgent call', async () => { await processAddedConvo(baseParams({ codeEnvAvailable: true })); diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js index 1c5d7c7b18..39f429a078 100644 --- a/api/server/services/Endpoints/agents/initialize.js +++ b/api/server/services/Endpoints/agents/initialize.js @@ -106,7 +106,7 @@ const { digestFile, } = require('~/server/services/Files/provision'); const { createProvisionFilesCallback } = require('~/server/services/Files/provisionCallback'); -const { getModelsConfig } = require('~/server/controllers/ModelController'); +const { getEndpointModelsConfig } = require('~/server/controllers/ModelController'); const { checkPermission, findAccessibleResources } = require('~/server/services/PermissionService'); const AgentClient = require('~/server/controllers/agents/client'); const { processAddedConvo } = require('./addedConvo'); @@ -376,9 +376,10 @@ const initializeClientWithProvider = async ({ }), ); const primaryAgentPromise = endpointOption.agent; - const modelsConfigPromise = getModelsConfig(req); - const validatedPrimaryAgentPromise = Promise.all([primaryAgentPromise, modelsConfigPromise]).then( - async ([primaryAgent, modelsConfig]) => { + /** Each validation, including handoff and subagents, loads only its agent's provider models. */ + const getEndpointModels = (endpoint) => getEndpointModelsConfig(req, endpoint); + const validatedPrimaryAgentPromise = Promise.resolve(primaryAgentPromise).then( + async (primaryAgent) => { if (!primaryAgent) { throw new Error('Agent not found'); } @@ -386,7 +387,7 @@ const initializeClientWithProvider = async ({ const validationResult = await validateAgentModel({ req, res, - modelsConfig, + getEndpointModels, logViolation, agent: primaryAgent, }); @@ -394,7 +395,7 @@ const initializeClientWithProvider = async ({ throw new Error(validationResult.error?.message); } - return { primaryAgent, modelsConfig }; + return primaryAgent; }, ); @@ -662,7 +663,7 @@ const initializeClientWithProvider = async ({ editableSkillIds, skillCreateAllowed, { skillStates, defaultActiveOnShare }, - { primaryAgent, modelsConfig }, + primaryAgent, requestConversation, chatProjectContext, toolRoleGrants, @@ -884,7 +885,7 @@ const initializeClientWithProvider = async ({ agent_ids: primaryConfig.agent_ids, endpointOption, allowedProviders, - modelsConfig, + getEndpointModels, loadTools, requestFiles, conversationId, @@ -985,7 +986,7 @@ const initializeClientWithProvider = async ({ res, loadTools, logViolation, - modelsConfig, + getEndpointModels, requestFiles, agentConfigs, primaryAgent, @@ -1391,7 +1392,7 @@ const initializeClientWithProvider = async ({ throw new Error(`You no longer have access to subagent ${agentId}.`); } const validation = await waitForAbort( - validateAgentModel({ req, res, agent, modelsConfig, logViolation }), + validateAgentModel({ req, res, agent, getEndpointModels, logViolation }), context.signal, ); throwIfAborted(context.signal); diff --git a/api/server/services/Endpoints/agents/initialize.spec.js b/api/server/services/Endpoints/agents/initialize.spec.js index a36f151c46..2df0118f11 100644 --- a/api/server/services/Endpoints/agents/initialize.spec.js +++ b/api/server/services/Endpoints/agents/initialize.spec.js @@ -74,7 +74,7 @@ jest.mock('~/server/services/ToolService', () => ({ })); jest.mock('~/server/controllers/ModelController', () => ({ - getModelsConfig: jest.fn().mockResolvedValue({}), + getEndpointModelsConfig: jest.fn().mockResolvedValue({}), })); let agentClientArgs; @@ -102,7 +102,7 @@ const { createInitializeClient, initializeClient } = require('./initialize'); const { processAddedConvo } = require('./addedConvo'); const { getSkillDbMethods, getSkillToolDeps } = require('./skillDeps'); const { loadAgentTools } = require('~/server/services/ToolService'); -const { getModelsConfig } = require('~/server/controllers/ModelController'); +const { getEndpointModelsConfig } = require('~/server/controllers/ModelController'); const { logger } = require('@librechat/data-schemas'); const { User, AclEntry } = require('~/db/models'); const { createAgent, createSkill, updateAgent } = require('~/models'); @@ -1131,7 +1131,11 @@ describe('initializeClient — processAgent ACL gate', () => { const req = makeReq(); req.config.endpoints.agents = { capabilities: ['skills'] }; mockInitializeAgent.mockResolvedValue(makePrimaryConfig([])); - getModelsConfig.mockReturnValueOnce(models.promise); + getEndpointModelsConfig.mockReturnValueOnce(models.promise); + mockValidateAgentModel.mockImplementationOnce(async ({ agent, getEndpointModels }) => { + await getEndpointModels(agent.provider); + return { isValid: true }; + }); const canCreateSkillSpy = jest .spyOn(getSkillToolDeps(), 'canCreateSkill') .mockReturnValue(createPermission.promise); @@ -1144,12 +1148,15 @@ describe('initializeClient — processAgent ACL gate', () => { endpointOption: makeEndpointOption(), }); - expect(getModelsConfig).toHaveBeenCalledWith(req); expect(canCreateSkillSpy).toHaveBeenCalledWith({ req }); + await new Promise((resolve) => setImmediate(resolve)); + expect(mockValidateAgentModel).toHaveBeenCalledTimes(1); + expect(getEndpointModelsConfig).toHaveBeenCalledTimes(1); + expect(getEndpointModelsConfig).toHaveBeenCalledWith(req, 'openai'); + expect(mockInitializeAgent).not.toHaveBeenCalled(); models.resolve({}); await new Promise((resolve) => setImmediate(resolve)); - expect(mockValidateAgentModel).toHaveBeenCalledTimes(1); expect(mockInitializeAgent).not.toHaveBeenCalled(); createPermission.resolve(false); @@ -1167,7 +1174,11 @@ describe('initializeClient — processAgent ACL gate', () => { delete req.resolvedConversation; const getConvoSpy = jest.spyOn(db, 'getConvo').mockReturnValue(conversation.promise); const getChatProjectSpy = jest.spyOn(db, 'getChatProject').mockReturnValue(project.promise); - getModelsConfig.mockReturnValueOnce(models.promise); + getEndpointModelsConfig.mockReturnValueOnce(models.promise); + mockValidateAgentModel.mockImplementationOnce(async ({ agent, getEndpointModels }) => { + await getEndpointModels(agent.provider); + return { isValid: true }; + }); mockInitializeAgent.mockResolvedValue(makePrimaryConfig([])); try { @@ -1279,7 +1290,7 @@ describe('initializeClient — processAgent ACL gate', () => { ], }, }; - getModelsConfig.mockResolvedValueOnce(catalog); + getEndpointModelsConfig.mockResolvedValueOnce(catalog); mockValidateAgentModel.mockImplementation(actualApi.validateAgentModel); mockInitializeAgent.mockImplementation(actualApi.initializeAgent); loadAgentTools.mockResolvedValue({ tools: [], toolContextMap: {}, toolDefinitions: [] }); @@ -3750,6 +3761,39 @@ describe('initializeClient — subagent loading', () => { expect(mockInitializeAgent).toHaveBeenCalledTimes(1); }); + it('validates a selected subagent through the same endpoint model loader as the primary', async () => { + await createViewableAgent(SUBAGENT_ID); + const primaryConfig = makePrimaryConfig({ + subagents: { enabled: true, allowSelf: false, agent_ids: [SUBAGENT_ID] }, + }); + const subagentConfig = makeSubagentConfig(SUBAGENT_ID); + let call = 0; + mockInitializeAgent.mockImplementation(() => + Promise.resolve(++call === 1 ? primaryConfig : subagentConfig), + ); + const req = makeSubagentReq(); + + await initializeClient({ + req, + res: {}, + signal: new AbortController().signal, + endpointOption: makeEndpointOption(), + }); + await agentClientArgs.agent.lazySubagentConfigs[0].resolve({ + signal: new AbortController().signal, + }); + + const validations = mockValidateAgentModel.mock.calls.map(([params]) => params); + expect(validations.map(({ agent }) => agent.id)).toEqual([PRIMARY_ID, SUBAGENT_ID]); + expect(validations[1].getEndpointModels).toBe(validations[0].getEndpointModels); + for (const params of validations) { + expect(params).not.toHaveProperty('modelsConfig'); + } + expect(getEndpointModelsConfig).not.toHaveBeenCalled(); + await validations[1].getEndpointModels('openai'); + expect(getEndpointModelsConfig).toHaveBeenCalledWith(req, 'openai'); + }); + it('preserves subagent tool context for ON_TOOL_EXECUTE (Codex P1 regression guard)', async () => { /** Verifies the Codex P1 fix: `agentToolContexts.delete(subagentId)` is * NOT called for subagent-only agents, so when the child dispatches diff --git a/e2e/specs/mock/scenarios/agent-model-catalog.spec.ts b/e2e/specs/mock/scenarios/agent-model-catalog.spec.ts new file mode 100644 index 0000000000..66572d02e3 --- /dev/null +++ b/e2e/specs/mock/scenarios/agent-model-catalog.spec.ts @@ -0,0 +1,137 @@ +import { expect, test } from '@playwright/test'; +import type { Page } from '@playwright/test'; +import type { AgentDetail } from '../agents.helpers'; +import { + MOCK_ENDPOINTS, + NEW_CHAT_PATH, + getAccessToken, + messagesView, + replyPrompt, + replyText, + requestJson, + sendMessage, + sendMessageAndWaitForCompletion, +} from '../helpers'; +import { cleanupAgent, uniqueAgentName } from '../agents.helpers'; + +/** + * An agent's model is checked against the model list of its own endpoint. The mock endpoints each + * serve one fixed model, so a model outside that list is the refusal case, and a handoff target on + * the second endpoint shows each connected agent being checked against its own list. + */ + +const [PRIMARY, OTHER] = MOCK_ENDPOINTS; + +const handoffMarker = (label: string, routes: unknown[]) => + `E2E_HANDOFF:${Buffer.from(JSON.stringify({ label, routes })).toString('base64url')}`; + +async function createAgent( + page: Page, + token: string, + name: string, + overrides: Record = {}, +) { + return requestJson(page, { + path: '/api/agents', + token, + method: 'POST', + body: { + name, + description: 'Agent model catalog acceptance fixture.', + instructions: 'Keep this fixture deterministic.', + provider: PRIMARY.label, + model: PRIMARY.model, + ...overrides, + }, + }); +} + +test.describe('agent models', () => { + test('an agent whose model its endpoint does not serve is refused and named @scenario:agent-on-an-unserved-model-is-refused', async ({ + page, + }) => { + test.setTimeout(120_000); + const name = uniqueAgentName('E2E Unserved Model'); + const model = `mock-unserved-${Date.now()}`; + const label = `unserved-${Date.now()}`; + let agentId: string | undefined; + + try { + await page.goto(NEW_CHAT_PATH, { timeout: 15_000 }); + const token = await getAccessToken(page); + agentId = (await createAgent(page, token, name, { model })).id; + + await page.goto(`${NEW_CHAT_PATH}?agent_id=${agentId}`, { timeout: 15_000 }); + await sendMessage(page, replyPrompt(label)); + + await expect( + messagesView(page).getByText( + `The model "${model}" is not available for ${PRIMARY.label}. Please select a different model.`, + ), + ).toBeVisible({ timeout: 30_000 }); + await expect(messagesView(page).getByText(replyText(label), { exact: true })).toHaveCount(0); + } finally { + await cleanupAgent(page, agentId); + } + }); + + test('a handoff target on another endpoint is checked against its own endpoint and answers @scenario:handoff-target-on-another-endpoint-answers', async ({ + page, + }) => { + test.setTimeout(180_000); + const targetName = uniqueAgentName('E2E Other Endpoint Target'); + const routerName = uniqueAgentName('E2E Other Endpoint Router'); + const label = `other-endpoint-${Date.now()}`; + const payload = `receipt-${Date.now()}`; + const description = 'Send the request to the specialist on the other endpoint.'; + const prompt = 'Pass the specialist the brief.'; + let targetId: string | undefined; + let routerId: string | undefined; + + try { + await page.goto(NEW_CHAT_PATH, { timeout: 15_000 }); + const token = await getAccessToken(page); + const target = await createAgent(page, token, targetName, { + provider: OTHER.label, + model: OTHER.model, + }); + targetId = target.id; + const router = await createAgent(page, token, routerName, { + edges: [ + { from: '', to: target.id, edgeType: 'handoff', description, prompt, promptKey: 'brief' }, + ], + }); + routerId = router.id; + + await page.goto(`${NEW_CHAT_PATH}?agent_id=${router.id}`, { timeout: 15_000 }); + const response = await sendMessageAndWaitForCompletion( + page, + handoffMarker(label, [ + { + from: router.id, + to: target.id, + description, + prompt, + promptKey: 'brief', + args: { brief: payload }, + receipt: payload, + }, + ]), + ); + expect(response.ok()).toBeTruthy(); + + await expect( + messagesView(page).getByText( + `E2E handoff complete ${label}: agent=${target.id}; received=${payload}`, + { exact: true }, + ), + ).toBeVisible({ timeout: 30_000 }); + await expect( + messagesView(page).getByRole('button', { name: `Transferred to ${targetName}` }), + ).toBeVisible(); + } finally { + await cleanupAgent(page, routerId); + await cleanupAgent(page, targetId); + } + }); +}); diff --git a/packages/api/src/agents/__tests__/initialize.test.ts b/packages/api/src/agents/__tests__/initialize.test.ts index ce7310b63c..538f008e51 100644 --- a/packages/api/src/agents/__tests__/initialize.test.ts +++ b/packages/api/src/agents/__tests__/initialize.test.ts @@ -660,7 +660,7 @@ describe('initializeAgent: ChatProject context', () => { res, primaryConfig, allowedProviders: new Set([Providers.OPENAI]), - modelsConfig: { openai: ['test-model'] }, + getEndpointModels: async () => ({ openai: ['test-model'] }), loadTools, useChatProjectContext: true, }, @@ -723,7 +723,7 @@ describe('initializeAgent: ChatProject context', () => { primaryConfig, rootConfigs: [primaryConfig], allowedProviders: new Set([Providers.OPENAI]), - modelsConfig: { openai: ['test-model'] }, + getEndpointModels: async () => ({ openai: ['test-model'] }), loadTools, useChatProjectContext: true, }, diff --git a/packages/api/src/agents/discovery.spec.ts b/packages/api/src/agents/discovery.spec.ts index 5b731ca275..97192fa7c7 100644 --- a/packages/api/src/agents/discovery.spec.ts +++ b/packages/api/src/agents/discovery.spec.ts @@ -68,7 +68,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -104,7 +104,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -126,6 +126,44 @@ describe('discoverConnectedAgents', () => { expect(onAgentInitialized.mock.calls.map((c) => c[0])).toEqual(['B', 'C']); }); + it('validates each referenced agent through the endpoint model loader', async () => { + const primaryConfig = makeConfig('A', [ + { from: 'A', to: 'B', edgeType: 'handoff' }, + { from: 'A', to: 'C', edgeType: 'handoff' }, + ]); + const agents: Record = { + B: { ...makeAgent('B'), provider: 'anthropic' } as Agent, + C: makeAgent('C'), + }; + const getEndpointModels = jest.fn(async () => ({})); + + await discoverConnectedAgents( + { + req: makeReq(), + res: makeRes(), + primaryConfig, + allowedProviders: new Set(), + getEndpointModels, + loadTools: jest.fn(), + }, + { + getAgent: jest.fn(async ({ id }: { id: string }) => agents[id] ?? null), + checkPermission: jest.fn().mockResolvedValue(true), + logViolation: jest.fn(), + db: {} as never, + }, + ); + + expect(mockValidateAgentModel).toHaveBeenCalledTimes(2); + for (const [params] of mockValidateAgentModel.mock.calls) { + expect(params.getEndpointModels).toBe(getEndpointModels); + expect(params).not.toHaveProperty('modelsConfig'); + } + expect( + mockValidateAgentModel.mock.calls.map(([params]) => params.agent.provider).sort(), + ).toEqual(['anthropic', 'openai']); + }); + it('skips orphaned agents and filters out edges pointing at them', async () => { const edges: GraphEdge[] = [ { from: 'A', to: 'B', edgeType: 'handoff' }, @@ -145,7 +183,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -187,7 +225,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -218,7 +256,7 @@ describe('discoverConnectedAgents', () => { // controllers do this — endpoint = primary provider) endpointOption: { endpoint: EModelEndpoint.openAI, model_parameters: {} }, allowedProviders: new Set(['openai']), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -255,7 +293,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), codeEnvAvailable: true, }, @@ -289,7 +327,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), fileSearchAvailable, }, @@ -321,7 +359,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), resolveWebSearchGrant, }, @@ -355,7 +393,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), resolveLinkedInstructions, recordLinkedPromptUsage: false, @@ -390,7 +428,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), requestBody, }, @@ -423,7 +461,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), codeEnvAvailable: false, }, @@ -452,7 +490,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), resourceType: 'REMOTE_AGENT', }, @@ -498,7 +536,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -540,7 +578,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -586,7 +624,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -629,7 +667,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -679,7 +717,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -742,7 +780,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -785,7 +823,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -833,7 +871,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -893,7 +931,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -934,7 +972,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -980,7 +1018,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -1016,7 +1054,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -1050,7 +1088,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -1086,7 +1124,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -1126,7 +1164,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -1157,7 +1195,7 @@ describe('discoverConnectedAgents', () => { primaryConfig, agent_ids: ['B', 'C'], allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -1195,7 +1233,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -1229,7 +1267,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -1263,7 +1301,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -1299,7 +1337,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -1335,7 +1373,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -1369,7 +1407,7 @@ describe('discoverConnectedAgents', () => { primaryConfig, agent_ids: ['B'], allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -1399,7 +1437,7 @@ describe('discoverConnectedAgents', () => { primaryConfig: makeConfig('A', edges), agent_ids: agentIds, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -1441,7 +1479,7 @@ describe('discoverConnectedAgents', () => { signal: controller.signal, primaryConfig: makeConfig('A', [{ from: 'A', to: 'B', edgeType: 'handoff' as const }]), allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -1479,7 +1517,7 @@ describe('discoverConnectedAgents', () => { res: makeRes(), primaryConfig, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), }, { @@ -1536,7 +1574,7 @@ describe('resolveSubagentGraphs', () => { primaryConfig, rootConfigs: [primaryConfig], allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), resourceType: 'remote_agent', statefulSessionsAvailable: true, @@ -1600,7 +1638,7 @@ describe('resolveSubagentGraphs', () => { primaryConfig, rootConfigs: [primaryConfig], allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), resourceType: 'remote_agent', }, @@ -1652,7 +1690,7 @@ describe('resolveSubagentGraphs', () => { primaryConfig: rootConfigs[0], rootConfigs, allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), resourceType: 'remote_agent', }, @@ -1693,7 +1731,7 @@ describe('resolveSubagentGraphs', () => { primaryConfig, rootConfigs: [primaryConfig], allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), resourceType: 'remote_agent', }, @@ -1746,7 +1784,7 @@ describe('resolveSubagentGraphs', () => { primaryConfig, rootConfigs: [primaryConfig], allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), resourceType: 'remote_agent', }, @@ -1791,7 +1829,7 @@ describe('resolveSubagentGraphs', () => { primaryConfig, rootConfigs: [primaryConfig], allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), resourceType: 'remote_agent', }, @@ -1845,7 +1883,7 @@ describe('resolveSubagentGraphs', () => { primaryConfig, rootConfigs: [primaryConfig], allowedProviders: new Set(), - modelsConfig: { openai: ['gpt-4o'] }, + getEndpointModels: async () => ({ openai: ['gpt-4o'] }), loadTools: jest.fn(), resourceType: 'remote_agent', }, diff --git a/packages/api/src/agents/discovery.ts b/packages/api/src/agents/discovery.ts index 65069601a1..a3e7ab0ec1 100644 --- a/packages/api/src/agents/discovery.ts +++ b/packages/api/src/agents/discovery.ts @@ -8,7 +8,6 @@ import { import type { Agent, GraphEdge, - TModelsConfig, TEndpointOption, AgentSubagentGraph, } from 'librechat-data-provider'; @@ -68,7 +67,8 @@ export interface DiscoverConnectedAgentsParams { agent_ids?: string[]; endpointOption?: Partial; allowedProviders: Set; - modelsConfig: TModelsConfig; + /** Each referenced agent's provider is known only once it is loaded, so its models load then. */ + getEndpointModels: ValidateAgentModelParams['getEndpointModels']; loadTools: InitializeAgentParams['loadTools']; requestFiles?: InitializeAgentParams['requestFiles']; conversationId?: string | null; @@ -246,7 +246,7 @@ async function initializeReferencedAgent( req: params.req, res: params.res, agent, - modelsConfig: params.modelsConfig, + getEndpointModels: params.getEndpointModels, logViolation: deps.logViolation, }), params.signal, diff --git a/packages/api/src/agents/pin.spec.ts b/packages/api/src/agents/pin.spec.ts index 8b1898c388..b94599c03b 100644 --- a/packages/api/src/agents/pin.spec.ts +++ b/packages/api/src/agents/pin.spec.ts @@ -194,7 +194,7 @@ describe('pinned provider configuration', () => { req: {} as Request, res: {} as Response, agent, - modelsConfig: { [EModelEndpoint.openAI]: ['gpt-4o-mini'] }, + getEndpointModels: async () => ({ [EModelEndpoint.openAI]: ['gpt-4o-mini'] }), logViolation, }); diff --git a/packages/api/src/agents/validation.spec.ts b/packages/api/src/agents/validation.spec.ts index b0607fb51d..ff48bb3663 100644 --- a/packages/api/src/agents/validation.spec.ts +++ b/packages/api/src/agents/validation.spec.ts @@ -1,12 +1,14 @@ import { + ErrorTypes, EModelEndpoint, MAX_SUBAGENTS, setMaxSubagents, MAX_SUBAGENT_GRAPH_NODES, MAX_GRAPH_SUBAGENT_MEMBERS, Providers, + ViolationTypes, } from 'librechat-data-provider'; -import type { Agent } from 'librechat-data-provider'; +import type { Agent, TModelsConfig } from 'librechat-data-provider'; import type { Request, Response } from 'express'; import { agentCreateSchema, @@ -692,17 +694,141 @@ describe('validateAgentModel', () => { const request = {} as Request; const response = {} as Response; const logViolation = jest.fn().mockResolvedValue(undefined); + const catalog = (models: TModelsConfig) => jest.fn(async (): Promise => models); beforeEach(() => { logViolation.mockClear(); }); + it('accepts a model in the endpoint catalog and requests only the agent provider', async () => { + const getEndpointModels = catalog({ [EModelEndpoint.openAI]: ['gpt-4o'] }); + + const result = await validateAgentModel({ + req: request, + res: response, + agent: { provider: EModelEndpoint.openAI, model: 'gpt-4o' } as Agent, + getEndpointModels, + logViolation, + }); + + expect(result).toEqual({ isValid: true }); + expect(getEndpointModels).toHaveBeenCalledTimes(1); + expect(getEndpointModels).toHaveBeenCalledWith(EModelEndpoint.openAI); + expect(logViolation).not.toHaveBeenCalled(); + }); + + it('rejects an agent without a model before loading any models', async () => { + const getEndpointModels = catalog({ [EModelEndpoint.openAI]: ['gpt-4o'] }); + + const result = await validateAgentModel({ + req: request, + res: response, + agent: { provider: EModelEndpoint.openAI, model: '' } as Agent, + getEndpointModels, + logViolation, + }); + + expect(result).toEqual({ + isValid: false, + error: { message: `{ "type": "${ErrorTypes.MISSING_MODEL}", "info": "openAI" }` }, + }); + expect(getEndpointModels).not.toHaveBeenCalled(); + expect(logViolation).not.toHaveBeenCalled(); + }); + + it('reports models that could not be loaded', async () => { + const result = await validateAgentModel({ + req: request, + res: response, + agent: { provider: EModelEndpoint.openAI, model: 'gpt-4o' } as Agent, + getEndpointModels: jest.fn().mockResolvedValue(undefined), + logViolation, + }); + + expect(result).toEqual({ + isValid: false, + error: { message: `{ "type": "${ErrorTypes.MODELS_NOT_LOADED}" }` }, + }); + expect(logViolation).not.toHaveBeenCalled(); + }); + + it.each([ + ['an empty catalog', {}], + ['a catalog without the provider', { [EModelEndpoint.anthropic]: ['claude'] }], + ])('reports an endpoint whose models are not loaded: %s', async (_label, models) => { + const result = await validateAgentModel({ + req: request, + res: response, + agent: { provider: EModelEndpoint.openAI, model: 'gpt-4o' } as Agent, + getEndpointModels: catalog(models), + logViolation, + }); + + expect(result).toEqual({ + isValid: false, + error: { + message: `{ "type": "${ErrorTypes.ENDPOINT_MODELS_NOT_LOADED}", "info": "openAI" }`, + }, + }); + expect(logViolation).not.toHaveBeenCalled(); + }); + + it('logs one violation and rejects a model outside the endpoint catalog', async () => { + const result = await validateAgentModel({ + req: request, + res: response, + agent: { provider: EModelEndpoint.openAI, model: 'gpt-5' } as Agent, + getEndpointModels: catalog({ [EModelEndpoint.openAI]: ['gpt-4o'] }), + logViolation, + }); + + expect(result).toEqual({ + isValid: false, + error: { + message: `{ "type": "${ViolationTypes.ILLEGAL_MODEL_REQUEST}", "info": "openAI|gpt-5" }`, + }, + }); + expect(logViolation).toHaveBeenCalledTimes(1); + expect(logViolation).toHaveBeenCalledWith( + request, + response, + ViolationTypes.ILLEGAL_MODEL_REQUEST, + { type: ViolationTypes.ILLEGAL_MODEL_REQUEST, model: 'gpt-5', endpoint: 'openAI' }, + expect.anything(), + ); + }); + + it('propagates a failure to load models instead of treating it as an empty catalog', async () => { + await expect( + validateAgentModel({ + req: request, + res: response, + agent: { provider: EModelEndpoint.openAI, model: 'gpt-4o' } as Agent, + getEndpointModels: jest.fn().mockRejectedValue(new Error('models unavailable')), + logViolation, + }), + ).rejects.toThrow('models unavailable'); + expect(logViolation).not.toHaveBeenCalled(); + }); + + it('reads the catalog under the served name for another spelling of a provider', async () => { + const result = await validateAgentModel({ + req: request, + res: response, + agent: { provider: 'Ollama', model: 'llama3.2' } as Agent, + getEndpointModels: catalog({ ollama: ['llama3.2'] }), + logViolation, + }); + + expect(result).toEqual({ isValid: true }); + }); + it('uses the Google catalog for a Vertex AI agent', async () => { const result = await validateAgentModel({ req: request, res: response, agent: { provider: Providers.VERTEXAI, model: 'gemini-3.7-flash' } as Agent, - modelsConfig: { [EModelEndpoint.google]: ['gemini-3.7-flash'] }, + getEndpointModels: catalog({ [EModelEndpoint.google]: ['gemini-3.7-flash'] }), logViolation, }); @@ -715,10 +841,10 @@ describe('validateAgentModel', () => { req: request, res: response, agent: { provider: Providers.VERTEXAI, model: 'custom-vertex-model' } as Agent, - modelsConfig: { + getEndpointModels: catalog({ [EModelEndpoint.google]: ['gemini-3.7-flash'], [Providers.VERTEXAI]: ['custom-vertex-model'], - }, + }), logViolation, }); @@ -731,7 +857,7 @@ describe('validateAgentModel', () => { req: request, res: response, agent: { provider: Providers.VERTEXAI, model: 'gemini-not-available' } as Agent, - modelsConfig: { [EModelEndpoint.google]: ['gemini-3.7-flash'] }, + getEndpointModels: catalog({ [EModelEndpoint.google]: ['gemini-3.7-flash'] }), logViolation, }); diff --git a/packages/api/src/agents/validation.ts b/packages/api/src/agents/validation.ts index e1569bbeab..35c2e58876 100644 --- a/packages/api/src/agents/validation.ts +++ b/packages/api/src/agents/validation.ts @@ -539,7 +539,11 @@ export interface ValidateAgentModelParams { req: LooseRequest; res: Response; agent: Agent; - modelsConfig: TModelsConfig; + /** + * Loads the model catalog entry for an endpoint. Only the agent's own provider is requested, + * so a request never loads the model lists of endpoints none of its agents use. + */ + getEndpointModels: (endpoint: string) => Promise; logViolation: ( req: LooseRequest, res: Response, @@ -567,7 +571,7 @@ interface ValidateAgentModelResult { export async function validateAgentModel( params: ValidateAgentModelParams, ): Promise { - const { req, res, agent, modelsConfig, logViolation } = params; + const { req, res, agent, getEndpointModels, logViolation } = params; const { model, provider: endpoint } = agent; if (!model) { @@ -579,6 +583,7 @@ export async function validateAgentModel( }; } + const modelsConfig = await getEndpointModels(endpoint); if (!modelsConfig) { return { isValid: false, diff --git a/packages/api/src/endpoints/config/service.spec.ts b/packages/api/src/endpoints/config/service.spec.ts index d58b4f5e19..6bd6b0a2f5 100644 --- a/packages/api/src/endpoints/config/service.spec.ts +++ b/packages/api/src/endpoints/config/service.spec.ts @@ -1,4 +1,4 @@ -import { FileSources } from 'librechat-data-provider'; +import { FileSources, resolveModelCatalogKey } from 'librechat-data-provider'; import type { TModelsConfig } from 'librechat-data-provider'; import type { AppConfig } from '@librechat/data-schemas'; import type { Response } from 'express'; @@ -238,3 +238,197 @@ describe('model discovery service', () => { expect(deps.loadConfigModels).not.toHaveBeenCalled(); }); }); + +describe('per-endpoint model loading', () => { + const hasOwn = (catalog: TModelsConfig, key: string) => + Object.prototype.hasOwnProperty.call(catalog, key); + const builtIns: TModelsConfig = { + openAI: ['gpt-default'], + anthropic: ['claude-default'], + google: ['gemini-default'], + azureOpenAI: ['azure-default'], + assistants: ['assistant-default'], + azureAssistants: ['azure-assistant-default'], + bedrock: ['bedrock-default'], + }; + const served: TModelsConfig = { + anthropic: ['claude-configured'], + ollama: ['llama3.2'], + 'My Proxy': ['proxy-model'], + }; + + /** + * Stand-in loaders that return the full catalog without a name and one entry with it. They test + * the service's own rules (lookup order, one load per request, no other endpoint loaded); that + * the entry equals the full catalog's under the real loaders is checked in + * `api/server/services/Config/catalog.spec.js`. + */ + function world(extra: TModelsConfig = {}) { + const configured = { ...served, ...extra }; + const deps = { + getAppConfig: jest.fn().mockResolvedValue({ endpoints: {} } as AppConfig), + loadDefaultModels: jest.fn(async (): Promise => ({ ...builtIns })), + loadConfigModels: jest.fn( + async (_req: ServerRequest, endpoint?: string): Promise => { + if (endpoint == null) { + return { ...configured }; + } + return hasOwn(configured, endpoint) ? { [endpoint]: configured[endpoint] } : {}; + }, + ), + defaultEndpointsConfig: { openAI: { userProvide: false } }, + loadAsyncEndpoints: jest.fn().mockResolvedValue({}), + loadDefaultEndpointModels: jest.fn( + async ( + _req: ServerRequest, + _config: AppConfig, + endpoint: string, + ): Promise => + hasOwn(builtIns, endpoint) ? { [endpoint]: builtIns[endpoint] } : {}, + ), + }; + return { deps, ...createModelsService(deps) }; + } + + /** The lookup agent validation performs on a catalog. */ + const lookup = (catalog: TModelsConfig, provider: string) => + catalog[resolveModelCatalogKey(provider, catalog)]; + + it.each([ + ['a built-in without a configured list', 'openAI', ['gpt-default']], + ['a built-in with a configured list', 'anthropic', ['claude-configured']], + ['a served custom endpoint', 'My Proxy', ['proxy-model']], + ['a custom endpoint saved under another spelling', 'Ollama', ['llama3.2']], + ['a provider served under an alias', 'vertexai', ['gemini-default']], + ['an endpoint with no model list', 'Unknown', undefined], + ])('reads %s', async (_label, provider, models) => { + const { getEndpointModelsConfig } = world(); + + const entry = await getEndpointModelsConfig(request(), provider); + + expect(lookup(entry, provider)).toEqual(models); + expect(Object.keys(entry).length).toBeLessThanOrEqual(1); + }); + + it('prefers an exact catalog over the alias', async () => { + const { getEndpointModelsConfig } = world({ vertexai: ['custom-vertex'] }); + + const entry = await getEndpointModelsConfig(request(), 'vertexai'); + + expect(lookup(entry, 'vertexai')).toEqual(['custom-vertex']); + }); + + it('loads the requested endpoint and no other', async () => { + const { deps, getEndpointModelsConfig } = world(); + const req = request(); + + await getEndpointModelsConfig(req, 'openAI'); + await getEndpointModelsConfig(req, 'My Proxy'); + + expect(deps.loadDefaultModels).not.toHaveBeenCalled(); + expect(deps.loadConfigModels.mock.calls.map(([, endpoint]) => endpoint)).toEqual([ + 'openAI', + 'My Proxy', + ]); + expect(deps.loadDefaultEndpointModels.mock.calls.map(([, , endpoint]) => endpoint)).toEqual([ + 'openAI', + ]); + }); + + it('loads an endpoint once per request, including concurrent and repeated calls', async () => { + const { deps, getEndpointModelsConfig } = world(); + const gate = deferred(); + deps.loadConfigModels.mockReturnValueOnce(gate.promise); + const req = request(); + + const first = getEndpointModelsConfig(req, 'My Proxy'); + const second = getEndpointModelsConfig(req, 'My Proxy'); + gate.resolve({ 'My Proxy': ['proxy-model'] }); + + await expect(Promise.all([first, second])).resolves.toEqual([ + { 'My Proxy': ['proxy-model'] }, + { 'My Proxy': ['proxy-model'] }, + ]); + await getEndpointModelsConfig(req, 'My Proxy'); + expect(deps.loadConfigModels).toHaveBeenCalledTimes(1); + }); + + it('shares the load between spellings that read the same endpoint', async () => { + const { deps, getEndpointModelsConfig } = world(); + const req = request(); + + await getEndpointModelsConfig(req, 'ollama'); + await getEndpointModelsConfig(req, 'Ollama'); + + expect(deps.loadConfigModels.mock.calls.filter(([, name]) => name === 'ollama')).toHaveLength( + 1, + ); + }); + + it('does not cache across requests', async () => { + const { deps, getEndpointModelsConfig } = world(); + + await getEndpointModelsConfig(request(), 'My Proxy'); + await getEndpointModelsConfig(request(), 'My Proxy'); + + expect(deps.loadConfigModels).toHaveBeenCalledTimes(2); + }); + + it('does not remember a failed load', async () => { + const { deps, getEndpointModelsConfig } = world(); + deps.loadConfigModels.mockRejectedValueOnce(new Error('provider down')); + const req = request(); + + await expect(getEndpointModelsConfig(req, 'My Proxy')).rejects.toThrow('provider down'); + await expect(getEndpointModelsConfig(req, 'My Proxy')).resolves.toEqual({ + 'My Proxy': ['proxy-model'], + }); + }); + + it('loads built-ins that ENDPOINTS does not enable', async () => { + const original = process.env.ENDPOINTS; + process.env.ENDPOINTS = 'agents'; + try { + const { getEndpointModelsConfig } = world(); + + await expect(getEndpointModelsConfig(request(), 'bedrock')).resolves.toEqual({ + bedrock: ['bedrock-default'], + }); + } finally { + if (original == null) { + delete process.env.ENDPOINTS; + } else { + process.env.ENDPOINTS = original; + } + } + }); + + it('resolves the principal config once for the request', async () => { + const { deps, getEndpointModelsConfig } = world(); + const req = request(); + delete req.config; + + await getEndpointModelsConfig(req, 'openAI'); + await getEndpointModelsConfig(req, 'My Proxy'); + + expect(deps.getAppConfig).toHaveBeenCalledTimes(1); + expect(deps.loadConfigModels).toHaveBeenCalledWith( + expect.objectContaining({ config: { endpoints: {} } }), + 'openAI', + ); + }); + + it.each([undefined, null, '', 42, ['openAI'], { name: 'openAI' }])( + 'loads nothing for the unusable endpoint %p', + async (endpoint) => { + const { deps, getEndpointModelsConfig } = world(); + + await expect( + getEndpointModelsConfig(request(), endpoint as string | null | undefined), + ).resolves.toEqual({}); + expect(deps.getAppConfig).not.toHaveBeenCalled(); + expect(deps.loadConfigModels).not.toHaveBeenCalled(); + expect(deps.loadDefaultEndpointModels).not.toHaveBeenCalled(); + }, + ); +}); diff --git a/packages/api/src/endpoints/config/service.ts b/packages/api/src/endpoints/config/service.ts index 0bad25fa21..9b7e0011a6 100644 --- a/packages/api/src/endpoints/config/service.ts +++ b/packages/api/src/endpoints/config/service.ts @@ -1,5 +1,9 @@ import { logger } from '@librechat/data-schemas'; -import { EModelEndpoint, getEnabledEndpoints } from 'librechat-data-provider'; +import { + EModelEndpoint, + getEnabledEndpoints, + resolveModelCatalogKey, +} from 'librechat-data-provider'; import type { TModelsConfig } from 'librechat-data-provider'; import type { AppConfig } from '@librechat/data-schemas'; import type { Response } from 'express'; @@ -27,12 +31,23 @@ export interface ModelsServiceDeps { loadDefaultEndpointModels?: typeof loadDefaultEndpointModels; } +function hasEntry(catalog: TModelsConfig, key: string): boolean { + return Object.prototype.hasOwnProperty.call(catalog, key) && catalog[key] != null; +} + export function createModelsService(deps: ModelsServiceDeps): { modelController: (req: ServerRequest, res: Response) => Promise; loadModels: (req: ServerRequest) => Promise; getModelsConfig: (req: ServerRequest) => Promise; getRequestModelsConfig: (req: ServerRequest) => Promise; + getEndpointModelsConfig: ( + req: ServerRequest, + endpoint: string | null | undefined, + ) => Promise; } { + /** Catalog entries loaded for a request, by the endpoint name they were requested under. */ + const requestEntries = new WeakMap>>(); + /** Internal callers keep the full catalog, regardless of HTTP query parameters. */ async function loadModels(req: ServerRequest): Promise { const [defaults, configured] = await Promise.all([ @@ -42,6 +57,53 @@ export function createModelsService(deps: ModelsServiceDeps): { return { ...defaults, ...configured }; } + async function resolveConfig(req: ServerRequest): Promise { + const config = req.config ?? (await deps.getAppConfig(getAppConfigOptionsFromUser(req.user))); + req.config = config; + return config; + } + + /** + * What the full catalog holds under one endpoint name: `{ [endpoint]: models }`, or `{}` when + * the name is neither a configured list, a served custom endpoint nor a built-in provider. + */ + async function loadEndpointModels( + req: ServerRequest, + config: AppConfig, + endpoint: string, + ): Promise { + const configured = await deps.loadConfigModels(req, endpoint); + if (Object.prototype.hasOwnProperty.call(configured, endpoint)) { + return { [endpoint]: configured[endpoint] }; + } + return (deps.loadDefaultEndpointModels ?? loadDefaultEndpointModels)(req, config, endpoint); + } + + /** One load per request and endpoint name, shared while in flight and kept once settled. */ + function loadRequestEntry(req: ServerRequest, endpoint: string): Promise { + let entries = requestEntries.get(req); + if (entries == null) { + entries = new Map(); + requestEntries.set(req, entries); + } + const loaded = entries.get(endpoint); + if (loaded != null) { + return loaded; + } + const known = entries; + const pending: Promise = resolveConfig(req) + .then((config) => loadEndpointModels(req, config, endpoint)) + .catch((error: unknown) => { + /** A failed load is not remembered, so a later validation in the request may retry. */ + if (known.get(endpoint) === pending) { + known.delete(endpoint); + } + throw error; + }); + known.set(endpoint, pending); + return pending; + } + async function getRequestModelsConfig(req: ServerRequest): Promise { const endpoint = req.query.endpoint; if (endpoint != null && (typeof endpoint !== 'string' || endpoint.length === 0)) { @@ -50,8 +112,7 @@ export function createModelsService(deps: ModelsServiceDeps): { if (endpoint == null) { return { ok: true, modelsConfig: await loadModels(req) }; } - const config = req.config ?? (await deps.getAppConfig(getAppConfigOptionsFromUser(req.user))); - req.config = config; + const config = await resolveConfig(req); const { getEndpointsConfig } = createEndpointsConfigService({ getAppConfig: deps.getAppConfig, loadDefaultEndpointsConfig: async () => { @@ -76,16 +137,31 @@ export function createModelsService(deps: ModelsServiceDeps): { if (!Object.prototype.hasOwnProperty.call(endpoints, endpoint)) { return { ok: false, status: 404, error: 'Endpoint not found' }; } - const configured = await deps.loadConfigModels(req, endpoint); - if (Object.prototype.hasOwnProperty.call(configured, endpoint)) { - return { ok: true, modelsConfig: { [endpoint]: configured[endpoint] } }; + const loaded = await loadEndpointModels(req, config, endpoint); + return { ok: true, modelsConfig: { [endpoint]: loaded[endpoint] ?? [] } }; + } + + /** + * The part of the full catalog that `resolveModelCatalogKey(endpoint, catalog)` reads for an + * agent provider or request endpoint: the entry under the exact name when the catalog has one, + * otherwise the entry under the alias or served name it falls back to (`vertexai` reads + * `google`, `Ollama` reads `ollama`). No other endpoint is loaded, and nothing is gated on the + * endpoint being enabled, as in the full catalog. Built-in defaults come from the same source + * as `GET /api/models?endpoint=`. Entries are loaded once per request and endpoint name. + */ + async function getEndpointModelsConfig( + req: ServerRequest, + endpoint: string | null | undefined, + ): Promise { + if (typeof endpoint !== 'string' || endpoint.length === 0) { + return {}; } - const models = await (deps.loadDefaultEndpointModels ?? loadDefaultEndpointModels)( - req, - config, - endpoint, - ); - return { ok: true, modelsConfig: { [endpoint]: models[endpoint] ?? [] } }; + const exact = await loadRequestEntry(req, endpoint); + if (hasEntry(exact, endpoint)) { + return exact; + } + const key = resolveModelCatalogKey(endpoint, {}); + return key === endpoint ? exact : loadRequestEntry(req, key); } async function modelController(req: ServerRequest, res: Response): Promise { @@ -101,5 +177,11 @@ export function createModelsService(deps: ModelsServiceDeps): { } } - return { modelController, loadModels, getModelsConfig: loadModels, getRequestModelsConfig }; + return { + modelController, + loadModels, + getModelsConfig: loadModels, + getRequestModelsConfig, + getEndpointModelsConfig, + }; } diff --git a/packages/api/src/endpoints/models.spec.ts b/packages/api/src/endpoints/models.spec.ts index 7995fe751d..b16d183f9a 100644 --- a/packages/api/src/endpoints/models.spec.ts +++ b/packages/api/src/endpoints/models.spec.ts @@ -1,6 +1,7 @@ import axios from 'axios'; import Anthropic from '@anthropic-ai/sdk'; import { Time, EModelEndpoint, defaultModels, AuthType } from 'librechat-data-provider'; +import type { FetchModelsParams } from './models'; import type { ServerRequest } from '~/types'; import { fetchModels, @@ -1334,3 +1335,169 @@ describe('configured model lists skip provider discovery', () => { expect(mockedAxios.get).toHaveBeenCalled(); }); }); + +describe('fetchModels in-flight sharing', () => { + type ModelsResponse = { data: { data: Array> } }; + + function deferred() { + let resolve!: (value: T) => void; + let reject!: (reason: unknown) => void; + const promise = new Promise((done, fail) => { + resolve = done; + reject = fail; + }); + return { promise, resolve, reject }; + } + + /** Lets every caller pass its cache lookup and reach the provider request. */ + const flush = () => new Promise((resolve) => setImmediate(resolve)); + const listing = (...ids: string[]): ModelsResponse => ({ + data: { data: ids.map((id) => ({ id })) }, + }); + + beforeEach(() => { + mockedAxios.get.mockReset(); + }); + + afterAll(() => { + mockedAxios.get.mockReset().mockResolvedValue(listing('model-1', 'model-2')); + }); + + it('sends one provider request for concurrent callers that miss the shared cache', async () => { + const gate = deferred(); + mockedAxios.get.mockReturnValue(gate.promise); + const params = { apiKey: 'key', baseURL: 'https://shared.test/v1', name: 'Shared' }; + + const calls = Array.from({ length: 5 }, () => fetchModels(params)); + await flush(); + gate.resolve(listing('model-a', 'model-b')); + const results = await Promise.all(calls); + + expect(mockedAxios.get).toHaveBeenCalledTimes(1); + expect(results).toEqual(Array.from({ length: 5 }, () => ['model-a', 'model-b'])); + }); + + it('keeps each caller storing the token config under its own key', async () => { + const gate = deferred(); + mockedAxios.get.mockReturnValue(gate.promise); + const params = { apiKey: 'key', baseURL: 'https://pricing.test/v1', name: 'Pricing' }; + const priced = { + id: 'model-a', + pricing: { prompt: '0.002', completion: '0.001' }, + context_length: 1024, + }; + + const calls = ['token-a', 'token-b'].map((tokenKey) => fetchModels({ ...params, tokenKey })); + await flush(); + gate.resolve({ data: { data: [priced] } }); + await expect(Promise.all(calls)).resolves.toEqual([['model-a'], ['model-a']]); + + expect(mockedAxios.get).toHaveBeenCalledTimes(1); + for (const tokenKey of ['token-a', 'token-b']) { + expect(mockTokenConfigSet).toHaveBeenCalledWith( + tokenKey, + expect.objectContaining({ 'model-a': expect.objectContaining({ context: 1024 }) }), + ); + } + }); + + it.each<[string, Partial]>([ + ['another base URL', { baseURL: 'https://other.test/v1' }], + ['another API key', { apiKey: 'other-key' }], + ['another native provider', { provider: EModelEndpoint.anthropic }], + ])('does not share requests that differ by %s', async (_label, other) => { + const gate = deferred(); + mockedAxios.get.mockReturnValue(gate.promise); + const params = { apiKey: 'key', baseURL: 'https://keyed.test/v1', name: 'Keyed' }; + + const calls = [fetchModels(params), fetchModels({ ...params, ...other })]; + await flush(); + gate.resolve(listing('model-a')); + await Promise.all(calls); + + expect(mockedAxios.get).toHaveBeenCalledTimes(2); + }); + + it.each([ + [ + 'headers resolved for a user', + { headers: { Authorization: 'Bearer user-token' }, userObject: { id: 'user-1' } }, + ], + ['a user query parameter', { userIdQuery: true, user: 'user-1' }], + ['a skipped cache', { skipCache: true }], + ])('never shares a request made with %s', async (_label, scoped) => { + const gate = deferred(); + mockedAxios.get.mockReturnValue(gate.promise); + const params = { apiKey: 'key', baseURL: 'https://scoped.test/v1', name: 'Scoped', ...scoped }; + + const calls = [fetchModels(params), fetchModels(params)]; + await flush(); + gate.resolve(listing('model-a')); + await Promise.all(calls); + + expect(mockedAxios.get).toHaveBeenCalledTimes(2); + }); + + it('does not remember a failed request for later callers', async () => { + const gate = deferred(); + mockedAxios.get.mockReturnValueOnce(gate.promise); + const params = { apiKey: 'key', baseURL: 'https://failing.test/v1', name: 'Failing' }; + + const calls = Array.from({ length: 3 }, () => fetchModels(params)); + await flush(); + gate.reject(new Error('provider unavailable')); + await expect(Promise.all(calls)).resolves.toEqual([[], [], []]); + expect(mockedAxios.get).toHaveBeenCalledTimes(1); + + mockedAxios.get.mockResolvedValueOnce(listing('recovered')); + await expect(fetchModels(params)).resolves.toEqual(['recovered']); + expect(mockedAxios.get).toHaveBeenCalledTimes(2); + }); + + it('does not keep a finished request for later callers', async () => { + mockedAxios.get.mockResolvedValue(listing('model-a')); + const params = { apiKey: 'key', baseURL: 'https://settled.test/v1', name: 'Settled' }; + + await fetchModels(params); + await fetchModels(params); + + expect(mockedAxios.get).toHaveBeenCalledTimes(2); + }); + + it('shares one Ollama request, and one fallback request when it fails', async () => { + const tags = deferred<{ data: { models: Array<{ name: string }> } }>(); + const models = deferred(); + mockedAxios.get.mockImplementation((url: string) => + url.endsWith('/api/tags') ? tags.promise : models.promise, + ); + const params = { apiKey: 'key', baseURL: 'https://ollama.test/v1', name: 'ollama' }; + + const calls = [fetchModels(params), fetchModels(params), fetchModels(params)]; + await flush(); + expect(mockedAxios.get).toHaveBeenCalledTimes(1); + tags.reject(new Error('no tags endpoint')); + await flush(); + expect(mockedAxios.get).toHaveBeenCalledTimes(2); + models.resolve(listing('llama3')); + + await expect(Promise.all(calls)).resolves.toEqual([['llama3'], ['llama3'], ['llama3']]); + expect(mockedAxios.get).toHaveBeenCalledTimes(2); + }); + + it('shares a successful Ollama request between concurrent callers', async () => { + const tags = deferred<{ data: { models: Array<{ name: string }> } }>(); + mockedAxios.get.mockReturnValue(tags.promise); + const params = { apiKey: 'key', baseURL: 'https://ollama-ok.test/v1', name: 'ollama-local' }; + + const calls = [fetchModels(params), fetchModels(params)]; + await flush(); + tags.resolve({ data: { models: [{ name: 'llama3' }] } }); + + await expect(Promise.all(calls)).resolves.toEqual([['llama3'], ['llama3']]); + expect(mockedAxios.get).toHaveBeenCalledTimes(1); + expect(mockedAxios.get).toHaveBeenCalledWith( + 'https://ollama-ok.test/api/tags', + expect.any(Object), + ); + }); +}); diff --git a/packages/api/src/endpoints/models.ts b/packages/api/src/endpoints/models.ts index ecac0f5542..812522088d 100644 --- a/packages/api/src/endpoints/models.ts +++ b/packages/api/src/endpoints/models.ts @@ -8,9 +8,9 @@ import { EModelEndpoint, defaultModels, } from 'librechat-data-provider'; +import type { AxiosRequestConfig, AxiosResponse } from 'axios'; import type { TEndpoint } from 'librechat-data-provider'; import type { IUser } from '@librechat/data-schemas'; -import type { AxiosRequestConfig } from 'axios'; import { processModelData, extractBaseURL, @@ -27,6 +27,36 @@ import { standardCache, tokenConfigCache } from '~/cache'; type SSRFSafeAgents = ReturnType; +/** Provider requests in flight by cache key, so concurrent cache misses for one key share one. */ +const ollamaRequests = new Map>(); +const modelsRequests = new Map>(); + +/** + * Runs `start` unless a request is already in flight under `key`, which is then joined. A request + * leaves the map as soon as it settles, so neither a result nor a failure outlives it. An empty + * key is never shared. + */ +function shareInFlight( + requests: Map>, + key: string, + start: () => Promise, +): Promise { + if (!key) { + return start(); + } + const active = requests.get(key); + if (active != null) { + return active; + } + const request: Promise = start().finally(() => { + if (requests.get(key) === request) { + requests.delete(key); + } + }); + requests.set(key, request); + return request; +} + export interface FetchModelsParams { /** User ID for API requests */ user?: string; @@ -226,11 +256,13 @@ export async function fetchModels({ if (!isCustomAnthropic && name && name.toLowerCase().startsWith(KnownEndpoints.ollama)) { let ollamaModels: string[] | null = null; try { - ollamaModels = await fetchOllamaModels(baseURL ?? '', { - headers, - ssrfAgents, - user: userObject, - }); + ollamaModels = await shareInFlight(ollamaRequests, cacheKey, () => + fetchOllamaModels(baseURL ?? '', { + headers, + ssrfAgents, + user: userObject, + }), + ); } catch (ollamaError) { logAxiosError({ message: @@ -297,7 +329,10 @@ export async function fetchModels({ } applyAxiosProxyConfig(options, url); applyUserProvidedBaseURLProtection(options, ssrfAgents); - const res = await axios.get(url.toString(), options); + /** Only the request is shared: each caller still derives and stores its own token config. */ + const res = await shareInFlight(modelsRequests, cacheKey, async () => + axios.get(url.toString(), options), + ); const input = res.data;