🧭 perf: Validate Agent Models Against Their Own Endpoint Catalog (#17972)

* perf: Validate Agent Models Against Their Own Endpoint Catalog
* perf: Share In-Flight Model List Fetches
* test: Cover Agent Model Validation Across Endpoints
* test: Check Agent Model Catalog Parity With the Real Loaders
This commit is contained in:
Marco Beretta 2026-11-10 17:33:29 +01:00 • committed by GitHub
parent bcbb8290ec
commit 91fd4b6833
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 1244 additions and 108 deletions

View file

@ -12,7 +12,7 @@ jest.mock('~/server/services/Config', () => ({
loadConfigModels: (...args) => mockLoadConfigModels(...args),
}));
const { loadModels } = require('./ModelController');
const { loadModels, getEndpointModelsConfig } = require('./ModelController');
function deferred() {
let resolve;
@ -55,3 +55,22 @@ describe('loadModels', () => {
});
});
});
describe('getEndpointModelsConfig', () => {
beforeEach(() => {
jest.clearAllMocks();
});
it('loads only the requested endpoint through the wired loaders', async () => {
const req = { user: { id: 'user-1' }, config: { endpoints: {} }, query: {} };
mockLoadConfigModels.mockResolvedValue({ Custom: ['custom-model'] });
await expect(getEndpointModelsConfig(req, 'Custom')).resolves.toEqual({
Custom: ['custom-model'],
});
expect(mockLoadConfigModels).toHaveBeenCalledTimes(1);
expect(mockLoadConfigModels).toHaveBeenCalledWith(req, 'Custom');
expect(mockLoadDefaultModels).not.toHaveBeenCalled();
});
});

View file

@ -389,7 +389,7 @@ jest.mock('@librechat/api', () => ({
}));
jest.mock('~/server/controllers/ModelController', () => ({
getModelsConfig: jest.fn().mockResolvedValue({}),
getEndpointModelsConfig: jest.fn().mockResolvedValue({}),
}));
jest.mock('~/server/services/MCP', () => ({
@ -956,6 +956,27 @@ describe('OpenAIChatCompletionController', () => {
);
});
it('validates handoff agents through a loader that reads only their own provider models', async () => {
const api = require('@librechat/api');
const { getEndpointModelsConfig } = require('~/server/controllers/ModelController');
api.initializeAgent.mockResolvedValueOnce({
id: 'agent-123',
model: 'gpt-4',
model_parameters: {},
toolRegistry: {},
edges: [{ source: 'agent-123', target: 'agent-handoff' }],
});
await OpenAIChatCompletionController(req, res);
const [params] = api.discoverConnectedAgents.mock.calls.at(-1);
expect(params).not.toHaveProperty('modelsConfig');
expect(getEndpointModelsConfig).not.toHaveBeenCalled();
await params.getEndpointModels('anthropic');
expect(getEndpointModelsConfig).toHaveBeenCalledTimes(1);
expect(getEndpointModelsConfig).toHaveBeenCalledWith(req, 'anthropic');
});
describe('content filtering', () => {
it('blocks opaque inline media before text inspection or agent loading', async () => {
const api = require('@librechat/api');

View file

@ -503,7 +503,7 @@ jest.mock('~/server/services/PermissionService', () => ({
}));
jest.mock('~/server/controllers/ModelController', () => ({
getModelsConfig: jest.fn().mockResolvedValue({}),
getEndpointModelsConfig: jest.fn().mockResolvedValue({}),
}));
jest.mock('~/server/services/MCP', () => ({
@ -2647,6 +2647,29 @@ describe('createResponse controller', () => {
});
});
describe('handoff model validation', () => {
it('validates handoff agents through a loader that reads only their own provider models', async () => {
const { initializeAgent, discoverConnectedAgents } = require('@librechat/api');
const { getEndpointModelsConfig } = require('~/server/controllers/ModelController');
initializeAgent.mockResolvedValueOnce({
id: 'agent-123',
model: 'claude-3',
model_parameters: {},
toolRegistry: new Map(),
edges: [{ source: 'agent-123', target: 'agent-handoff' }],
});
await createResponse(req, res);
const [params] = discoverConnectedAgents.mock.calls.at(-1);
expect(params).not.toHaveProperty('modelsConfig');
expect(getEndpointModelsConfig).not.toHaveBeenCalled();
await params.getEndpointModels('anthropic');
expect(getEndpointModelsConfig).toHaveBeenCalledTimes(1);
expect(getEndpointModelsConfig).toHaveBeenCalledWith(req, 'anthropic');
});
});
describe('sub-agent skill priming', () => {
it('passes the sub-agent primed skill IDs into non-streaming tool execution', async () => {
const {

View file

@ -110,7 +110,7 @@ const { checkSessionsAlive, loadCodeApiKey } = require('~/server/services/Files/
const {
getLinkedInstructionsResolver,
} = require('~/server/services/Endpoints/agents/linkedInstructions');
const { getModelsConfig } = require('~/server/controllers/ModelController');
const { getEndpointModelsConfig } = require('~/server/controllers/ModelController');
const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions');
const { resolveConfigServers } = require('~/server/services/MCP');
const { getMCPManager } = require('~/config');
@ -698,7 +698,6 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
primaryConfig.subagents?.enabled === true &&
(primaryConfig.subagents.graphs?.length ?? 0) > 0;
if (primaryConfig.edges?.length || primaryHasGraphSubagents) {
const modelsConfig = await getModelsConfig(req);
const discoveryParams = {
req,
res,
@ -706,7 +705,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
primaryConfig,
endpointOption,
allowedProviders,
modelsConfig,
getEndpointModels: (endpoint) => getEndpointModelsConfig(req, endpoint),
loadTools,
requestFiles: [],
conversationId,

View file

@ -126,7 +126,7 @@ const { checkSessionsAlive, loadCodeApiKey } = require('~/server/services/Files/
const {
getLinkedInstructionsResolver,
} = require('~/server/services/Endpoints/agents/linkedInstructions');
const { getModelsConfig } = require('~/server/controllers/ModelController');
const { getEndpointModelsConfig } = require('~/server/controllers/ModelController');
const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions');
const { resolveConfigServers, getAccessibleMcpServerNames } = require('~/server/services/MCP');
const { resolveConversationTitle } = require('~/server/services/Endpoints/titlePolicy');
@ -934,7 +934,6 @@ const executeResponse = async (envelope, { req, res }) => {
primaryConfig.subagents?.enabled === true &&
(primaryConfig.subagents.graphs?.length ?? 0) > 0;
if (primaryConfig.edges?.length || primaryHasGraphSubagents) {
const modelsConfig = await getModelsConfig(req);
const discoveryParams = {
req,
res,
@ -942,7 +941,7 @@ const executeResponse = async (envelope, { req, res }) => {
primaryConfig,
endpointOption,
allowedProviders,
modelsConfig,
getEndpointModels: (endpoint) => getEndpointModelsConfig(req, endpoint),
loadTools,
requestFiles: [],
conversationId,

View file

@ -0,0 +1,231 @@
jest.mock('@librechat/api', () => ({
...jest.requireActual('@librechat/api'),
fetchModels: jest.fn(),
}));
jest.mock('./app');
jest.mock('~/models', () => ({ getUserKeyValues: jest.fn(), initializeMessageBudget: jest.fn() }));
jest.mock('@librechat/data-schemas', () => ({
...jest.requireActual('@librechat/data-schemas'),
logger: { debug: jest.fn(), error: jest.fn(), warn: jest.fn(), info: jest.fn() },
}));
const { resolveModelCatalogKey } = require('librechat-data-provider');
const { createModelsService, fetchModels } = require('@librechat/api');
const { getUserKeyValues } = require('~/models');
const loadDefaultModels = require('./loadDefaultModels');
const loadConfigModels = require('./loadConfigModels');
/**
* The per-endpoint catalog agents are validated against, compared with the full catalog through
* the loaders the app wires: the aggregate default loader, `loadConfigModels` and the service. Only
* what leaves the process is replaced: the provider model request (`fetchModels`), the user key
* store and the config service. `getEndpointModelsConfig` must read the entry the full catalog holds.
*/
const ENV_KEYS = [
'OPENAI_API_KEY',
'OPENAI_MODELS',
'OPENAI_REVERSE_PROXY',
'ANTHROPIC_API_KEY',
'ANTHROPIC_MODELS',
'ANTHROPIC_REVERSE_PROXY',
'GOOGLE_MODELS',
'BEDROCK_AWS_MODELS',
'AZURE_OPENAI_MODELS',
'ASSISTANTS_MODELS',
'ASSISTANTS_BASE_URL',
'ENDPOINTS',
];
const customEndpoints = [
{
name: 'Ollama',
baseURL: 'http://localhost:11434/v1',
apiKey: 'ollama',
models: { default: ['llama3.2'], fetch: false },
},
{
name: 'My Proxy',
baseURL: 'https://proxy.example/v1',
apiKey: 'proxy-key',
models: { default: ['proxy-model'], fetch: false },
},
{
name: 'Fetching',
baseURL: 'https://fetch.example/v1',
apiKey: 'fetch-key',
headers: { 'X-Team': 'ml' },
models: { default: ['fallback'], fetch: true },
},
{
name: 'Personal',
baseURL: 'https://personal.example/v1',
apiKey: 'user_provided',
models: { default: ['personal-fallback'], fetch: true },
},
{ name: 'Unserved', baseURL: 'https://unserved.example/v1', apiKey: 'unserved-key' },
];
/** Every provider an agent can be saved with, in the spellings the catalog lookup resolves. */
const providers = [
'openAI',
'anthropic',
'google',
'vertexai',
'azureOpenAI',
'assistants',
'bedrock',
'Ollama',
'ollama',
'OLLAMA',
'My Proxy',
'my proxy',
'Fetching',
'Personal',
'Unserved',
'Unknown',
];
const variants = [
['built-in defaults', {}, {}, { Ollama: ['llama3.2'], 'My Proxy': ['proxy-model'] }],
[
'environment lists',
{},
{
OPENAI_MODELS: 'env-gpt',
ANTHROPIC_MODELS: 'env-claude',
GOOGLE_MODELS: 'env-gemini',
BEDROCK_AWS_MODELS: 'env-bedrock',
AZURE_OPENAI_MODELS: 'env-azure',
ASSISTANTS_MODELS: 'env-assistant',
},
{
openAI: ['env-gpt'],
anthropic: ['env-claude'],
google: ['env-gemini'],
bedrock: ['env-bedrock'],
azureOpenAI: ['env-azure'],
assistants: ['env-assistant'],
},
],
[
'configured lists',
{
openAI: { models: ['cfg-gpt'] },
anthropic: { models: ['cfg-claude'] },
google: { models: ['cfg-gemini'] },
bedrock: { models: ['cfg-bedrock'] },
},
{ OPENAI_MODELS: 'env-gpt' },
{
openAI: ['cfg-gpt'],
anthropic: ['cfg-claude'],
google: ['cfg-gemini'],
bedrock: ['cfg-bedrock'],
},
],
[
'Vertex model names',
{ anthropic: { vertexConfig: { modelNames: ['vertex-claude'] } } },
{},
{ anthropic: ['vertex-claude'] },
],
[
'Azure model names',
{ azureOpenAI: { modelNames: ['azure-deployment'] } },
{},
{ azureOpenAI: ['azure-deployment'] },
],
[
'user provided keys',
{},
{ OPENAI_API_KEY: 'user_provided', ANTHROPIC_API_KEY: 'user_provided' },
],
['built-ins that ENDPOINTS does not enable', {}, { ENDPOINTS: 'agents' }],
];
const lookup = (catalog, provider) => catalog[resolveModelCatalogKey(provider, catalog)];
describe('per-endpoint model catalog against the full catalog, with the real loaders', () => {
const saved = {};
const service = createModelsService({
getAppConfig: async () => {
throw new Error('the config is on the request');
},
loadDefaultModels,
loadConfigModels,
loadAsyncEndpoints: async () => ({}),
defaultEndpointsConfig: {},
});
const requestFor = (endpoints) => ({
user: { id: 'user-1' },
config: { endpoints: { custom: customEndpoints, ...endpoints } },
query: {},
body: {},
});
beforeEach(() => {
jest.clearAllMocks();
for (const key of ENV_KEYS) {
saved[key] = process.env[key];
delete process.env[key];
}
fetchModels.mockResolvedValue(['fetched-a', 'fetched-b']);
getUserKeyValues.mockResolvedValue({ apiKey: 'personal-key' });
});
afterEach(() => {
for (const key of ENV_KEYS) {
if (saved[key] == null) {
delete process.env[key];
} else {
process.env[key] = saved[key];
}
}
});
it.each(variants)(
'reads the entry the full catalog holds with %s',
async (_label, endpoints, env, anchors = {}) => {
Object.assign(process.env, env);
const full = await service.loadModels(requestFor(endpoints));
const read = {};
const expected = {};
for (const provider of providers) {
read[provider] = lookup(
await service.getEndpointModelsConfig(requestFor(endpoints), provider),
provider,
);
expected[provider] = lookup(full, provider);
}
expect(read).toEqual(expected);
expect(read).toMatchObject({ ...anchors, Fetching: ['fetched-a', 'fetched-b'] });
expect(read.openAI).toEqual(expect.any(Array));
expect(read.Unknown).toBeUndefined();
expect(read.Unserved).toBeUndefined();
},
);
it.each(['Fetching', 'Personal'])(
'asks the provider for %s exactly as the full catalog does',
async (name) => {
await service.loadModels(requestFor({}));
const full = fetchModels.mock.calls.map(([params]) => params).filter((p) => p.name === name);
fetchModels.mockClear();
await service.getEndpointModelsConfig(requestFor({}), name);
const scoped = fetchModels.mock.calls.map(([params]) => params);
expect(full).toHaveLength(1);
expect(scoped).toEqual(full);
},
);
it('is not reproduced for Azure Assistants without assistantModels, which keeps the Assistants middleware on the full catalog', async () => {
const full = await service.loadModels(requestFor({}));
const entry = await service.getEndpointModelsConfig(requestFor({}), 'azureAssistants');
expect(lookup(entry, 'azureAssistants')).not.toEqual(lookup(full, 'azureAssistants'));
});
});

View file

@ -45,7 +45,7 @@ const loadAddedAgent = (params) =>
* @param {import('express').Request} params.req
* @param {import('express').Response} params.res
* @param {Object} params.endpointOption - The endpoint option containing addedConvo
* @param {Object} params.modelsConfig - The models configuration
* @param {Function} params.getEndpointModels - Loads the model catalog entry for an endpoint
* @param {Function} params.logViolation - Function to log violations
* @param {Function} params.loadTools - Function to load agent tools
* @param {Array} params.requestFiles - Request files
@ -82,7 +82,7 @@ const processAddedConvo = async ({
req,
res,
endpointOption,
modelsConfig,
getEndpointModels,
logViolation,
loadTools,
requestFiles,
@ -132,7 +132,7 @@ const processAddedConvo = async ({
const addedValidation = await validateAgentModel({
req,
res,
modelsConfig,
getEndpointModels,
logViolation,
agent: addedAgent,
});

View file

@ -97,7 +97,7 @@ describe('processAddedConvo', () => {
req: makeReq(),
res: {},
endpointOption: { addedConvo: { model: 'gpt-4o', agent_id: 'added-agent' } },
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: jest.fn().mockResolvedValue({ openai: ['gpt-4o'] }),
logViolation: jest.fn(),
loadTools: jest.fn(),
requestFiles: [],
@ -111,6 +111,21 @@ describe('processAddedConvo', () => {
...overrides,
});
it('validates the added agent through the endpoint model loader', async () => {
const params = baseParams();
await processAddedConvo(params);
expect(mockValidateAgentModel).toHaveBeenCalledTimes(1);
expect(mockValidateAgentModel).toHaveBeenCalledWith(
expect.objectContaining({
agent: { id: 'added-agent', provider: 'openai' },
getEndpointModels: params.getEndpointModels,
}),
);
expect(mockValidateAgentModel.mock.calls[0][0]).not.toHaveProperty('modelsConfig');
});
it('forwards codeEnvAvailable=true to the added-convo initializeAgent call', async () => {
await processAddedConvo(baseParams({ codeEnvAvailable: true }));

View file

@ -106,7 +106,7 @@ const {
digestFile,
} = require('~/server/services/Files/provision');
const { createProvisionFilesCallback } = require('~/server/services/Files/provisionCallback');
const { getModelsConfig } = require('~/server/controllers/ModelController');
const { getEndpointModelsConfig } = require('~/server/controllers/ModelController');
const { checkPermission, findAccessibleResources } = require('~/server/services/PermissionService');
const AgentClient = require('~/server/controllers/agents/client');
const { processAddedConvo } = require('./addedConvo');
@ -376,9 +376,10 @@ const initializeClientWithProvider = async ({
}),
);
const primaryAgentPromise = endpointOption.agent;
const modelsConfigPromise = getModelsConfig(req);
const validatedPrimaryAgentPromise = Promise.all([primaryAgentPromise, modelsConfigPromise]).then(
async ([primaryAgent, modelsConfig]) => {
/** Each validation, including handoff and subagents, loads only its agent's provider models. */
const getEndpointModels = (endpoint) => getEndpointModelsConfig(req, endpoint);
const validatedPrimaryAgentPromise = Promise.resolve(primaryAgentPromise).then(
async (primaryAgent) => {
if (!primaryAgent) {
throw new Error('Agent not found');
}
@ -386,7 +387,7 @@ const initializeClientWithProvider = async ({
const validationResult = await validateAgentModel({
req,
res,
modelsConfig,
getEndpointModels,
logViolation,
agent: primaryAgent,
});
@ -394,7 +395,7 @@ const initializeClientWithProvider = async ({
throw new Error(validationResult.error?.message);
}
return { primaryAgent, modelsConfig };
return primaryAgent;
},
);
@ -662,7 +663,7 @@ const initializeClientWithProvider = async ({
editableSkillIds,
skillCreateAllowed,
{ skillStates, defaultActiveOnShare },
{ primaryAgent, modelsConfig },
primaryAgent,
requestConversation,
chatProjectContext,
toolRoleGrants,
@ -884,7 +885,7 @@ const initializeClientWithProvider = async ({
agent_ids: primaryConfig.agent_ids,
endpointOption,
allowedProviders,
modelsConfig,
getEndpointModels,
loadTools,
requestFiles,
conversationId,
@ -985,7 +986,7 @@ const initializeClientWithProvider = async ({
res,
loadTools,
logViolation,
modelsConfig,
getEndpointModels,
requestFiles,
agentConfigs,
primaryAgent,
@ -1391,7 +1392,7 @@ const initializeClientWithProvider = async ({
throw new Error(`You no longer have access to subagent ${agentId}.`);
}
const validation = await waitForAbort(
validateAgentModel({ req, res, agent, modelsConfig, logViolation }),
validateAgentModel({ req, res, agent, getEndpointModels, logViolation }),
context.signal,
);
throwIfAborted(context.signal);

View file

@ -74,7 +74,7 @@ jest.mock('~/server/services/ToolService', () => ({
}));
jest.mock('~/server/controllers/ModelController', () => ({
getModelsConfig: jest.fn().mockResolvedValue({}),
getEndpointModelsConfig: jest.fn().mockResolvedValue({}),
}));
let agentClientArgs;
@ -102,7 +102,7 @@ const { createInitializeClient, initializeClient } = require('./initialize');
const { processAddedConvo } = require('./addedConvo');
const { getSkillDbMethods, getSkillToolDeps } = require('./skillDeps');
const { loadAgentTools } = require('~/server/services/ToolService');
const { getModelsConfig } = require('~/server/controllers/ModelController');
const { getEndpointModelsConfig } = require('~/server/controllers/ModelController');
const { logger } = require('@librechat/data-schemas');
const { User, AclEntry } = require('~/db/models');
const { createAgent, createSkill, updateAgent } = require('~/models');
@ -1131,7 +1131,11 @@ describe('initializeClient — processAgent ACL gate', () => {
const req = makeReq();
req.config.endpoints.agents = { capabilities: ['skills'] };
mockInitializeAgent.mockResolvedValue(makePrimaryConfig([]));
getModelsConfig.mockReturnValueOnce(models.promise);
getEndpointModelsConfig.mockReturnValueOnce(models.promise);
mockValidateAgentModel.mockImplementationOnce(async ({ agent, getEndpointModels }) => {
await getEndpointModels(agent.provider);
return { isValid: true };
});
const canCreateSkillSpy = jest
.spyOn(getSkillToolDeps(), 'canCreateSkill')
.mockReturnValue(createPermission.promise);
@ -1144,12 +1148,15 @@ describe('initializeClient — processAgent ACL gate', () => {
endpointOption: makeEndpointOption(),
});
expect(getModelsConfig).toHaveBeenCalledWith(req);
expect(canCreateSkillSpy).toHaveBeenCalledWith({ req });
await new Promise((resolve) => setImmediate(resolve));
expect(mockValidateAgentModel).toHaveBeenCalledTimes(1);
expect(getEndpointModelsConfig).toHaveBeenCalledTimes(1);
expect(getEndpointModelsConfig).toHaveBeenCalledWith(req, 'openai');
expect(mockInitializeAgent).not.toHaveBeenCalled();
models.resolve({});
await new Promise((resolve) => setImmediate(resolve));
expect(mockValidateAgentModel).toHaveBeenCalledTimes(1);
expect(mockInitializeAgent).not.toHaveBeenCalled();
createPermission.resolve(false);
@ -1167,7 +1174,11 @@ describe('initializeClient — processAgent ACL gate', () => {
delete req.resolvedConversation;
const getConvoSpy = jest.spyOn(db, 'getConvo').mockReturnValue(conversation.promise);
const getChatProjectSpy = jest.spyOn(db, 'getChatProject').mockReturnValue(project.promise);
getModelsConfig.mockReturnValueOnce(models.promise);
getEndpointModelsConfig.mockReturnValueOnce(models.promise);
mockValidateAgentModel.mockImplementationOnce(async ({ agent, getEndpointModels }) => {
await getEndpointModels(agent.provider);
return { isValid: true };
});
mockInitializeAgent.mockResolvedValue(makePrimaryConfig([]));
try {
@ -1279,7 +1290,7 @@ describe('initializeClient — processAgent ACL gate', () => {
],
},
};
getModelsConfig.mockResolvedValueOnce(catalog);
getEndpointModelsConfig.mockResolvedValueOnce(catalog);
mockValidateAgentModel.mockImplementation(actualApi.validateAgentModel);
mockInitializeAgent.mockImplementation(actualApi.initializeAgent);
loadAgentTools.mockResolvedValue({ tools: [], toolContextMap: {}, toolDefinitions: [] });
@ -3750,6 +3761,39 @@ describe('initializeClient — subagent loading', () => {
expect(mockInitializeAgent).toHaveBeenCalledTimes(1);
});
it('validates a selected subagent through the same endpoint model loader as the primary', async () => {
await createViewableAgent(SUBAGENT_ID);
const primaryConfig = makePrimaryConfig({
subagents: { enabled: true, allowSelf: false, agent_ids: [SUBAGENT_ID] },
});
const subagentConfig = makeSubagentConfig(SUBAGENT_ID);
let call = 0;
mockInitializeAgent.mockImplementation(() =>
Promise.resolve(++call === 1 ? primaryConfig : subagentConfig),
);
const req = makeSubagentReq();
await initializeClient({
req,
res: {},
signal: new AbortController().signal,
endpointOption: makeEndpointOption(),
});
await agentClientArgs.agent.lazySubagentConfigs[0].resolve({
signal: new AbortController().signal,
});
const validations = mockValidateAgentModel.mock.calls.map(([params]) => params);
expect(validations.map(({ agent }) => agent.id)).toEqual([PRIMARY_ID, SUBAGENT_ID]);
expect(validations[1].getEndpointModels).toBe(validations[0].getEndpointModels);
for (const params of validations) {
expect(params).not.toHaveProperty('modelsConfig');
}
expect(getEndpointModelsConfig).not.toHaveBeenCalled();
await validations[1].getEndpointModels('openai');
expect(getEndpointModelsConfig).toHaveBeenCalledWith(req, 'openai');
});
it('preserves subagent tool context for ON_TOOL_EXECUTE (Codex P1 regression guard)', async () => {
/** Verifies the Codex P1 fix: `agentToolContexts.delete(subagentId)` is
* NOT called for subagent-only agents, so when the child dispatches

View file

@ -0,0 +1,137 @@
import { expect, test } from '@playwright/test';
import type { Page } from '@playwright/test';
import type { AgentDetail } from '../agents.helpers';
import {
MOCK_ENDPOINTS,
NEW_CHAT_PATH,
getAccessToken,
messagesView,
replyPrompt,
replyText,
requestJson,
sendMessage,
sendMessageAndWaitForCompletion,
} from '../helpers';
import { cleanupAgent, uniqueAgentName } from '../agents.helpers';
/**
* An agent's model is checked against the model list of its own endpoint. The mock endpoints each
* serve one fixed model, so a model outside that list is the refusal case, and a handoff target on
* the second endpoint shows each connected agent being checked against its own list.
*/
const [PRIMARY, OTHER] = MOCK_ENDPOINTS;
const handoffMarker = (label: string, routes: unknown[]) =>
`E2E_HANDOFF:${Buffer.from(JSON.stringify({ label, routes })).toString('base64url')}`;
async function createAgent(
page: Page,
token: string,
name: string,
overrides: Record<string, unknown> = {},
) {
return requestJson<AgentDetail>(page, {
path: '/api/agents',
token,
method: 'POST',
body: {
name,
description: 'Agent model catalog acceptance fixture.',
instructions: 'Keep this fixture deterministic.',
provider: PRIMARY.label,
model: PRIMARY.model,
...overrides,
},
});
}
test.describe('agent models', () => {
test('an agent whose model its endpoint does not serve is refused and named @scenario:agent-on-an-unserved-model-is-refused', async ({
page,
}) => {
test.setTimeout(120_000);
const name = uniqueAgentName('E2E Unserved Model');
const model = `mock-unserved-${Date.now()}`;
const label = `unserved-${Date.now()}`;
let agentId: string | undefined;
try {
await page.goto(NEW_CHAT_PATH, { timeout: 15_000 });
const token = await getAccessToken(page);
agentId = (await createAgent(page, token, name, { model })).id;
await page.goto(`${NEW_CHAT_PATH}?agent_id=${agentId}`, { timeout: 15_000 });
await sendMessage(page, replyPrompt(label));
await expect(
messagesView(page).getByText(
`The model "${model}" is not available for ${PRIMARY.label}. Please select a different model.`,
),
).toBeVisible({ timeout: 30_000 });
await expect(messagesView(page).getByText(replyText(label), { exact: true })).toHaveCount(0);
} finally {
await cleanupAgent(page, agentId);
}
});
test('a handoff target on another endpoint is checked against its own endpoint and answers @scenario:handoff-target-on-another-endpoint-answers', async ({
page,
}) => {
test.setTimeout(180_000);
const targetName = uniqueAgentName('E2E Other Endpoint Target');
const routerName = uniqueAgentName('E2E Other Endpoint Router');
const label = `other-endpoint-${Date.now()}`;
const payload = `receipt-${Date.now()}`;
const description = 'Send the request to the specialist on the other endpoint.';
const prompt = 'Pass the specialist the brief.';
let targetId: string | undefined;
let routerId: string | undefined;
try {
await page.goto(NEW_CHAT_PATH, { timeout: 15_000 });
const token = await getAccessToken(page);
const target = await createAgent(page, token, targetName, {
provider: OTHER.label,
model: OTHER.model,
});
targetId = target.id;
const router = await createAgent(page, token, routerName, {
edges: [
{ from: '', to: target.id, edgeType: 'handoff', description, prompt, promptKey: 'brief' },
],
});
routerId = router.id;
await page.goto(`${NEW_CHAT_PATH}?agent_id=${router.id}`, { timeout: 15_000 });
const response = await sendMessageAndWaitForCompletion(
page,
handoffMarker(label, [
{
from: router.id,
to: target.id,
description,
prompt,
promptKey: 'brief',
args: { brief: payload },
receipt: payload,
},
]),
);
expect(response.ok()).toBeTruthy();
await expect(
messagesView(page).getByText(
`E2E handoff complete ${label}: agent=${target.id}; received=${payload}`,
{ exact: true },
),
).toBeVisible({ timeout: 30_000 });
await expect(
messagesView(page).getByRole('button', { name: `Transferred to ${targetName}` }),
).toBeVisible();
} finally {
await cleanupAgent(page, routerId);
await cleanupAgent(page, targetId);
}
});
});

View file

@ -660,7 +660,7 @@ describe('initializeAgent: ChatProject context', () => {
res,
primaryConfig,
allowedProviders: new Set([Providers.OPENAI]),
modelsConfig: { openai: ['test-model'] },
getEndpointModels: async () => ({ openai: ['test-model'] }),
loadTools,
useChatProjectContext: true,
},
@ -723,7 +723,7 @@ describe('initializeAgent: ChatProject context', () => {
primaryConfig,
rootConfigs: [primaryConfig],
allowedProviders: new Set([Providers.OPENAI]),
modelsConfig: { openai: ['test-model'] },
getEndpointModels: async () => ({ openai: ['test-model'] }),
loadTools,
useChatProjectContext: true,
},

View file

@ -68,7 +68,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -104,7 +104,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -126,6 +126,44 @@ describe('discoverConnectedAgents', () => {
expect(onAgentInitialized.mock.calls.map((c) => c[0])).toEqual(['B', 'C']);
});
it('validates each referenced agent through the endpoint model loader', async () => {
const primaryConfig = makeConfig('A', [
{ from: 'A', to: 'B', edgeType: 'handoff' },
{ from: 'A', to: 'C', edgeType: 'handoff' },
]);
const agents: Record<string, Agent> = {
B: { ...makeAgent('B'), provider: 'anthropic' } as Agent,
C: makeAgent('C'),
};
const getEndpointModels = jest.fn(async () => ({}));
await discoverConnectedAgents(
{
req: makeReq(),
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
getEndpointModels,
loadTools: jest.fn(),
},
{
getAgent: jest.fn(async ({ id }: { id: string }) => agents[id] ?? null),
checkPermission: jest.fn().mockResolvedValue(true),
logViolation: jest.fn(),
db: {} as never,
},
);
expect(mockValidateAgentModel).toHaveBeenCalledTimes(2);
for (const [params] of mockValidateAgentModel.mock.calls) {
expect(params.getEndpointModels).toBe(getEndpointModels);
expect(params).not.toHaveProperty('modelsConfig');
}
expect(
mockValidateAgentModel.mock.calls.map(([params]) => params.agent.provider).sort(),
).toEqual(['anthropic', 'openai']);
});
it('skips orphaned agents and filters out edges pointing at them', async () => {
const edges: GraphEdge[] = [
{ from: 'A', to: 'B', edgeType: 'handoff' },
@ -145,7 +183,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -187,7 +225,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -218,7 +256,7 @@ describe('discoverConnectedAgents', () => {
// controllers do this — endpoint = primary provider)
endpointOption: { endpoint: EModelEndpoint.openAI, model_parameters: {} },
allowedProviders: new Set(['openai']),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -255,7 +293,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
codeEnvAvailable: true,
},
@ -289,7 +327,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
fileSearchAvailable,
},
@ -321,7 +359,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
resolveWebSearchGrant,
},
@ -355,7 +393,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
resolveLinkedInstructions,
recordLinkedPromptUsage: false,
@ -390,7 +428,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
requestBody,
},
@ -423,7 +461,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
codeEnvAvailable: false,
},
@ -452,7 +490,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
resourceType: 'REMOTE_AGENT',
},
@ -498,7 +536,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -540,7 +578,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -586,7 +624,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -629,7 +667,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -679,7 +717,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -742,7 +780,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -785,7 +823,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -833,7 +871,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -893,7 +931,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -934,7 +972,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -980,7 +1018,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -1016,7 +1054,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -1050,7 +1088,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -1086,7 +1124,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -1126,7 +1164,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -1157,7 +1195,7 @@ describe('discoverConnectedAgents', () => {
primaryConfig,
agent_ids: ['B', 'C'],
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -1195,7 +1233,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -1229,7 +1267,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -1263,7 +1301,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -1299,7 +1337,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -1335,7 +1373,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -1369,7 +1407,7 @@ describe('discoverConnectedAgents', () => {
primaryConfig,
agent_ids: ['B'],
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -1399,7 +1437,7 @@ describe('discoverConnectedAgents', () => {
primaryConfig: makeConfig('A', edges),
agent_ids: agentIds,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -1441,7 +1479,7 @@ describe('discoverConnectedAgents', () => {
signal: controller.signal,
primaryConfig: makeConfig('A', [{ from: 'A', to: 'B', edgeType: 'handoff' as const }]),
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -1479,7 +1517,7 @@ describe('discoverConnectedAgents', () => {
res: makeRes(),
primaryConfig,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
},
{
@ -1536,7 +1574,7 @@ describe('resolveSubagentGraphs', () => {
primaryConfig,
rootConfigs: [primaryConfig],
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
resourceType: 'remote_agent',
statefulSessionsAvailable: true,
@ -1600,7 +1638,7 @@ describe('resolveSubagentGraphs', () => {
primaryConfig,
rootConfigs: [primaryConfig],
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
resourceType: 'remote_agent',
},
@ -1652,7 +1690,7 @@ describe('resolveSubagentGraphs', () => {
primaryConfig: rootConfigs[0],
rootConfigs,
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
resourceType: 'remote_agent',
},
@ -1693,7 +1731,7 @@ describe('resolveSubagentGraphs', () => {
primaryConfig,
rootConfigs: [primaryConfig],
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
resourceType: 'remote_agent',
},
@ -1746,7 +1784,7 @@ describe('resolveSubagentGraphs', () => {
primaryConfig,
rootConfigs: [primaryConfig],
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
resourceType: 'remote_agent',
},
@ -1791,7 +1829,7 @@ describe('resolveSubagentGraphs', () => {
primaryConfig,
rootConfigs: [primaryConfig],
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
resourceType: 'remote_agent',
},
@ -1845,7 +1883,7 @@ describe('resolveSubagentGraphs', () => {
primaryConfig,
rootConfigs: [primaryConfig],
allowedProviders: new Set(),
modelsConfig: { openai: ['gpt-4o'] },
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
loadTools: jest.fn(),
resourceType: 'remote_agent',
},

View file

@ -8,7 +8,6 @@ import {
import type {
Agent,
GraphEdge,
TModelsConfig,
TEndpointOption,
AgentSubagentGraph,
} from 'librechat-data-provider';
@ -68,7 +67,8 @@ export interface DiscoverConnectedAgentsParams {
agent_ids?: string[];
endpointOption?: Partial<TEndpointOption>;
allowedProviders: Set<string>;
modelsConfig: TModelsConfig;
/** Each referenced agent's provider is known only once it is loaded, so its models load then. */
getEndpointModels: ValidateAgentModelParams['getEndpointModels'];
loadTools: InitializeAgentParams['loadTools'];
requestFiles?: InitializeAgentParams['requestFiles'];
conversationId?: string | null;
@ -246,7 +246,7 @@ async function initializeReferencedAgent(
req: params.req,
res: params.res,
agent,
modelsConfig: params.modelsConfig,
getEndpointModels: params.getEndpointModels,
logViolation: deps.logViolation,
}),
params.signal,

View file

@ -194,7 +194,7 @@ describe('pinned provider configuration', () => {
req: {} as Request<unknown, unknown, unknown>,
res: {} as Response,
agent,
modelsConfig: { [EModelEndpoint.openAI]: ['gpt-4o-mini'] },
getEndpointModels: async () => ({ [EModelEndpoint.openAI]: ['gpt-4o-mini'] }),
logViolation,
});

View file

@ -1,12 +1,14 @@
import {
ErrorTypes,
EModelEndpoint,
MAX_SUBAGENTS,
setMaxSubagents,
MAX_SUBAGENT_GRAPH_NODES,
MAX_GRAPH_SUBAGENT_MEMBERS,
Providers,
ViolationTypes,
} from 'librechat-data-provider';
import type { Agent } from 'librechat-data-provider';
import type { Agent, TModelsConfig } from 'librechat-data-provider';
import type { Request, Response } from 'express';
import {
agentCreateSchema,
@ -692,17 +694,141 @@ describe('validateAgentModel', () => {
const request = {} as Request<unknown, unknown, unknown>;
const response = {} as Response;
const logViolation = jest.fn().mockResolvedValue(undefined);
const catalog = (models: TModelsConfig) => jest.fn(async (): Promise<TModelsConfig> => models);
beforeEach(() => {
logViolation.mockClear();
});
it('accepts a model in the endpoint catalog and requests only the agent provider', async () => {
const getEndpointModels = catalog({ [EModelEndpoint.openAI]: ['gpt-4o'] });
const result = await validateAgentModel({
req: request,
res: response,
agent: { provider: EModelEndpoint.openAI, model: 'gpt-4o' } as Agent,
getEndpointModels,
logViolation,
});
expect(result).toEqual({ isValid: true });
expect(getEndpointModels).toHaveBeenCalledTimes(1);
expect(getEndpointModels).toHaveBeenCalledWith(EModelEndpoint.openAI);
expect(logViolation).not.toHaveBeenCalled();
});
it('rejects an agent without a model before loading any models', async () => {
const getEndpointModels = catalog({ [EModelEndpoint.openAI]: ['gpt-4o'] });
const result = await validateAgentModel({
req: request,
res: response,
agent: { provider: EModelEndpoint.openAI, model: '' } as Agent,
getEndpointModels,
logViolation,
});
expect(result).toEqual({
isValid: false,
error: { message: `{ "type": "${ErrorTypes.MISSING_MODEL}", "info": "openAI" }` },
});
expect(getEndpointModels).not.toHaveBeenCalled();
expect(logViolation).not.toHaveBeenCalled();
});
it('reports models that could not be loaded', async () => {
const result = await validateAgentModel({
req: request,
res: response,
agent: { provider: EModelEndpoint.openAI, model: 'gpt-4o' } as Agent,
getEndpointModels: jest.fn().mockResolvedValue(undefined),
logViolation,
});
expect(result).toEqual({
isValid: false,
error: { message: `{ "type": "${ErrorTypes.MODELS_NOT_LOADED}" }` },
});
expect(logViolation).not.toHaveBeenCalled();
});
it.each([
['an empty catalog', {}],
['a catalog without the provider', { [EModelEndpoint.anthropic]: ['claude'] }],
])('reports an endpoint whose models are not loaded: %s', async (_label, models) => {
const result = await validateAgentModel({
req: request,
res: response,
agent: { provider: EModelEndpoint.openAI, model: 'gpt-4o' } as Agent,
getEndpointModels: catalog(models),
logViolation,
});
expect(result).toEqual({
isValid: false,
error: {
message: `{ "type": "${ErrorTypes.ENDPOINT_MODELS_NOT_LOADED}", "info": "openAI" }`,
},
});
expect(logViolation).not.toHaveBeenCalled();
});
it('logs one violation and rejects a model outside the endpoint catalog', async () => {
const result = await validateAgentModel({
req: request,
res: response,
agent: { provider: EModelEndpoint.openAI, model: 'gpt-5' } as Agent,
getEndpointModels: catalog({ [EModelEndpoint.openAI]: ['gpt-4o'] }),
logViolation,
});
expect(result).toEqual({
isValid: false,
error: {
message: `{ "type": "${ViolationTypes.ILLEGAL_MODEL_REQUEST}", "info": "openAI|gpt-5" }`,
},
});
expect(logViolation).toHaveBeenCalledTimes(1);
expect(logViolation).toHaveBeenCalledWith(
request,
response,
ViolationTypes.ILLEGAL_MODEL_REQUEST,
{ type: ViolationTypes.ILLEGAL_MODEL_REQUEST, model: 'gpt-5', endpoint: 'openAI' },
expect.anything(),
);
});
it('propagates a failure to load models instead of treating it as an empty catalog', async () => {
await expect(
validateAgentModel({
req: request,
res: response,
agent: { provider: EModelEndpoint.openAI, model: 'gpt-4o' } as Agent,
getEndpointModels: jest.fn().mockRejectedValue(new Error('models unavailable')),
logViolation,
}),
).rejects.toThrow('models unavailable');
expect(logViolation).not.toHaveBeenCalled();
});
it('reads the catalog under the served name for another spelling of a provider', async () => {
const result = await validateAgentModel({
req: request,
res: response,
agent: { provider: 'Ollama', model: 'llama3.2' } as Agent,
getEndpointModels: catalog({ ollama: ['llama3.2'] }),
logViolation,
});
expect(result).toEqual({ isValid: true });
});
it('uses the Google catalog for a Vertex AI agent', async () => {
const result = await validateAgentModel({
req: request,
res: response,
agent: { provider: Providers.VERTEXAI, model: 'gemini-3.7-flash' } as Agent,
modelsConfig: { [EModelEndpoint.google]: ['gemini-3.7-flash'] },
getEndpointModels: catalog({ [EModelEndpoint.google]: ['gemini-3.7-flash'] }),
logViolation,
});
@ -715,10 +841,10 @@ describe('validateAgentModel', () => {
req: request,
res: response,
agent: { provider: Providers.VERTEXAI, model: 'custom-vertex-model' } as Agent,
modelsConfig: {
getEndpointModels: catalog({
[EModelEndpoint.google]: ['gemini-3.7-flash'],
[Providers.VERTEXAI]: ['custom-vertex-model'],
},
}),
logViolation,
});
@ -731,7 +857,7 @@ describe('validateAgentModel', () => {
req: request,
res: response,
agent: { provider: Providers.VERTEXAI, model: 'gemini-not-available' } as Agent,
modelsConfig: { [EModelEndpoint.google]: ['gemini-3.7-flash'] },
getEndpointModels: catalog({ [EModelEndpoint.google]: ['gemini-3.7-flash'] }),
logViolation,
});

View file

@ -539,7 +539,11 @@ export interface ValidateAgentModelParams {
req: LooseRequest;
res: Response;
agent: Agent;
modelsConfig: TModelsConfig;
/**
* Loads the model catalog entry for an endpoint. Only the agent's own provider is requested,
* so a request never loads the model lists of endpoints none of its agents use.
*/
getEndpointModels: (endpoint: string) => Promise<TModelsConfig>;
logViolation: (
req: LooseRequest,
res: Response,
@ -567,7 +571,7 @@ interface ValidateAgentModelResult {
export async function validateAgentModel(
params: ValidateAgentModelParams,
): Promise<ValidateAgentModelResult> {
const { req, res, agent, modelsConfig, logViolation } = params;
const { req, res, agent, getEndpointModels, logViolation } = params;
const { model, provider: endpoint } = agent;
if (!model) {
@ -579,6 +583,7 @@ export async function validateAgentModel(
};
}
const modelsConfig = await getEndpointModels(endpoint);
if (!modelsConfig) {
return {
isValid: false,

View file

@ -1,4 +1,4 @@
import { FileSources } from 'librechat-data-provider';
import { FileSources, resolveModelCatalogKey } from 'librechat-data-provider';
import type { TModelsConfig } from 'librechat-data-provider';
import type { AppConfig } from '@librechat/data-schemas';
import type { Response } from 'express';
@ -238,3 +238,197 @@ describe('model discovery service', () => {
expect(deps.loadConfigModels).not.toHaveBeenCalled();
});
});
describe('per-endpoint model loading', () => {
const hasOwn = (catalog: TModelsConfig, key: string) =>
Object.prototype.hasOwnProperty.call(catalog, key);
const builtIns: TModelsConfig = {
openAI: ['gpt-default'],
anthropic: ['claude-default'],
google: ['gemini-default'],
azureOpenAI: ['azure-default'],
assistants: ['assistant-default'],
azureAssistants: ['azure-assistant-default'],
bedrock: ['bedrock-default'],
};
const served: TModelsConfig = {
anthropic: ['claude-configured'],
ollama: ['llama3.2'],
'My Proxy': ['proxy-model'],
};
/**
* Stand-in loaders that return the full catalog without a name and one entry with it. They test
* the service's own rules (lookup order, one load per request, no other endpoint loaded); that
* the entry equals the full catalog's under the real loaders is checked in
* `api/server/services/Config/catalog.spec.js`.
*/
function world(extra: TModelsConfig = {}) {
const configured = { ...served, ...extra };
const deps = {
getAppConfig: jest.fn().mockResolvedValue({ endpoints: {} } as AppConfig),
loadDefaultModels: jest.fn(async (): Promise<TModelsConfig> => ({ ...builtIns })),
loadConfigModels: jest.fn(
async (_req: ServerRequest, endpoint?: string): Promise<TModelsConfig> => {
if (endpoint == null) {
return { ...configured };
}
return hasOwn(configured, endpoint) ? { [endpoint]: configured[endpoint] } : {};
},
),
defaultEndpointsConfig: { openAI: { userProvide: false } },
loadAsyncEndpoints: jest.fn().mockResolvedValue({}),
loadDefaultEndpointModels: jest.fn(
async (
_req: ServerRequest,
_config: AppConfig,
endpoint: string,
): Promise<TModelsConfig> =>
hasOwn(builtIns, endpoint) ? { [endpoint]: builtIns[endpoint] } : {},
),
};
return { deps, ...createModelsService(deps) };
}
/** The lookup agent validation performs on a catalog. */
const lookup = (catalog: TModelsConfig, provider: string) =>
catalog[resolveModelCatalogKey(provider, catalog)];
it.each([
['a built-in without a configured list', 'openAI', ['gpt-default']],
['a built-in with a configured list', 'anthropic', ['claude-configured']],
['a served custom endpoint', 'My Proxy', ['proxy-model']],
['a custom endpoint saved under another spelling', 'Ollama', ['llama3.2']],
['a provider served under an alias', 'vertexai', ['gemini-default']],
['an endpoint with no model list', 'Unknown', undefined],
])('reads %s', async (_label, provider, models) => {
const { getEndpointModelsConfig } = world();
const entry = await getEndpointModelsConfig(request(), provider);
expect(lookup(entry, provider)).toEqual(models);
expect(Object.keys(entry).length).toBeLessThanOrEqual(1);
});
it('prefers an exact catalog over the alias', async () => {
const { getEndpointModelsConfig } = world({ vertexai: ['custom-vertex'] });
const entry = await getEndpointModelsConfig(request(), 'vertexai');
expect(lookup(entry, 'vertexai')).toEqual(['custom-vertex']);
});
it('loads the requested endpoint and no other', async () => {
const { deps, getEndpointModelsConfig } = world();
const req = request();
await getEndpointModelsConfig(req, 'openAI');
await getEndpointModelsConfig(req, 'My Proxy');
expect(deps.loadDefaultModels).not.toHaveBeenCalled();
expect(deps.loadConfigModels.mock.calls.map(([, endpoint]) => endpoint)).toEqual([
'openAI',
'My Proxy',
]);
expect(deps.loadDefaultEndpointModels.mock.calls.map(([, , endpoint]) => endpoint)).toEqual([
'openAI',
]);
});
it('loads an endpoint once per request, including concurrent and repeated calls', async () => {
const { deps, getEndpointModelsConfig } = world();
const gate = deferred<TModelsConfig>();
deps.loadConfigModels.mockReturnValueOnce(gate.promise);
const req = request();
const first = getEndpointModelsConfig(req, 'My Proxy');
const second = getEndpointModelsConfig(req, 'My Proxy');
gate.resolve({ 'My Proxy': ['proxy-model'] });
await expect(Promise.all([first, second])).resolves.toEqual([
{ 'My Proxy': ['proxy-model'] },
{ 'My Proxy': ['proxy-model'] },
]);
await getEndpointModelsConfig(req, 'My Proxy');
expect(deps.loadConfigModels).toHaveBeenCalledTimes(1);
});
it('shares the load between spellings that read the same endpoint', async () => {
const { deps, getEndpointModelsConfig } = world();
const req = request();
await getEndpointModelsConfig(req, 'ollama');
await getEndpointModelsConfig(req, 'Ollama');
expect(deps.loadConfigModels.mock.calls.filter(([, name]) => name === 'ollama')).toHaveLength(
1,
);
});
it('does not cache across requests', async () => {
const { deps, getEndpointModelsConfig } = world();
await getEndpointModelsConfig(request(), 'My Proxy');
await getEndpointModelsConfig(request(), 'My Proxy');
expect(deps.loadConfigModels).toHaveBeenCalledTimes(2);
});
it('does not remember a failed load', async () => {
const { deps, getEndpointModelsConfig } = world();
deps.loadConfigModels.mockRejectedValueOnce(new Error('provider down'));
const req = request();
await expect(getEndpointModelsConfig(req, 'My Proxy')).rejects.toThrow('provider down');
await expect(getEndpointModelsConfig(req, 'My Proxy')).resolves.toEqual({
'My Proxy': ['proxy-model'],
});
});
it('loads built-ins that ENDPOINTS does not enable', async () => {
const original = process.env.ENDPOINTS;
process.env.ENDPOINTS = 'agents';
try {
const { getEndpointModelsConfig } = world();
await expect(getEndpointModelsConfig(request(), 'bedrock')).resolves.toEqual({
bedrock: ['bedrock-default'],
});
} finally {
if (original == null) {
delete process.env.ENDPOINTS;
} else {
process.env.ENDPOINTS = original;
}
}
});
it('resolves the principal config once for the request', async () => {
const { deps, getEndpointModelsConfig } = world();
const req = request();
delete req.config;
await getEndpointModelsConfig(req, 'openAI');
await getEndpointModelsConfig(req, 'My Proxy');
expect(deps.getAppConfig).toHaveBeenCalledTimes(1);
expect(deps.loadConfigModels).toHaveBeenCalledWith(
expect.objectContaining({ config: { endpoints: {} } }),
'openAI',
);
});
it.each([undefined, null, '', 42, ['openAI'], { name: 'openAI' }])(
'loads nothing for the unusable endpoint %p',
async (endpoint) => {
const { deps, getEndpointModelsConfig } = world();
await expect(
getEndpointModelsConfig(request(), endpoint as string | null | undefined),
).resolves.toEqual({});
expect(deps.getAppConfig).not.toHaveBeenCalled();
expect(deps.loadConfigModels).not.toHaveBeenCalled();
expect(deps.loadDefaultEndpointModels).not.toHaveBeenCalled();
},
);
});

View file

@ -1,5 +1,9 @@
import { logger } from '@librechat/data-schemas';
import { EModelEndpoint, getEnabledEndpoints } from 'librechat-data-provider';
import {
EModelEndpoint,
getEnabledEndpoints,
resolveModelCatalogKey,
} from 'librechat-data-provider';
import type { TModelsConfig } from 'librechat-data-provider';
import type { AppConfig } from '@librechat/data-schemas';
import type { Response } from 'express';
@ -27,12 +31,23 @@ export interface ModelsServiceDeps {
loadDefaultEndpointModels?: typeof loadDefaultEndpointModels;
}
function hasEntry(catalog: TModelsConfig, key: string): boolean {
return Object.prototype.hasOwnProperty.call(catalog, key) && catalog[key] != null;
}
export function createModelsService(deps: ModelsServiceDeps): {
modelController: (req: ServerRequest, res: Response) => Promise<Response>;
loadModels: (req: ServerRequest) => Promise<TModelsConfig>;
getModelsConfig: (req: ServerRequest) => Promise<TModelsConfig>;
getRequestModelsConfig: (req: ServerRequest) => Promise<RequestModelsConfigResult>;
getEndpointModelsConfig: (
req: ServerRequest,
endpoint: string | null | undefined,
) => Promise<TModelsConfig>;
} {
/** Catalog entries loaded for a request, by the endpoint name they were requested under. */
const requestEntries = new WeakMap<ServerRequest, Map<string, Promise<TModelsConfig>>>();
/** Internal callers keep the full catalog, regardless of HTTP query parameters. */
async function loadModels(req: ServerRequest): Promise<TModelsConfig> {
const [defaults, configured] = await Promise.all([
@ -42,6 +57,53 @@ export function createModelsService(deps: ModelsServiceDeps): {
return { ...defaults, ...configured };
}
async function resolveConfig(req: ServerRequest): Promise<AppConfig> {
const config = req.config ?? (await deps.getAppConfig(getAppConfigOptionsFromUser(req.user)));
req.config = config;
return config;
}
/**
* What the full catalog holds under one endpoint name: `{ [endpoint]: models }`, or `{}` when
* the name is neither a configured list, a served custom endpoint nor a built-in provider.
*/
async function loadEndpointModels(
req: ServerRequest,
config: AppConfig,
endpoint: string,
): Promise<TModelsConfig> {
const configured = await deps.loadConfigModels(req, endpoint);
if (Object.prototype.hasOwnProperty.call(configured, endpoint)) {
return { [endpoint]: configured[endpoint] };
}
return (deps.loadDefaultEndpointModels ?? loadDefaultEndpointModels)(req, config, endpoint);
}
/** One load per request and endpoint name, shared while in flight and kept once settled. */
function loadRequestEntry(req: ServerRequest, endpoint: string): Promise<TModelsConfig> {
let entries = requestEntries.get(req);
if (entries == null) {
entries = new Map();
requestEntries.set(req, entries);
}
const loaded = entries.get(endpoint);
if (loaded != null) {
return loaded;
}
const known = entries;
const pending: Promise<TModelsConfig> = resolveConfig(req)
.then((config) => loadEndpointModels(req, config, endpoint))
.catch((error: unknown) => {
/** A failed load is not remembered, so a later validation in the request may retry. */
if (known.get(endpoint) === pending) {
known.delete(endpoint);
}
throw error;
});
known.set(endpoint, pending);
return pending;
}
async function getRequestModelsConfig(req: ServerRequest): Promise<RequestModelsConfigResult> {
const endpoint = req.query.endpoint;
if (endpoint != null && (typeof endpoint !== 'string' || endpoint.length === 0)) {
@ -50,8 +112,7 @@ export function createModelsService(deps: ModelsServiceDeps): {
if (endpoint == null) {
return { ok: true, modelsConfig: await loadModels(req) };
}
const config = req.config ?? (await deps.getAppConfig(getAppConfigOptionsFromUser(req.user)));
req.config = config;
const config = await resolveConfig(req);
const { getEndpointsConfig } = createEndpointsConfigService({
getAppConfig: deps.getAppConfig,
loadDefaultEndpointsConfig: async () => {
@ -76,16 +137,31 @@ export function createModelsService(deps: ModelsServiceDeps): {
if (!Object.prototype.hasOwnProperty.call(endpoints, endpoint)) {
return { ok: false, status: 404, error: 'Endpoint not found' };
}
const configured = await deps.loadConfigModels(req, endpoint);
if (Object.prototype.hasOwnProperty.call(configured, endpoint)) {
return { ok: true, modelsConfig: { [endpoint]: configured[endpoint] } };
const loaded = await loadEndpointModels(req, config, endpoint);
return { ok: true, modelsConfig: { [endpoint]: loaded[endpoint] ?? [] } };
}
/**
* The part of the full catalog that `resolveModelCatalogKey(endpoint, catalog)` reads for an
* agent provider or request endpoint: the entry under the exact name when the catalog has one,
* otherwise the entry under the alias or served name it falls back to (`vertexai` reads
* `google`, `Ollama` reads `ollama`). No other endpoint is loaded, and nothing is gated on the
* endpoint being enabled, as in the full catalog. Built-in defaults come from the same source
* as `GET /api/models?endpoint=`. Entries are loaded once per request and endpoint name.
*/
async function getEndpointModelsConfig(
req: ServerRequest,
endpoint: string | null | undefined,
): Promise<TModelsConfig> {
if (typeof endpoint !== 'string' || endpoint.length === 0) {
return {};
}
const models = await (deps.loadDefaultEndpointModels ?? loadDefaultEndpointModels)(
req,
config,
endpoint,
);
return { ok: true, modelsConfig: { [endpoint]: models[endpoint] ?? [] } };
const exact = await loadRequestEntry(req, endpoint);
if (hasEntry(exact, endpoint)) {
return exact;
}
const key = resolveModelCatalogKey(endpoint, {});
return key === endpoint ? exact : loadRequestEntry(req, key);
}
async function modelController(req: ServerRequest, res: Response): Promise<Response> {
@ -101,5 +177,11 @@ export function createModelsService(deps: ModelsServiceDeps): {
}
}
return { modelController, loadModels, getModelsConfig: loadModels, getRequestModelsConfig };
return {
modelController,
loadModels,
getModelsConfig: loadModels,
getRequestModelsConfig,
getEndpointModelsConfig,
};
}

View file

@ -1,6 +1,7 @@
import axios from 'axios';
import Anthropic from '@anthropic-ai/sdk';
import { Time, EModelEndpoint, defaultModels, AuthType } from 'librechat-data-provider';
import type { FetchModelsParams } from './models';
import type { ServerRequest } from '~/types';
import {
fetchModels,
@ -1334,3 +1335,169 @@ describe('configured model lists skip provider discovery', () => {
expect(mockedAxios.get).toHaveBeenCalled();
});
});
describe('fetchModels in-flight sharing', () => {
type ModelsResponse = { data: { data: Array<Record<string, unknown>> } };
function deferred<T>() {
let resolve!: (value: T) => void;
let reject!: (reason: unknown) => void;
const promise = new Promise<T>((done, fail) => {
resolve = done;
reject = fail;
});
return { promise, resolve, reject };
}
/** Lets every caller pass its cache lookup and reach the provider request. */
const flush = () => new Promise((resolve) => setImmediate(resolve));
const listing = (...ids: string[]): ModelsResponse => ({
data: { data: ids.map((id) => ({ id })) },
});
beforeEach(() => {
mockedAxios.get.mockReset();
});
afterAll(() => {
mockedAxios.get.mockReset().mockResolvedValue(listing('model-1', 'model-2'));
});
it('sends one provider request for concurrent callers that miss the shared cache', async () => {
const gate = deferred<ModelsResponse>();
mockedAxios.get.mockReturnValue(gate.promise);
const params = { apiKey: 'key', baseURL: 'https://shared.test/v1', name: 'Shared' };
const calls = Array.from({ length: 5 }, () => fetchModels(params));
await flush();
gate.resolve(listing('model-a', 'model-b'));
const results = await Promise.all(calls);
expect(mockedAxios.get).toHaveBeenCalledTimes(1);
expect(results).toEqual(Array.from({ length: 5 }, () => ['model-a', 'model-b']));
});
it('keeps each caller storing the token config under its own key', async () => {
const gate = deferred<ModelsResponse>();
mockedAxios.get.mockReturnValue(gate.promise);
const params = { apiKey: 'key', baseURL: 'https://pricing.test/v1', name: 'Pricing' };
const priced = {
id: 'model-a',
pricing: { prompt: '0.002', completion: '0.001' },
context_length: 1024,
};
const calls = ['token-a', 'token-b'].map((tokenKey) => fetchModels({ ...params, tokenKey }));
await flush();
gate.resolve({ data: { data: [priced] } });
await expect(Promise.all(calls)).resolves.toEqual([['model-a'], ['model-a']]);
expect(mockedAxios.get).toHaveBeenCalledTimes(1);
for (const tokenKey of ['token-a', 'token-b']) {
expect(mockTokenConfigSet).toHaveBeenCalledWith(
tokenKey,
expect.objectContaining({ 'model-a': expect.objectContaining({ context: 1024 }) }),
);
}
});
it.each<[string, Partial<FetchModelsParams>]>([
['another base URL', { baseURL: 'https://other.test/v1' }],
['another API key', { apiKey: 'other-key' }],
['another native provider', { provider: EModelEndpoint.anthropic }],
])('does not share requests that differ by %s', async (_label, other) => {
const gate = deferred<ModelsResponse>();
mockedAxios.get.mockReturnValue(gate.promise);
const params = { apiKey: 'key', baseURL: 'https://keyed.test/v1', name: 'Keyed' };
const calls = [fetchModels(params), fetchModels({ ...params, ...other })];
await flush();
gate.resolve(listing('model-a'));
await Promise.all(calls);
expect(mockedAxios.get).toHaveBeenCalledTimes(2);
});
it.each([
[
'headers resolved for a user',
{ headers: { Authorization: 'Bearer user-token' }, userObject: { id: 'user-1' } },
],
['a user query parameter', { userIdQuery: true, user: 'user-1' }],
['a skipped cache', { skipCache: true }],
])('never shares a request made with %s', async (_label, scoped) => {
const gate = deferred<ModelsResponse>();
mockedAxios.get.mockReturnValue(gate.promise);
const params = { apiKey: 'key', baseURL: 'https://scoped.test/v1', name: 'Scoped', ...scoped };
const calls = [fetchModels(params), fetchModels(params)];
await flush();
gate.resolve(listing('model-a'));
await Promise.all(calls);
expect(mockedAxios.get).toHaveBeenCalledTimes(2);
});
it('does not remember a failed request for later callers', async () => {
const gate = deferred<ModelsResponse>();
mockedAxios.get.mockReturnValueOnce(gate.promise);
const params = { apiKey: 'key', baseURL: 'https://failing.test/v1', name: 'Failing' };
const calls = Array.from({ length: 3 }, () => fetchModels(params));
await flush();
gate.reject(new Error('provider unavailable'));
await expect(Promise.all(calls)).resolves.toEqual([[], [], []]);
expect(mockedAxios.get).toHaveBeenCalledTimes(1);
mockedAxios.get.mockResolvedValueOnce(listing('recovered'));
await expect(fetchModels(params)).resolves.toEqual(['recovered']);
expect(mockedAxios.get).toHaveBeenCalledTimes(2);
});
it('does not keep a finished request for later callers', async () => {
mockedAxios.get.mockResolvedValue(listing('model-a'));
const params = { apiKey: 'key', baseURL: 'https://settled.test/v1', name: 'Settled' };
await fetchModels(params);
await fetchModels(params);
expect(mockedAxios.get).toHaveBeenCalledTimes(2);
});
it('shares one Ollama request, and one fallback request when it fails', async () => {
const tags = deferred<{ data: { models: Array<{ name: string }> } }>();
const models = deferred<ModelsResponse>();
mockedAxios.get.mockImplementation((url: string) =>
url.endsWith('/api/tags') ? tags.promise : models.promise,
);
const params = { apiKey: 'key', baseURL: 'https://ollama.test/v1', name: 'ollama' };
const calls = [fetchModels(params), fetchModels(params), fetchModels(params)];
await flush();
expect(mockedAxios.get).toHaveBeenCalledTimes(1);
tags.reject(new Error('no tags endpoint'));
await flush();
expect(mockedAxios.get).toHaveBeenCalledTimes(2);
models.resolve(listing('llama3'));
await expect(Promise.all(calls)).resolves.toEqual([['llama3'], ['llama3'], ['llama3']]);
expect(mockedAxios.get).toHaveBeenCalledTimes(2);
});
it('shares a successful Ollama request between concurrent callers', async () => {
const tags = deferred<{ data: { models: Array<{ name: string }> } }>();
mockedAxios.get.mockReturnValue(tags.promise);
const params = { apiKey: 'key', baseURL: 'https://ollama-ok.test/v1', name: 'ollama-local' };
const calls = [fetchModels(params), fetchModels(params)];
await flush();
tags.resolve({ data: { models: [{ name: 'llama3' }] } });
await expect(Promise.all(calls)).resolves.toEqual([['llama3'], ['llama3']]);
expect(mockedAxios.get).toHaveBeenCalledTimes(1);
expect(mockedAxios.get).toHaveBeenCalledWith(
'https://ollama-ok.test/api/tags',
expect.any(Object),
);
});
});

View file

@ -8,9 +8,9 @@ import {
EModelEndpoint,
defaultModels,
} from 'librechat-data-provider';
import type { AxiosRequestConfig, AxiosResponse } from 'axios';
import type { TEndpoint } from 'librechat-data-provider';
import type { IUser } from '@librechat/data-schemas';
import type { AxiosRequestConfig } from 'axios';
import {
processModelData,
extractBaseURL,
@ -27,6 +27,36 @@ import { standardCache, tokenConfigCache } from '~/cache';
type SSRFSafeAgents = ReturnType<typeof createSSRFSafeAgents>;
/** Provider requests in flight by cache key, so concurrent cache misses for one key share one. */
const ollamaRequests = new Map<string, Promise<string[]>>();
const modelsRequests = new Map<string, Promise<AxiosResponse>>();
/**
* Runs `start` unless a request is already in flight under `key`, which is then joined. A request
* leaves the map as soon as it settles, so neither a result nor a failure outlives it. An empty
* key is never shared.
*/
function shareInFlight<T>(
requests: Map<string, Promise<T>>,
key: string,
start: () => Promise<T>,
): Promise<T> {
if (!key) {
return start();
}
const active = requests.get(key);
if (active != null) {
return active;
}
const request: Promise<T> = start().finally(() => {
if (requests.get(key) === request) {
requests.delete(key);
}
});
requests.set(key, request);
return request;
}
export interface FetchModelsParams {
/** User ID for API requests */
user?: string;
@ -226,11 +256,13 @@ export async function fetchModels({
if (!isCustomAnthropic && name && name.toLowerCase().startsWith(KnownEndpoints.ollama)) {
let ollamaModels: string[] | null = null;
try {
ollamaModels = await fetchOllamaModels(baseURL ?? '', {
headers,
ssrfAgents,
user: userObject,
});
ollamaModels = await shareInFlight(ollamaRequests, cacheKey, () =>
fetchOllamaModels(baseURL ?? '', {
headers,
ssrfAgents,
user: userObject,
}),
);
} catch (ollamaError) {
logAxiosError({
message:
@ -297,7 +329,10 @@ export async function fetchModels({
}
applyAxiosProxyConfig(options, url);
applyUserProvidedBaseURLProtection(options, ssrfAgents);
const res = await axios.get(url.toString(), options);
/** Only the request is shared: each caller still derives and stores its own token config. */
const res = await shareInFlight(modelsRequests, cacheKey, async () =>
axios.get(url.toString(), options),
);
const input = res.data;