mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-19 22:43:38 +00:00
🧭 perf: Validate Agent Models Against Their Own Endpoint Catalog (#17972)
* perf: Validate Agent Models Against Their Own Endpoint Catalog * perf: Share In-Flight Model List Fetches * test: Cover Agent Model Validation Across Endpoints * test: Check Agent Model Catalog Parity With the Real Loaders
This commit is contained in:
parent
bcbb8290ec
commit
91fd4b6833
21 changed files with 1244 additions and 108 deletions
|
|
@ -12,7 +12,7 @@ jest.mock('~/server/services/Config', () => ({
|
|||
loadConfigModels: (...args) => mockLoadConfigModels(...args),
|
||||
}));
|
||||
|
||||
const { loadModels } = require('./ModelController');
|
||||
const { loadModels, getEndpointModelsConfig } = require('./ModelController');
|
||||
|
||||
function deferred() {
|
||||
let resolve;
|
||||
|
|
@ -55,3 +55,22 @@ describe('loadModels', () => {
|
|||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('getEndpointModelsConfig', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('loads only the requested endpoint through the wired loaders', async () => {
|
||||
const req = { user: { id: 'user-1' }, config: { endpoints: {} }, query: {} };
|
||||
mockLoadConfigModels.mockResolvedValue({ Custom: ['custom-model'] });
|
||||
|
||||
await expect(getEndpointModelsConfig(req, 'Custom')).resolves.toEqual({
|
||||
Custom: ['custom-model'],
|
||||
});
|
||||
|
||||
expect(mockLoadConfigModels).toHaveBeenCalledTimes(1);
|
||||
expect(mockLoadConfigModels).toHaveBeenCalledWith(req, 'Custom');
|
||||
expect(mockLoadDefaultModels).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -389,7 +389,7 @@ jest.mock('@librechat/api', () => ({
|
|||
}));
|
||||
|
||||
jest.mock('~/server/controllers/ModelController', () => ({
|
||||
getModelsConfig: jest.fn().mockResolvedValue({}),
|
||||
getEndpointModelsConfig: jest.fn().mockResolvedValue({}),
|
||||
}));
|
||||
|
||||
jest.mock('~/server/services/MCP', () => ({
|
||||
|
|
@ -956,6 +956,27 @@ describe('OpenAIChatCompletionController', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('validates handoff agents through a loader that reads only their own provider models', async () => {
|
||||
const api = require('@librechat/api');
|
||||
const { getEndpointModelsConfig } = require('~/server/controllers/ModelController');
|
||||
api.initializeAgent.mockResolvedValueOnce({
|
||||
id: 'agent-123',
|
||||
model: 'gpt-4',
|
||||
model_parameters: {},
|
||||
toolRegistry: {},
|
||||
edges: [{ source: 'agent-123', target: 'agent-handoff' }],
|
||||
});
|
||||
|
||||
await OpenAIChatCompletionController(req, res);
|
||||
|
||||
const [params] = api.discoverConnectedAgents.mock.calls.at(-1);
|
||||
expect(params).not.toHaveProperty('modelsConfig');
|
||||
expect(getEndpointModelsConfig).not.toHaveBeenCalled();
|
||||
await params.getEndpointModels('anthropic');
|
||||
expect(getEndpointModelsConfig).toHaveBeenCalledTimes(1);
|
||||
expect(getEndpointModelsConfig).toHaveBeenCalledWith(req, 'anthropic');
|
||||
});
|
||||
|
||||
describe('content filtering', () => {
|
||||
it('blocks opaque inline media before text inspection or agent loading', async () => {
|
||||
const api = require('@librechat/api');
|
||||
|
|
|
|||
|
|
@ -503,7 +503,7 @@ jest.mock('~/server/services/PermissionService', () => ({
|
|||
}));
|
||||
|
||||
jest.mock('~/server/controllers/ModelController', () => ({
|
||||
getModelsConfig: jest.fn().mockResolvedValue({}),
|
||||
getEndpointModelsConfig: jest.fn().mockResolvedValue({}),
|
||||
}));
|
||||
|
||||
jest.mock('~/server/services/MCP', () => ({
|
||||
|
|
@ -2647,6 +2647,29 @@ describe('createResponse controller', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('handoff model validation', () => {
|
||||
it('validates handoff agents through a loader that reads only their own provider models', async () => {
|
||||
const { initializeAgent, discoverConnectedAgents } = require('@librechat/api');
|
||||
const { getEndpointModelsConfig } = require('~/server/controllers/ModelController');
|
||||
initializeAgent.mockResolvedValueOnce({
|
||||
id: 'agent-123',
|
||||
model: 'claude-3',
|
||||
model_parameters: {},
|
||||
toolRegistry: new Map(),
|
||||
edges: [{ source: 'agent-123', target: 'agent-handoff' }],
|
||||
});
|
||||
|
||||
await createResponse(req, res);
|
||||
|
||||
const [params] = discoverConnectedAgents.mock.calls.at(-1);
|
||||
expect(params).not.toHaveProperty('modelsConfig');
|
||||
expect(getEndpointModelsConfig).not.toHaveBeenCalled();
|
||||
await params.getEndpointModels('anthropic');
|
||||
expect(getEndpointModelsConfig).toHaveBeenCalledTimes(1);
|
||||
expect(getEndpointModelsConfig).toHaveBeenCalledWith(req, 'anthropic');
|
||||
});
|
||||
});
|
||||
|
||||
describe('sub-agent skill priming', () => {
|
||||
it('passes the sub-agent primed skill IDs into non-streaming tool execution', async () => {
|
||||
const {
|
||||
|
|
|
|||
|
|
@ -110,7 +110,7 @@ const { checkSessionsAlive, loadCodeApiKey } = require('~/server/services/Files/
|
|||
const {
|
||||
getLinkedInstructionsResolver,
|
||||
} = require('~/server/services/Endpoints/agents/linkedInstructions');
|
||||
const { getModelsConfig } = require('~/server/controllers/ModelController');
|
||||
const { getEndpointModelsConfig } = require('~/server/controllers/ModelController');
|
||||
const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions');
|
||||
const { resolveConfigServers } = require('~/server/services/MCP');
|
||||
const { getMCPManager } = require('~/config');
|
||||
|
|
@ -698,7 +698,6 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
primaryConfig.subagents?.enabled === true &&
|
||||
(primaryConfig.subagents.graphs?.length ?? 0) > 0;
|
||||
if (primaryConfig.edges?.length || primaryHasGraphSubagents) {
|
||||
const modelsConfig = await getModelsConfig(req);
|
||||
const discoveryParams = {
|
||||
req,
|
||||
res,
|
||||
|
|
@ -706,7 +705,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
primaryConfig,
|
||||
endpointOption,
|
||||
allowedProviders,
|
||||
modelsConfig,
|
||||
getEndpointModels: (endpoint) => getEndpointModelsConfig(req, endpoint),
|
||||
loadTools,
|
||||
requestFiles: [],
|
||||
conversationId,
|
||||
|
|
|
|||
|
|
@ -126,7 +126,7 @@ const { checkSessionsAlive, loadCodeApiKey } = require('~/server/services/Files/
|
|||
const {
|
||||
getLinkedInstructionsResolver,
|
||||
} = require('~/server/services/Endpoints/agents/linkedInstructions');
|
||||
const { getModelsConfig } = require('~/server/controllers/ModelController');
|
||||
const { getEndpointModelsConfig } = require('~/server/controllers/ModelController');
|
||||
const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions');
|
||||
const { resolveConfigServers, getAccessibleMcpServerNames } = require('~/server/services/MCP');
|
||||
const { resolveConversationTitle } = require('~/server/services/Endpoints/titlePolicy');
|
||||
|
|
@ -934,7 +934,6 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
primaryConfig.subagents?.enabled === true &&
|
||||
(primaryConfig.subagents.graphs?.length ?? 0) > 0;
|
||||
if (primaryConfig.edges?.length || primaryHasGraphSubagents) {
|
||||
const modelsConfig = await getModelsConfig(req);
|
||||
const discoveryParams = {
|
||||
req,
|
||||
res,
|
||||
|
|
@ -942,7 +941,7 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
primaryConfig,
|
||||
endpointOption,
|
||||
allowedProviders,
|
||||
modelsConfig,
|
||||
getEndpointModels: (endpoint) => getEndpointModelsConfig(req, endpoint),
|
||||
loadTools,
|
||||
requestFiles: [],
|
||||
conversationId,
|
||||
|
|
|
|||
231
api/server/services/Config/catalog.spec.js
Normal file
231
api/server/services/Config/catalog.spec.js
Normal file
|
|
@ -0,0 +1,231 @@
|
|||
jest.mock('@librechat/api', () => ({
|
||||
...jest.requireActual('@librechat/api'),
|
||||
fetchModels: jest.fn(),
|
||||
}));
|
||||
jest.mock('./app');
|
||||
jest.mock('~/models', () => ({ getUserKeyValues: jest.fn(), initializeMessageBudget: jest.fn() }));
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
...jest.requireActual('@librechat/data-schemas'),
|
||||
logger: { debug: jest.fn(), error: jest.fn(), warn: jest.fn(), info: jest.fn() },
|
||||
}));
|
||||
|
||||
const { resolveModelCatalogKey } = require('librechat-data-provider');
|
||||
const { createModelsService, fetchModels } = require('@librechat/api');
|
||||
const { getUserKeyValues } = require('~/models');
|
||||
const loadDefaultModels = require('./loadDefaultModels');
|
||||
const loadConfigModels = require('./loadConfigModels');
|
||||
|
||||
/**
|
||||
* The per-endpoint catalog agents are validated against, compared with the full catalog through
|
||||
* the loaders the app wires: the aggregate default loader, `loadConfigModels` and the service. Only
|
||||
* what leaves the process is replaced: the provider model request (`fetchModels`), the user key
|
||||
* store and the config service. `getEndpointModelsConfig` must read the entry the full catalog holds.
|
||||
*/
|
||||
|
||||
const ENV_KEYS = [
|
||||
'OPENAI_API_KEY',
|
||||
'OPENAI_MODELS',
|
||||
'OPENAI_REVERSE_PROXY',
|
||||
'ANTHROPIC_API_KEY',
|
||||
'ANTHROPIC_MODELS',
|
||||
'ANTHROPIC_REVERSE_PROXY',
|
||||
'GOOGLE_MODELS',
|
||||
'BEDROCK_AWS_MODELS',
|
||||
'AZURE_OPENAI_MODELS',
|
||||
'ASSISTANTS_MODELS',
|
||||
'ASSISTANTS_BASE_URL',
|
||||
'ENDPOINTS',
|
||||
];
|
||||
|
||||
const customEndpoints = [
|
||||
{
|
||||
name: 'Ollama',
|
||||
baseURL: 'http://localhost:11434/v1',
|
||||
apiKey: 'ollama',
|
||||
models: { default: ['llama3.2'], fetch: false },
|
||||
},
|
||||
{
|
||||
name: 'My Proxy',
|
||||
baseURL: 'https://proxy.example/v1',
|
||||
apiKey: 'proxy-key',
|
||||
models: { default: ['proxy-model'], fetch: false },
|
||||
},
|
||||
{
|
||||
name: 'Fetching',
|
||||
baseURL: 'https://fetch.example/v1',
|
||||
apiKey: 'fetch-key',
|
||||
headers: { 'X-Team': 'ml' },
|
||||
models: { default: ['fallback'], fetch: true },
|
||||
},
|
||||
{
|
||||
name: 'Personal',
|
||||
baseURL: 'https://personal.example/v1',
|
||||
apiKey: 'user_provided',
|
||||
models: { default: ['personal-fallback'], fetch: true },
|
||||
},
|
||||
{ name: 'Unserved', baseURL: 'https://unserved.example/v1', apiKey: 'unserved-key' },
|
||||
];
|
||||
|
||||
/** Every provider an agent can be saved with, in the spellings the catalog lookup resolves. */
|
||||
const providers = [
|
||||
'openAI',
|
||||
'anthropic',
|
||||
'google',
|
||||
'vertexai',
|
||||
'azureOpenAI',
|
||||
'assistants',
|
||||
'bedrock',
|
||||
'Ollama',
|
||||
'ollama',
|
||||
'OLLAMA',
|
||||
'My Proxy',
|
||||
'my proxy',
|
||||
'Fetching',
|
||||
'Personal',
|
||||
'Unserved',
|
||||
'Unknown',
|
||||
];
|
||||
|
||||
const variants = [
|
||||
['built-in defaults', {}, {}, { Ollama: ['llama3.2'], 'My Proxy': ['proxy-model'] }],
|
||||
[
|
||||
'environment lists',
|
||||
{},
|
||||
{
|
||||
OPENAI_MODELS: 'env-gpt',
|
||||
ANTHROPIC_MODELS: 'env-claude',
|
||||
GOOGLE_MODELS: 'env-gemini',
|
||||
BEDROCK_AWS_MODELS: 'env-bedrock',
|
||||
AZURE_OPENAI_MODELS: 'env-azure',
|
||||
ASSISTANTS_MODELS: 'env-assistant',
|
||||
},
|
||||
{
|
||||
openAI: ['env-gpt'],
|
||||
anthropic: ['env-claude'],
|
||||
google: ['env-gemini'],
|
||||
bedrock: ['env-bedrock'],
|
||||
azureOpenAI: ['env-azure'],
|
||||
assistants: ['env-assistant'],
|
||||
},
|
||||
],
|
||||
[
|
||||
'configured lists',
|
||||
{
|
||||
openAI: { models: ['cfg-gpt'] },
|
||||
anthropic: { models: ['cfg-claude'] },
|
||||
google: { models: ['cfg-gemini'] },
|
||||
bedrock: { models: ['cfg-bedrock'] },
|
||||
},
|
||||
{ OPENAI_MODELS: 'env-gpt' },
|
||||
{
|
||||
openAI: ['cfg-gpt'],
|
||||
anthropic: ['cfg-claude'],
|
||||
google: ['cfg-gemini'],
|
||||
bedrock: ['cfg-bedrock'],
|
||||
},
|
||||
],
|
||||
[
|
||||
'Vertex model names',
|
||||
{ anthropic: { vertexConfig: { modelNames: ['vertex-claude'] } } },
|
||||
{},
|
||||
{ anthropic: ['vertex-claude'] },
|
||||
],
|
||||
[
|
||||
'Azure model names',
|
||||
{ azureOpenAI: { modelNames: ['azure-deployment'] } },
|
||||
{},
|
||||
{ azureOpenAI: ['azure-deployment'] },
|
||||
],
|
||||
[
|
||||
'user provided keys',
|
||||
{},
|
||||
{ OPENAI_API_KEY: 'user_provided', ANTHROPIC_API_KEY: 'user_provided' },
|
||||
],
|
||||
['built-ins that ENDPOINTS does not enable', {}, { ENDPOINTS: 'agents' }],
|
||||
];
|
||||
|
||||
const lookup = (catalog, provider) => catalog[resolveModelCatalogKey(provider, catalog)];
|
||||
|
||||
describe('per-endpoint model catalog against the full catalog, with the real loaders', () => {
|
||||
const saved = {};
|
||||
const service = createModelsService({
|
||||
getAppConfig: async () => {
|
||||
throw new Error('the config is on the request');
|
||||
},
|
||||
loadDefaultModels,
|
||||
loadConfigModels,
|
||||
loadAsyncEndpoints: async () => ({}),
|
||||
defaultEndpointsConfig: {},
|
||||
});
|
||||
const requestFor = (endpoints) => ({
|
||||
user: { id: 'user-1' },
|
||||
config: { endpoints: { custom: customEndpoints, ...endpoints } },
|
||||
query: {},
|
||||
body: {},
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
for (const key of ENV_KEYS) {
|
||||
saved[key] = process.env[key];
|
||||
delete process.env[key];
|
||||
}
|
||||
fetchModels.mockResolvedValue(['fetched-a', 'fetched-b']);
|
||||
getUserKeyValues.mockResolvedValue({ apiKey: 'personal-key' });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const key of ENV_KEYS) {
|
||||
if (saved[key] == null) {
|
||||
delete process.env[key];
|
||||
} else {
|
||||
process.env[key] = saved[key];
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it.each(variants)(
|
||||
'reads the entry the full catalog holds with %s',
|
||||
async (_label, endpoints, env, anchors = {}) => {
|
||||
Object.assign(process.env, env);
|
||||
const full = await service.loadModels(requestFor(endpoints));
|
||||
const read = {};
|
||||
const expected = {};
|
||||
for (const provider of providers) {
|
||||
read[provider] = lookup(
|
||||
await service.getEndpointModelsConfig(requestFor(endpoints), provider),
|
||||
provider,
|
||||
);
|
||||
expected[provider] = lookup(full, provider);
|
||||
}
|
||||
|
||||
expect(read).toEqual(expected);
|
||||
expect(read).toMatchObject({ ...anchors, Fetching: ['fetched-a', 'fetched-b'] });
|
||||
expect(read.openAI).toEqual(expect.any(Array));
|
||||
expect(read.Unknown).toBeUndefined();
|
||||
expect(read.Unserved).toBeUndefined();
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['Fetching', 'Personal'])(
|
||||
'asks the provider for %s exactly as the full catalog does',
|
||||
async (name) => {
|
||||
await service.loadModels(requestFor({}));
|
||||
const full = fetchModels.mock.calls.map(([params]) => params).filter((p) => p.name === name);
|
||||
fetchModels.mockClear();
|
||||
|
||||
await service.getEndpointModelsConfig(requestFor({}), name);
|
||||
const scoped = fetchModels.mock.calls.map(([params]) => params);
|
||||
|
||||
expect(full).toHaveLength(1);
|
||||
expect(scoped).toEqual(full);
|
||||
},
|
||||
);
|
||||
|
||||
it('is not reproduced for Azure Assistants without assistantModels, which keeps the Assistants middleware on the full catalog', async () => {
|
||||
const full = await service.loadModels(requestFor({}));
|
||||
const entry = await service.getEndpointModelsConfig(requestFor({}), 'azureAssistants');
|
||||
|
||||
expect(lookup(entry, 'azureAssistants')).not.toEqual(lookup(full, 'azureAssistants'));
|
||||
});
|
||||
});
|
||||
|
|
@ -45,7 +45,7 @@ const loadAddedAgent = (params) =>
|
|||
* @param {import('express').Request} params.req
|
||||
* @param {import('express').Response} params.res
|
||||
* @param {Object} params.endpointOption - The endpoint option containing addedConvo
|
||||
* @param {Object} params.modelsConfig - The models configuration
|
||||
* @param {Function} params.getEndpointModels - Loads the model catalog entry for an endpoint
|
||||
* @param {Function} params.logViolation - Function to log violations
|
||||
* @param {Function} params.loadTools - Function to load agent tools
|
||||
* @param {Array} params.requestFiles - Request files
|
||||
|
|
@ -82,7 +82,7 @@ const processAddedConvo = async ({
|
|||
req,
|
||||
res,
|
||||
endpointOption,
|
||||
modelsConfig,
|
||||
getEndpointModels,
|
||||
logViolation,
|
||||
loadTools,
|
||||
requestFiles,
|
||||
|
|
@ -132,7 +132,7 @@ const processAddedConvo = async ({
|
|||
const addedValidation = await validateAgentModel({
|
||||
req,
|
||||
res,
|
||||
modelsConfig,
|
||||
getEndpointModels,
|
||||
logViolation,
|
||||
agent: addedAgent,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ describe('processAddedConvo', () => {
|
|||
req: makeReq(),
|
||||
res: {},
|
||||
endpointOption: { addedConvo: { model: 'gpt-4o', agent_id: 'added-agent' } },
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: jest.fn().mockResolvedValue({ openai: ['gpt-4o'] }),
|
||||
logViolation: jest.fn(),
|
||||
loadTools: jest.fn(),
|
||||
requestFiles: [],
|
||||
|
|
@ -111,6 +111,21 @@ describe('processAddedConvo', () => {
|
|||
...overrides,
|
||||
});
|
||||
|
||||
it('validates the added agent through the endpoint model loader', async () => {
|
||||
const params = baseParams();
|
||||
|
||||
await processAddedConvo(params);
|
||||
|
||||
expect(mockValidateAgentModel).toHaveBeenCalledTimes(1);
|
||||
expect(mockValidateAgentModel).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
agent: { id: 'added-agent', provider: 'openai' },
|
||||
getEndpointModels: params.getEndpointModels,
|
||||
}),
|
||||
);
|
||||
expect(mockValidateAgentModel.mock.calls[0][0]).not.toHaveProperty('modelsConfig');
|
||||
});
|
||||
|
||||
it('forwards codeEnvAvailable=true to the added-convo initializeAgent call', async () => {
|
||||
await processAddedConvo(baseParams({ codeEnvAvailable: true }));
|
||||
|
||||
|
|
|
|||
|
|
@ -106,7 +106,7 @@ const {
|
|||
digestFile,
|
||||
} = require('~/server/services/Files/provision');
|
||||
const { createProvisionFilesCallback } = require('~/server/services/Files/provisionCallback');
|
||||
const { getModelsConfig } = require('~/server/controllers/ModelController');
|
||||
const { getEndpointModelsConfig } = require('~/server/controllers/ModelController');
|
||||
const { checkPermission, findAccessibleResources } = require('~/server/services/PermissionService');
|
||||
const AgentClient = require('~/server/controllers/agents/client');
|
||||
const { processAddedConvo } = require('./addedConvo');
|
||||
|
|
@ -376,9 +376,10 @@ const initializeClientWithProvider = async ({
|
|||
}),
|
||||
);
|
||||
const primaryAgentPromise = endpointOption.agent;
|
||||
const modelsConfigPromise = getModelsConfig(req);
|
||||
const validatedPrimaryAgentPromise = Promise.all([primaryAgentPromise, modelsConfigPromise]).then(
|
||||
async ([primaryAgent, modelsConfig]) => {
|
||||
/** Each validation, including handoff and subagents, loads only its agent's provider models. */
|
||||
const getEndpointModels = (endpoint) => getEndpointModelsConfig(req, endpoint);
|
||||
const validatedPrimaryAgentPromise = Promise.resolve(primaryAgentPromise).then(
|
||||
async (primaryAgent) => {
|
||||
if (!primaryAgent) {
|
||||
throw new Error('Agent not found');
|
||||
}
|
||||
|
|
@ -386,7 +387,7 @@ const initializeClientWithProvider = async ({
|
|||
const validationResult = await validateAgentModel({
|
||||
req,
|
||||
res,
|
||||
modelsConfig,
|
||||
getEndpointModels,
|
||||
logViolation,
|
||||
agent: primaryAgent,
|
||||
});
|
||||
|
|
@ -394,7 +395,7 @@ const initializeClientWithProvider = async ({
|
|||
throw new Error(validationResult.error?.message);
|
||||
}
|
||||
|
||||
return { primaryAgent, modelsConfig };
|
||||
return primaryAgent;
|
||||
},
|
||||
);
|
||||
|
||||
|
|
@ -662,7 +663,7 @@ const initializeClientWithProvider = async ({
|
|||
editableSkillIds,
|
||||
skillCreateAllowed,
|
||||
{ skillStates, defaultActiveOnShare },
|
||||
{ primaryAgent, modelsConfig },
|
||||
primaryAgent,
|
||||
requestConversation,
|
||||
chatProjectContext,
|
||||
toolRoleGrants,
|
||||
|
|
@ -884,7 +885,7 @@ const initializeClientWithProvider = async ({
|
|||
agent_ids: primaryConfig.agent_ids,
|
||||
endpointOption,
|
||||
allowedProviders,
|
||||
modelsConfig,
|
||||
getEndpointModels,
|
||||
loadTools,
|
||||
requestFiles,
|
||||
conversationId,
|
||||
|
|
@ -985,7 +986,7 @@ const initializeClientWithProvider = async ({
|
|||
res,
|
||||
loadTools,
|
||||
logViolation,
|
||||
modelsConfig,
|
||||
getEndpointModels,
|
||||
requestFiles,
|
||||
agentConfigs,
|
||||
primaryAgent,
|
||||
|
|
@ -1391,7 +1392,7 @@ const initializeClientWithProvider = async ({
|
|||
throw new Error(`You no longer have access to subagent ${agentId}.`);
|
||||
}
|
||||
const validation = await waitForAbort(
|
||||
validateAgentModel({ req, res, agent, modelsConfig, logViolation }),
|
||||
validateAgentModel({ req, res, agent, getEndpointModels, logViolation }),
|
||||
context.signal,
|
||||
);
|
||||
throwIfAborted(context.signal);
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ jest.mock('~/server/services/ToolService', () => ({
|
|||
}));
|
||||
|
||||
jest.mock('~/server/controllers/ModelController', () => ({
|
||||
getModelsConfig: jest.fn().mockResolvedValue({}),
|
||||
getEndpointModelsConfig: jest.fn().mockResolvedValue({}),
|
||||
}));
|
||||
|
||||
let agentClientArgs;
|
||||
|
|
@ -102,7 +102,7 @@ const { createInitializeClient, initializeClient } = require('./initialize');
|
|||
const { processAddedConvo } = require('./addedConvo');
|
||||
const { getSkillDbMethods, getSkillToolDeps } = require('./skillDeps');
|
||||
const { loadAgentTools } = require('~/server/services/ToolService');
|
||||
const { getModelsConfig } = require('~/server/controllers/ModelController');
|
||||
const { getEndpointModelsConfig } = require('~/server/controllers/ModelController');
|
||||
const { logger } = require('@librechat/data-schemas');
|
||||
const { User, AclEntry } = require('~/db/models');
|
||||
const { createAgent, createSkill, updateAgent } = require('~/models');
|
||||
|
|
@ -1131,7 +1131,11 @@ describe('initializeClient — processAgent ACL gate', () => {
|
|||
const req = makeReq();
|
||||
req.config.endpoints.agents = { capabilities: ['skills'] };
|
||||
mockInitializeAgent.mockResolvedValue(makePrimaryConfig([]));
|
||||
getModelsConfig.mockReturnValueOnce(models.promise);
|
||||
getEndpointModelsConfig.mockReturnValueOnce(models.promise);
|
||||
mockValidateAgentModel.mockImplementationOnce(async ({ agent, getEndpointModels }) => {
|
||||
await getEndpointModels(agent.provider);
|
||||
return { isValid: true };
|
||||
});
|
||||
const canCreateSkillSpy = jest
|
||||
.spyOn(getSkillToolDeps(), 'canCreateSkill')
|
||||
.mockReturnValue(createPermission.promise);
|
||||
|
|
@ -1144,12 +1148,15 @@ describe('initializeClient — processAgent ACL gate', () => {
|
|||
endpointOption: makeEndpointOption(),
|
||||
});
|
||||
|
||||
expect(getModelsConfig).toHaveBeenCalledWith(req);
|
||||
expect(canCreateSkillSpy).toHaveBeenCalledWith({ req });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(mockValidateAgentModel).toHaveBeenCalledTimes(1);
|
||||
expect(getEndpointModelsConfig).toHaveBeenCalledTimes(1);
|
||||
expect(getEndpointModelsConfig).toHaveBeenCalledWith(req, 'openai');
|
||||
expect(mockInitializeAgent).not.toHaveBeenCalled();
|
||||
|
||||
models.resolve({});
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(mockValidateAgentModel).toHaveBeenCalledTimes(1);
|
||||
expect(mockInitializeAgent).not.toHaveBeenCalled();
|
||||
|
||||
createPermission.resolve(false);
|
||||
|
|
@ -1167,7 +1174,11 @@ describe('initializeClient — processAgent ACL gate', () => {
|
|||
delete req.resolvedConversation;
|
||||
const getConvoSpy = jest.spyOn(db, 'getConvo').mockReturnValue(conversation.promise);
|
||||
const getChatProjectSpy = jest.spyOn(db, 'getChatProject').mockReturnValue(project.promise);
|
||||
getModelsConfig.mockReturnValueOnce(models.promise);
|
||||
getEndpointModelsConfig.mockReturnValueOnce(models.promise);
|
||||
mockValidateAgentModel.mockImplementationOnce(async ({ agent, getEndpointModels }) => {
|
||||
await getEndpointModels(agent.provider);
|
||||
return { isValid: true };
|
||||
});
|
||||
mockInitializeAgent.mockResolvedValue(makePrimaryConfig([]));
|
||||
|
||||
try {
|
||||
|
|
@ -1279,7 +1290,7 @@ describe('initializeClient — processAgent ACL gate', () => {
|
|||
],
|
||||
},
|
||||
};
|
||||
getModelsConfig.mockResolvedValueOnce(catalog);
|
||||
getEndpointModelsConfig.mockResolvedValueOnce(catalog);
|
||||
mockValidateAgentModel.mockImplementation(actualApi.validateAgentModel);
|
||||
mockInitializeAgent.mockImplementation(actualApi.initializeAgent);
|
||||
loadAgentTools.mockResolvedValue({ tools: [], toolContextMap: {}, toolDefinitions: [] });
|
||||
|
|
@ -3750,6 +3761,39 @@ describe('initializeClient — subagent loading', () => {
|
|||
expect(mockInitializeAgent).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('validates a selected subagent through the same endpoint model loader as the primary', async () => {
|
||||
await createViewableAgent(SUBAGENT_ID);
|
||||
const primaryConfig = makePrimaryConfig({
|
||||
subagents: { enabled: true, allowSelf: false, agent_ids: [SUBAGENT_ID] },
|
||||
});
|
||||
const subagentConfig = makeSubagentConfig(SUBAGENT_ID);
|
||||
let call = 0;
|
||||
mockInitializeAgent.mockImplementation(() =>
|
||||
Promise.resolve(++call === 1 ? primaryConfig : subagentConfig),
|
||||
);
|
||||
const req = makeSubagentReq();
|
||||
|
||||
await initializeClient({
|
||||
req,
|
||||
res: {},
|
||||
signal: new AbortController().signal,
|
||||
endpointOption: makeEndpointOption(),
|
||||
});
|
||||
await agentClientArgs.agent.lazySubagentConfigs[0].resolve({
|
||||
signal: new AbortController().signal,
|
||||
});
|
||||
|
||||
const validations = mockValidateAgentModel.mock.calls.map(([params]) => params);
|
||||
expect(validations.map(({ agent }) => agent.id)).toEqual([PRIMARY_ID, SUBAGENT_ID]);
|
||||
expect(validations[1].getEndpointModels).toBe(validations[0].getEndpointModels);
|
||||
for (const params of validations) {
|
||||
expect(params).not.toHaveProperty('modelsConfig');
|
||||
}
|
||||
expect(getEndpointModelsConfig).not.toHaveBeenCalled();
|
||||
await validations[1].getEndpointModels('openai');
|
||||
expect(getEndpointModelsConfig).toHaveBeenCalledWith(req, 'openai');
|
||||
});
|
||||
|
||||
it('preserves subagent tool context for ON_TOOL_EXECUTE (Codex P1 regression guard)', async () => {
|
||||
/** Verifies the Codex P1 fix: `agentToolContexts.delete(subagentId)` is
|
||||
* NOT called for subagent-only agents, so when the child dispatches
|
||||
|
|
|
|||
137
e2e/specs/mock/scenarios/agent-model-catalog.spec.ts
Normal file
137
e2e/specs/mock/scenarios/agent-model-catalog.spec.ts
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import type { Page } from '@playwright/test';
|
||||
import type { AgentDetail } from '../agents.helpers';
|
||||
import {
|
||||
MOCK_ENDPOINTS,
|
||||
NEW_CHAT_PATH,
|
||||
getAccessToken,
|
||||
messagesView,
|
||||
replyPrompt,
|
||||
replyText,
|
||||
requestJson,
|
||||
sendMessage,
|
||||
sendMessageAndWaitForCompletion,
|
||||
} from '../helpers';
|
||||
import { cleanupAgent, uniqueAgentName } from '../agents.helpers';
|
||||
|
||||
/**
|
||||
* An agent's model is checked against the model list of its own endpoint. The mock endpoints each
|
||||
* serve one fixed model, so a model outside that list is the refusal case, and a handoff target on
|
||||
* the second endpoint shows each connected agent being checked against its own list.
|
||||
*/
|
||||
|
||||
const [PRIMARY, OTHER] = MOCK_ENDPOINTS;
|
||||
|
||||
const handoffMarker = (label: string, routes: unknown[]) =>
|
||||
`E2E_HANDOFF:${Buffer.from(JSON.stringify({ label, routes })).toString('base64url')}`;
|
||||
|
||||
async function createAgent(
|
||||
page: Page,
|
||||
token: string,
|
||||
name: string,
|
||||
overrides: Record<string, unknown> = {},
|
||||
) {
|
||||
return requestJson<AgentDetail>(page, {
|
||||
path: '/api/agents',
|
||||
token,
|
||||
method: 'POST',
|
||||
body: {
|
||||
name,
|
||||
description: 'Agent model catalog acceptance fixture.',
|
||||
instructions: 'Keep this fixture deterministic.',
|
||||
provider: PRIMARY.label,
|
||||
model: PRIMARY.model,
|
||||
...overrides,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
test.describe('agent models', () => {
|
||||
test('an agent whose model its endpoint does not serve is refused and named @scenario:agent-on-an-unserved-model-is-refused', async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(120_000);
|
||||
const name = uniqueAgentName('E2E Unserved Model');
|
||||
const model = `mock-unserved-${Date.now()}`;
|
||||
const label = `unserved-${Date.now()}`;
|
||||
let agentId: string | undefined;
|
||||
|
||||
try {
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 15_000 });
|
||||
const token = await getAccessToken(page);
|
||||
agentId = (await createAgent(page, token, name, { model })).id;
|
||||
|
||||
await page.goto(`${NEW_CHAT_PATH}?agent_id=${agentId}`, { timeout: 15_000 });
|
||||
await sendMessage(page, replyPrompt(label));
|
||||
|
||||
await expect(
|
||||
messagesView(page).getByText(
|
||||
`The model "${model}" is not available for ${PRIMARY.label}. Please select a different model.`,
|
||||
),
|
||||
).toBeVisible({ timeout: 30_000 });
|
||||
await expect(messagesView(page).getByText(replyText(label), { exact: true })).toHaveCount(0);
|
||||
} finally {
|
||||
await cleanupAgent(page, agentId);
|
||||
}
|
||||
});
|
||||
|
||||
test('a handoff target on another endpoint is checked against its own endpoint and answers @scenario:handoff-target-on-another-endpoint-answers', async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(180_000);
|
||||
const targetName = uniqueAgentName('E2E Other Endpoint Target');
|
||||
const routerName = uniqueAgentName('E2E Other Endpoint Router');
|
||||
const label = `other-endpoint-${Date.now()}`;
|
||||
const payload = `receipt-${Date.now()}`;
|
||||
const description = 'Send the request to the specialist on the other endpoint.';
|
||||
const prompt = 'Pass the specialist the brief.';
|
||||
let targetId: string | undefined;
|
||||
let routerId: string | undefined;
|
||||
|
||||
try {
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 15_000 });
|
||||
const token = await getAccessToken(page);
|
||||
const target = await createAgent(page, token, targetName, {
|
||||
provider: OTHER.label,
|
||||
model: OTHER.model,
|
||||
});
|
||||
targetId = target.id;
|
||||
const router = await createAgent(page, token, routerName, {
|
||||
edges: [
|
||||
{ from: '', to: target.id, edgeType: 'handoff', description, prompt, promptKey: 'brief' },
|
||||
],
|
||||
});
|
||||
routerId = router.id;
|
||||
|
||||
await page.goto(`${NEW_CHAT_PATH}?agent_id=${router.id}`, { timeout: 15_000 });
|
||||
const response = await sendMessageAndWaitForCompletion(
|
||||
page,
|
||||
handoffMarker(label, [
|
||||
{
|
||||
from: router.id,
|
||||
to: target.id,
|
||||
description,
|
||||
prompt,
|
||||
promptKey: 'brief',
|
||||
args: { brief: payload },
|
||||
receipt: payload,
|
||||
},
|
||||
]),
|
||||
);
|
||||
expect(response.ok()).toBeTruthy();
|
||||
|
||||
await expect(
|
||||
messagesView(page).getByText(
|
||||
`E2E handoff complete ${label}: agent=${target.id}; received=${payload}`,
|
||||
{ exact: true },
|
||||
),
|
||||
).toBeVisible({ timeout: 30_000 });
|
||||
await expect(
|
||||
messagesView(page).getByRole('button', { name: `Transferred to ${targetName}` }),
|
||||
).toBeVisible();
|
||||
} finally {
|
||||
await cleanupAgent(page, routerId);
|
||||
await cleanupAgent(page, targetId);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
@ -660,7 +660,7 @@ describe('initializeAgent: ChatProject context', () => {
|
|||
res,
|
||||
primaryConfig,
|
||||
allowedProviders: new Set([Providers.OPENAI]),
|
||||
modelsConfig: { openai: ['test-model'] },
|
||||
getEndpointModels: async () => ({ openai: ['test-model'] }),
|
||||
loadTools,
|
||||
useChatProjectContext: true,
|
||||
},
|
||||
|
|
@ -723,7 +723,7 @@ describe('initializeAgent: ChatProject context', () => {
|
|||
primaryConfig,
|
||||
rootConfigs: [primaryConfig],
|
||||
allowedProviders: new Set([Providers.OPENAI]),
|
||||
modelsConfig: { openai: ['test-model'] },
|
||||
getEndpointModels: async () => ({ openai: ['test-model'] }),
|
||||
loadTools,
|
||||
useChatProjectContext: true,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -104,7 +104,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -126,6 +126,44 @@ describe('discoverConnectedAgents', () => {
|
|||
expect(onAgentInitialized.mock.calls.map((c) => c[0])).toEqual(['B', 'C']);
|
||||
});
|
||||
|
||||
it('validates each referenced agent through the endpoint model loader', async () => {
|
||||
const primaryConfig = makeConfig('A', [
|
||||
{ from: 'A', to: 'B', edgeType: 'handoff' },
|
||||
{ from: 'A', to: 'C', edgeType: 'handoff' },
|
||||
]);
|
||||
const agents: Record<string, Agent> = {
|
||||
B: { ...makeAgent('B'), provider: 'anthropic' } as Agent,
|
||||
C: makeAgent('C'),
|
||||
};
|
||||
const getEndpointModels = jest.fn(async () => ({}));
|
||||
|
||||
await discoverConnectedAgents(
|
||||
{
|
||||
req: makeReq(),
|
||||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
getEndpointModels,
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
getAgent: jest.fn(async ({ id }: { id: string }) => agents[id] ?? null),
|
||||
checkPermission: jest.fn().mockResolvedValue(true),
|
||||
logViolation: jest.fn(),
|
||||
db: {} as never,
|
||||
},
|
||||
);
|
||||
|
||||
expect(mockValidateAgentModel).toHaveBeenCalledTimes(2);
|
||||
for (const [params] of mockValidateAgentModel.mock.calls) {
|
||||
expect(params.getEndpointModels).toBe(getEndpointModels);
|
||||
expect(params).not.toHaveProperty('modelsConfig');
|
||||
}
|
||||
expect(
|
||||
mockValidateAgentModel.mock.calls.map(([params]) => params.agent.provider).sort(),
|
||||
).toEqual(['anthropic', 'openai']);
|
||||
});
|
||||
|
||||
it('skips orphaned agents and filters out edges pointing at them', async () => {
|
||||
const edges: GraphEdge[] = [
|
||||
{ from: 'A', to: 'B', edgeType: 'handoff' },
|
||||
|
|
@ -145,7 +183,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -187,7 +225,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -218,7 +256,7 @@ describe('discoverConnectedAgents', () => {
|
|||
// controllers do this — endpoint = primary provider)
|
||||
endpointOption: { endpoint: EModelEndpoint.openAI, model_parameters: {} },
|
||||
allowedProviders: new Set(['openai']),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -255,7 +293,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
codeEnvAvailable: true,
|
||||
},
|
||||
|
|
@ -289,7 +327,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
fileSearchAvailable,
|
||||
},
|
||||
|
|
@ -321,7 +359,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
resolveWebSearchGrant,
|
||||
},
|
||||
|
|
@ -355,7 +393,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
resolveLinkedInstructions,
|
||||
recordLinkedPromptUsage: false,
|
||||
|
|
@ -390,7 +428,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
requestBody,
|
||||
},
|
||||
|
|
@ -423,7 +461,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
codeEnvAvailable: false,
|
||||
},
|
||||
|
|
@ -452,7 +490,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
resourceType: 'REMOTE_AGENT',
|
||||
},
|
||||
|
|
@ -498,7 +536,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -540,7 +578,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -586,7 +624,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -629,7 +667,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -679,7 +717,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -742,7 +780,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -785,7 +823,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -833,7 +871,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -893,7 +931,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -934,7 +972,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -980,7 +1018,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -1016,7 +1054,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -1050,7 +1088,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -1086,7 +1124,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -1126,7 +1164,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -1157,7 +1195,7 @@ describe('discoverConnectedAgents', () => {
|
|||
primaryConfig,
|
||||
agent_ids: ['B', 'C'],
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -1195,7 +1233,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -1229,7 +1267,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -1263,7 +1301,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -1299,7 +1337,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -1335,7 +1373,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -1369,7 +1407,7 @@ describe('discoverConnectedAgents', () => {
|
|||
primaryConfig,
|
||||
agent_ids: ['B'],
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -1399,7 +1437,7 @@ describe('discoverConnectedAgents', () => {
|
|||
primaryConfig: makeConfig('A', edges),
|
||||
agent_ids: agentIds,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -1441,7 +1479,7 @@ describe('discoverConnectedAgents', () => {
|
|||
signal: controller.signal,
|
||||
primaryConfig: makeConfig('A', [{ from: 'A', to: 'B', edgeType: 'handoff' as const }]),
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -1479,7 +1517,7 @@ describe('discoverConnectedAgents', () => {
|
|||
res: makeRes(),
|
||||
primaryConfig,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
},
|
||||
{
|
||||
|
|
@ -1536,7 +1574,7 @@ describe('resolveSubagentGraphs', () => {
|
|||
primaryConfig,
|
||||
rootConfigs: [primaryConfig],
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
resourceType: 'remote_agent',
|
||||
statefulSessionsAvailable: true,
|
||||
|
|
@ -1600,7 +1638,7 @@ describe('resolveSubagentGraphs', () => {
|
|||
primaryConfig,
|
||||
rootConfigs: [primaryConfig],
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
resourceType: 'remote_agent',
|
||||
},
|
||||
|
|
@ -1652,7 +1690,7 @@ describe('resolveSubagentGraphs', () => {
|
|||
primaryConfig: rootConfigs[0],
|
||||
rootConfigs,
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
resourceType: 'remote_agent',
|
||||
},
|
||||
|
|
@ -1693,7 +1731,7 @@ describe('resolveSubagentGraphs', () => {
|
|||
primaryConfig,
|
||||
rootConfigs: [primaryConfig],
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
resourceType: 'remote_agent',
|
||||
},
|
||||
|
|
@ -1746,7 +1784,7 @@ describe('resolveSubagentGraphs', () => {
|
|||
primaryConfig,
|
||||
rootConfigs: [primaryConfig],
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
resourceType: 'remote_agent',
|
||||
},
|
||||
|
|
@ -1791,7 +1829,7 @@ describe('resolveSubagentGraphs', () => {
|
|||
primaryConfig,
|
||||
rootConfigs: [primaryConfig],
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
resourceType: 'remote_agent',
|
||||
},
|
||||
|
|
@ -1845,7 +1883,7 @@ describe('resolveSubagentGraphs', () => {
|
|||
primaryConfig,
|
||||
rootConfigs: [primaryConfig],
|
||||
allowedProviders: new Set(),
|
||||
modelsConfig: { openai: ['gpt-4o'] },
|
||||
getEndpointModels: async () => ({ openai: ['gpt-4o'] }),
|
||||
loadTools: jest.fn(),
|
||||
resourceType: 'remote_agent',
|
||||
},
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import {
|
|||
import type {
|
||||
Agent,
|
||||
GraphEdge,
|
||||
TModelsConfig,
|
||||
TEndpointOption,
|
||||
AgentSubagentGraph,
|
||||
} from 'librechat-data-provider';
|
||||
|
|
@ -68,7 +67,8 @@ export interface DiscoverConnectedAgentsParams {
|
|||
agent_ids?: string[];
|
||||
endpointOption?: Partial<TEndpointOption>;
|
||||
allowedProviders: Set<string>;
|
||||
modelsConfig: TModelsConfig;
|
||||
/** Each referenced agent's provider is known only once it is loaded, so its models load then. */
|
||||
getEndpointModels: ValidateAgentModelParams['getEndpointModels'];
|
||||
loadTools: InitializeAgentParams['loadTools'];
|
||||
requestFiles?: InitializeAgentParams['requestFiles'];
|
||||
conversationId?: string | null;
|
||||
|
|
@ -246,7 +246,7 @@ async function initializeReferencedAgent(
|
|||
req: params.req,
|
||||
res: params.res,
|
||||
agent,
|
||||
modelsConfig: params.modelsConfig,
|
||||
getEndpointModels: params.getEndpointModels,
|
||||
logViolation: deps.logViolation,
|
||||
}),
|
||||
params.signal,
|
||||
|
|
|
|||
|
|
@ -194,7 +194,7 @@ describe('pinned provider configuration', () => {
|
|||
req: {} as Request<unknown, unknown, unknown>,
|
||||
res: {} as Response,
|
||||
agent,
|
||||
modelsConfig: { [EModelEndpoint.openAI]: ['gpt-4o-mini'] },
|
||||
getEndpointModels: async () => ({ [EModelEndpoint.openAI]: ['gpt-4o-mini'] }),
|
||||
logViolation,
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,14 @@
|
|||
import {
|
||||
ErrorTypes,
|
||||
EModelEndpoint,
|
||||
MAX_SUBAGENTS,
|
||||
setMaxSubagents,
|
||||
MAX_SUBAGENT_GRAPH_NODES,
|
||||
MAX_GRAPH_SUBAGENT_MEMBERS,
|
||||
Providers,
|
||||
ViolationTypes,
|
||||
} from 'librechat-data-provider';
|
||||
import type { Agent } from 'librechat-data-provider';
|
||||
import type { Agent, TModelsConfig } from 'librechat-data-provider';
|
||||
import type { Request, Response } from 'express';
|
||||
import {
|
||||
agentCreateSchema,
|
||||
|
|
@ -692,17 +694,141 @@ describe('validateAgentModel', () => {
|
|||
const request = {} as Request<unknown, unknown, unknown>;
|
||||
const response = {} as Response;
|
||||
const logViolation = jest.fn().mockResolvedValue(undefined);
|
||||
const catalog = (models: TModelsConfig) => jest.fn(async (): Promise<TModelsConfig> => models);
|
||||
|
||||
beforeEach(() => {
|
||||
logViolation.mockClear();
|
||||
});
|
||||
|
||||
it('accepts a model in the endpoint catalog and requests only the agent provider', async () => {
|
||||
const getEndpointModels = catalog({ [EModelEndpoint.openAI]: ['gpt-4o'] });
|
||||
|
||||
const result = await validateAgentModel({
|
||||
req: request,
|
||||
res: response,
|
||||
agent: { provider: EModelEndpoint.openAI, model: 'gpt-4o' } as Agent,
|
||||
getEndpointModels,
|
||||
logViolation,
|
||||
});
|
||||
|
||||
expect(result).toEqual({ isValid: true });
|
||||
expect(getEndpointModels).toHaveBeenCalledTimes(1);
|
||||
expect(getEndpointModels).toHaveBeenCalledWith(EModelEndpoint.openAI);
|
||||
expect(logViolation).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects an agent without a model before loading any models', async () => {
|
||||
const getEndpointModels = catalog({ [EModelEndpoint.openAI]: ['gpt-4o'] });
|
||||
|
||||
const result = await validateAgentModel({
|
||||
req: request,
|
||||
res: response,
|
||||
agent: { provider: EModelEndpoint.openAI, model: '' } as Agent,
|
||||
getEndpointModels,
|
||||
logViolation,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
isValid: false,
|
||||
error: { message: `{ "type": "${ErrorTypes.MISSING_MODEL}", "info": "openAI" }` },
|
||||
});
|
||||
expect(getEndpointModels).not.toHaveBeenCalled();
|
||||
expect(logViolation).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('reports models that could not be loaded', async () => {
|
||||
const result = await validateAgentModel({
|
||||
req: request,
|
||||
res: response,
|
||||
agent: { provider: EModelEndpoint.openAI, model: 'gpt-4o' } as Agent,
|
||||
getEndpointModels: jest.fn().mockResolvedValue(undefined),
|
||||
logViolation,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
isValid: false,
|
||||
error: { message: `{ "type": "${ErrorTypes.MODELS_NOT_LOADED}" }` },
|
||||
});
|
||||
expect(logViolation).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
['an empty catalog', {}],
|
||||
['a catalog without the provider', { [EModelEndpoint.anthropic]: ['claude'] }],
|
||||
])('reports an endpoint whose models are not loaded: %s', async (_label, models) => {
|
||||
const result = await validateAgentModel({
|
||||
req: request,
|
||||
res: response,
|
||||
agent: { provider: EModelEndpoint.openAI, model: 'gpt-4o' } as Agent,
|
||||
getEndpointModels: catalog(models),
|
||||
logViolation,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
isValid: false,
|
||||
error: {
|
||||
message: `{ "type": "${ErrorTypes.ENDPOINT_MODELS_NOT_LOADED}", "info": "openAI" }`,
|
||||
},
|
||||
});
|
||||
expect(logViolation).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('logs one violation and rejects a model outside the endpoint catalog', async () => {
|
||||
const result = await validateAgentModel({
|
||||
req: request,
|
||||
res: response,
|
||||
agent: { provider: EModelEndpoint.openAI, model: 'gpt-5' } as Agent,
|
||||
getEndpointModels: catalog({ [EModelEndpoint.openAI]: ['gpt-4o'] }),
|
||||
logViolation,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
isValid: false,
|
||||
error: {
|
||||
message: `{ "type": "${ViolationTypes.ILLEGAL_MODEL_REQUEST}", "info": "openAI|gpt-5" }`,
|
||||
},
|
||||
});
|
||||
expect(logViolation).toHaveBeenCalledTimes(1);
|
||||
expect(logViolation).toHaveBeenCalledWith(
|
||||
request,
|
||||
response,
|
||||
ViolationTypes.ILLEGAL_MODEL_REQUEST,
|
||||
{ type: ViolationTypes.ILLEGAL_MODEL_REQUEST, model: 'gpt-5', endpoint: 'openAI' },
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
|
||||
it('propagates a failure to load models instead of treating it as an empty catalog', async () => {
|
||||
await expect(
|
||||
validateAgentModel({
|
||||
req: request,
|
||||
res: response,
|
||||
agent: { provider: EModelEndpoint.openAI, model: 'gpt-4o' } as Agent,
|
||||
getEndpointModels: jest.fn().mockRejectedValue(new Error('models unavailable')),
|
||||
logViolation,
|
||||
}),
|
||||
).rejects.toThrow('models unavailable');
|
||||
expect(logViolation).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('reads the catalog under the served name for another spelling of a provider', async () => {
|
||||
const result = await validateAgentModel({
|
||||
req: request,
|
||||
res: response,
|
||||
agent: { provider: 'Ollama', model: 'llama3.2' } as Agent,
|
||||
getEndpointModels: catalog({ ollama: ['llama3.2'] }),
|
||||
logViolation,
|
||||
});
|
||||
|
||||
expect(result).toEqual({ isValid: true });
|
||||
});
|
||||
|
||||
it('uses the Google catalog for a Vertex AI agent', async () => {
|
||||
const result = await validateAgentModel({
|
||||
req: request,
|
||||
res: response,
|
||||
agent: { provider: Providers.VERTEXAI, model: 'gemini-3.7-flash' } as Agent,
|
||||
modelsConfig: { [EModelEndpoint.google]: ['gemini-3.7-flash'] },
|
||||
getEndpointModels: catalog({ [EModelEndpoint.google]: ['gemini-3.7-flash'] }),
|
||||
logViolation,
|
||||
});
|
||||
|
||||
|
|
@ -715,10 +841,10 @@ describe('validateAgentModel', () => {
|
|||
req: request,
|
||||
res: response,
|
||||
agent: { provider: Providers.VERTEXAI, model: 'custom-vertex-model' } as Agent,
|
||||
modelsConfig: {
|
||||
getEndpointModels: catalog({
|
||||
[EModelEndpoint.google]: ['gemini-3.7-flash'],
|
||||
[Providers.VERTEXAI]: ['custom-vertex-model'],
|
||||
},
|
||||
}),
|
||||
logViolation,
|
||||
});
|
||||
|
||||
|
|
@ -731,7 +857,7 @@ describe('validateAgentModel', () => {
|
|||
req: request,
|
||||
res: response,
|
||||
agent: { provider: Providers.VERTEXAI, model: 'gemini-not-available' } as Agent,
|
||||
modelsConfig: { [EModelEndpoint.google]: ['gemini-3.7-flash'] },
|
||||
getEndpointModels: catalog({ [EModelEndpoint.google]: ['gemini-3.7-flash'] }),
|
||||
logViolation,
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -539,7 +539,11 @@ export interface ValidateAgentModelParams {
|
|||
req: LooseRequest;
|
||||
res: Response;
|
||||
agent: Agent;
|
||||
modelsConfig: TModelsConfig;
|
||||
/**
|
||||
* Loads the model catalog entry for an endpoint. Only the agent's own provider is requested,
|
||||
* so a request never loads the model lists of endpoints none of its agents use.
|
||||
*/
|
||||
getEndpointModels: (endpoint: string) => Promise<TModelsConfig>;
|
||||
logViolation: (
|
||||
req: LooseRequest,
|
||||
res: Response,
|
||||
|
|
@ -567,7 +571,7 @@ interface ValidateAgentModelResult {
|
|||
export async function validateAgentModel(
|
||||
params: ValidateAgentModelParams,
|
||||
): Promise<ValidateAgentModelResult> {
|
||||
const { req, res, agent, modelsConfig, logViolation } = params;
|
||||
const { req, res, agent, getEndpointModels, logViolation } = params;
|
||||
const { model, provider: endpoint } = agent;
|
||||
|
||||
if (!model) {
|
||||
|
|
@ -579,6 +583,7 @@ export async function validateAgentModel(
|
|||
};
|
||||
}
|
||||
|
||||
const modelsConfig = await getEndpointModels(endpoint);
|
||||
if (!modelsConfig) {
|
||||
return {
|
||||
isValid: false,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { FileSources } from 'librechat-data-provider';
|
||||
import { FileSources, resolveModelCatalogKey } from 'librechat-data-provider';
|
||||
import type { TModelsConfig } from 'librechat-data-provider';
|
||||
import type { AppConfig } from '@librechat/data-schemas';
|
||||
import type { Response } from 'express';
|
||||
|
|
@ -238,3 +238,197 @@ describe('model discovery service', () => {
|
|||
expect(deps.loadConfigModels).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('per-endpoint model loading', () => {
|
||||
const hasOwn = (catalog: TModelsConfig, key: string) =>
|
||||
Object.prototype.hasOwnProperty.call(catalog, key);
|
||||
const builtIns: TModelsConfig = {
|
||||
openAI: ['gpt-default'],
|
||||
anthropic: ['claude-default'],
|
||||
google: ['gemini-default'],
|
||||
azureOpenAI: ['azure-default'],
|
||||
assistants: ['assistant-default'],
|
||||
azureAssistants: ['azure-assistant-default'],
|
||||
bedrock: ['bedrock-default'],
|
||||
};
|
||||
const served: TModelsConfig = {
|
||||
anthropic: ['claude-configured'],
|
||||
ollama: ['llama3.2'],
|
||||
'My Proxy': ['proxy-model'],
|
||||
};
|
||||
|
||||
/**
|
||||
* Stand-in loaders that return the full catalog without a name and one entry with it. They test
|
||||
* the service's own rules (lookup order, one load per request, no other endpoint loaded); that
|
||||
* the entry equals the full catalog's under the real loaders is checked in
|
||||
* `api/server/services/Config/catalog.spec.js`.
|
||||
*/
|
||||
function world(extra: TModelsConfig = {}) {
|
||||
const configured = { ...served, ...extra };
|
||||
const deps = {
|
||||
getAppConfig: jest.fn().mockResolvedValue({ endpoints: {} } as AppConfig),
|
||||
loadDefaultModels: jest.fn(async (): Promise<TModelsConfig> => ({ ...builtIns })),
|
||||
loadConfigModels: jest.fn(
|
||||
async (_req: ServerRequest, endpoint?: string): Promise<TModelsConfig> => {
|
||||
if (endpoint == null) {
|
||||
return { ...configured };
|
||||
}
|
||||
return hasOwn(configured, endpoint) ? { [endpoint]: configured[endpoint] } : {};
|
||||
},
|
||||
),
|
||||
defaultEndpointsConfig: { openAI: { userProvide: false } },
|
||||
loadAsyncEndpoints: jest.fn().mockResolvedValue({}),
|
||||
loadDefaultEndpointModels: jest.fn(
|
||||
async (
|
||||
_req: ServerRequest,
|
||||
_config: AppConfig,
|
||||
endpoint: string,
|
||||
): Promise<TModelsConfig> =>
|
||||
hasOwn(builtIns, endpoint) ? { [endpoint]: builtIns[endpoint] } : {},
|
||||
),
|
||||
};
|
||||
return { deps, ...createModelsService(deps) };
|
||||
}
|
||||
|
||||
/** The lookup agent validation performs on a catalog. */
|
||||
const lookup = (catalog: TModelsConfig, provider: string) =>
|
||||
catalog[resolveModelCatalogKey(provider, catalog)];
|
||||
|
||||
it.each([
|
||||
['a built-in without a configured list', 'openAI', ['gpt-default']],
|
||||
['a built-in with a configured list', 'anthropic', ['claude-configured']],
|
||||
['a served custom endpoint', 'My Proxy', ['proxy-model']],
|
||||
['a custom endpoint saved under another spelling', 'Ollama', ['llama3.2']],
|
||||
['a provider served under an alias', 'vertexai', ['gemini-default']],
|
||||
['an endpoint with no model list', 'Unknown', undefined],
|
||||
])('reads %s', async (_label, provider, models) => {
|
||||
const { getEndpointModelsConfig } = world();
|
||||
|
||||
const entry = await getEndpointModelsConfig(request(), provider);
|
||||
|
||||
expect(lookup(entry, provider)).toEqual(models);
|
||||
expect(Object.keys(entry).length).toBeLessThanOrEqual(1);
|
||||
});
|
||||
|
||||
it('prefers an exact catalog over the alias', async () => {
|
||||
const { getEndpointModelsConfig } = world({ vertexai: ['custom-vertex'] });
|
||||
|
||||
const entry = await getEndpointModelsConfig(request(), 'vertexai');
|
||||
|
||||
expect(lookup(entry, 'vertexai')).toEqual(['custom-vertex']);
|
||||
});
|
||||
|
||||
it('loads the requested endpoint and no other', async () => {
|
||||
const { deps, getEndpointModelsConfig } = world();
|
||||
const req = request();
|
||||
|
||||
await getEndpointModelsConfig(req, 'openAI');
|
||||
await getEndpointModelsConfig(req, 'My Proxy');
|
||||
|
||||
expect(deps.loadDefaultModels).not.toHaveBeenCalled();
|
||||
expect(deps.loadConfigModels.mock.calls.map(([, endpoint]) => endpoint)).toEqual([
|
||||
'openAI',
|
||||
'My Proxy',
|
||||
]);
|
||||
expect(deps.loadDefaultEndpointModels.mock.calls.map(([, , endpoint]) => endpoint)).toEqual([
|
||||
'openAI',
|
||||
]);
|
||||
});
|
||||
|
||||
it('loads an endpoint once per request, including concurrent and repeated calls', async () => {
|
||||
const { deps, getEndpointModelsConfig } = world();
|
||||
const gate = deferred<TModelsConfig>();
|
||||
deps.loadConfigModels.mockReturnValueOnce(gate.promise);
|
||||
const req = request();
|
||||
|
||||
const first = getEndpointModelsConfig(req, 'My Proxy');
|
||||
const second = getEndpointModelsConfig(req, 'My Proxy');
|
||||
gate.resolve({ 'My Proxy': ['proxy-model'] });
|
||||
|
||||
await expect(Promise.all([first, second])).resolves.toEqual([
|
||||
{ 'My Proxy': ['proxy-model'] },
|
||||
{ 'My Proxy': ['proxy-model'] },
|
||||
]);
|
||||
await getEndpointModelsConfig(req, 'My Proxy');
|
||||
expect(deps.loadConfigModels).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('shares the load between spellings that read the same endpoint', async () => {
|
||||
const { deps, getEndpointModelsConfig } = world();
|
||||
const req = request();
|
||||
|
||||
await getEndpointModelsConfig(req, 'ollama');
|
||||
await getEndpointModelsConfig(req, 'Ollama');
|
||||
|
||||
expect(deps.loadConfigModels.mock.calls.filter(([, name]) => name === 'ollama')).toHaveLength(
|
||||
1,
|
||||
);
|
||||
});
|
||||
|
||||
it('does not cache across requests', async () => {
|
||||
const { deps, getEndpointModelsConfig } = world();
|
||||
|
||||
await getEndpointModelsConfig(request(), 'My Proxy');
|
||||
await getEndpointModelsConfig(request(), 'My Proxy');
|
||||
|
||||
expect(deps.loadConfigModels).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('does not remember a failed load', async () => {
|
||||
const { deps, getEndpointModelsConfig } = world();
|
||||
deps.loadConfigModels.mockRejectedValueOnce(new Error('provider down'));
|
||||
const req = request();
|
||||
|
||||
await expect(getEndpointModelsConfig(req, 'My Proxy')).rejects.toThrow('provider down');
|
||||
await expect(getEndpointModelsConfig(req, 'My Proxy')).resolves.toEqual({
|
||||
'My Proxy': ['proxy-model'],
|
||||
});
|
||||
});
|
||||
|
||||
it('loads built-ins that ENDPOINTS does not enable', async () => {
|
||||
const original = process.env.ENDPOINTS;
|
||||
process.env.ENDPOINTS = 'agents';
|
||||
try {
|
||||
const { getEndpointModelsConfig } = world();
|
||||
|
||||
await expect(getEndpointModelsConfig(request(), 'bedrock')).resolves.toEqual({
|
||||
bedrock: ['bedrock-default'],
|
||||
});
|
||||
} finally {
|
||||
if (original == null) {
|
||||
delete process.env.ENDPOINTS;
|
||||
} else {
|
||||
process.env.ENDPOINTS = original;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('resolves the principal config once for the request', async () => {
|
||||
const { deps, getEndpointModelsConfig } = world();
|
||||
const req = request();
|
||||
delete req.config;
|
||||
|
||||
await getEndpointModelsConfig(req, 'openAI');
|
||||
await getEndpointModelsConfig(req, 'My Proxy');
|
||||
|
||||
expect(deps.getAppConfig).toHaveBeenCalledTimes(1);
|
||||
expect(deps.loadConfigModels).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ config: { endpoints: {} } }),
|
||||
'openAI',
|
||||
);
|
||||
});
|
||||
|
||||
it.each([undefined, null, '', 42, ['openAI'], { name: 'openAI' }])(
|
||||
'loads nothing for the unusable endpoint %p',
|
||||
async (endpoint) => {
|
||||
const { deps, getEndpointModelsConfig } = world();
|
||||
|
||||
await expect(
|
||||
getEndpointModelsConfig(request(), endpoint as string | null | undefined),
|
||||
).resolves.toEqual({});
|
||||
expect(deps.getAppConfig).not.toHaveBeenCalled();
|
||||
expect(deps.loadConfigModels).not.toHaveBeenCalled();
|
||||
expect(deps.loadDefaultEndpointModels).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
import { logger } from '@librechat/data-schemas';
|
||||
import { EModelEndpoint, getEnabledEndpoints } from 'librechat-data-provider';
|
||||
import {
|
||||
EModelEndpoint,
|
||||
getEnabledEndpoints,
|
||||
resolveModelCatalogKey,
|
||||
} from 'librechat-data-provider';
|
||||
import type { TModelsConfig } from 'librechat-data-provider';
|
||||
import type { AppConfig } from '@librechat/data-schemas';
|
||||
import type { Response } from 'express';
|
||||
|
|
@ -27,12 +31,23 @@ export interface ModelsServiceDeps {
|
|||
loadDefaultEndpointModels?: typeof loadDefaultEndpointModels;
|
||||
}
|
||||
|
||||
function hasEntry(catalog: TModelsConfig, key: string): boolean {
|
||||
return Object.prototype.hasOwnProperty.call(catalog, key) && catalog[key] != null;
|
||||
}
|
||||
|
||||
export function createModelsService(deps: ModelsServiceDeps): {
|
||||
modelController: (req: ServerRequest, res: Response) => Promise<Response>;
|
||||
loadModels: (req: ServerRequest) => Promise<TModelsConfig>;
|
||||
getModelsConfig: (req: ServerRequest) => Promise<TModelsConfig>;
|
||||
getRequestModelsConfig: (req: ServerRequest) => Promise<RequestModelsConfigResult>;
|
||||
getEndpointModelsConfig: (
|
||||
req: ServerRequest,
|
||||
endpoint: string | null | undefined,
|
||||
) => Promise<TModelsConfig>;
|
||||
} {
|
||||
/** Catalog entries loaded for a request, by the endpoint name they were requested under. */
|
||||
const requestEntries = new WeakMap<ServerRequest, Map<string, Promise<TModelsConfig>>>();
|
||||
|
||||
/** Internal callers keep the full catalog, regardless of HTTP query parameters. */
|
||||
async function loadModels(req: ServerRequest): Promise<TModelsConfig> {
|
||||
const [defaults, configured] = await Promise.all([
|
||||
|
|
@ -42,6 +57,53 @@ export function createModelsService(deps: ModelsServiceDeps): {
|
|||
return { ...defaults, ...configured };
|
||||
}
|
||||
|
||||
async function resolveConfig(req: ServerRequest): Promise<AppConfig> {
|
||||
const config = req.config ?? (await deps.getAppConfig(getAppConfigOptionsFromUser(req.user)));
|
||||
req.config = config;
|
||||
return config;
|
||||
}
|
||||
|
||||
/**
|
||||
* What the full catalog holds under one endpoint name: `{ [endpoint]: models }`, or `{}` when
|
||||
* the name is neither a configured list, a served custom endpoint nor a built-in provider.
|
||||
*/
|
||||
async function loadEndpointModels(
|
||||
req: ServerRequest,
|
||||
config: AppConfig,
|
||||
endpoint: string,
|
||||
): Promise<TModelsConfig> {
|
||||
const configured = await deps.loadConfigModels(req, endpoint);
|
||||
if (Object.prototype.hasOwnProperty.call(configured, endpoint)) {
|
||||
return { [endpoint]: configured[endpoint] };
|
||||
}
|
||||
return (deps.loadDefaultEndpointModels ?? loadDefaultEndpointModels)(req, config, endpoint);
|
||||
}
|
||||
|
||||
/** One load per request and endpoint name, shared while in flight and kept once settled. */
|
||||
function loadRequestEntry(req: ServerRequest, endpoint: string): Promise<TModelsConfig> {
|
||||
let entries = requestEntries.get(req);
|
||||
if (entries == null) {
|
||||
entries = new Map();
|
||||
requestEntries.set(req, entries);
|
||||
}
|
||||
const loaded = entries.get(endpoint);
|
||||
if (loaded != null) {
|
||||
return loaded;
|
||||
}
|
||||
const known = entries;
|
||||
const pending: Promise<TModelsConfig> = resolveConfig(req)
|
||||
.then((config) => loadEndpointModels(req, config, endpoint))
|
||||
.catch((error: unknown) => {
|
||||
/** A failed load is not remembered, so a later validation in the request may retry. */
|
||||
if (known.get(endpoint) === pending) {
|
||||
known.delete(endpoint);
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
known.set(endpoint, pending);
|
||||
return pending;
|
||||
}
|
||||
|
||||
async function getRequestModelsConfig(req: ServerRequest): Promise<RequestModelsConfigResult> {
|
||||
const endpoint = req.query.endpoint;
|
||||
if (endpoint != null && (typeof endpoint !== 'string' || endpoint.length === 0)) {
|
||||
|
|
@ -50,8 +112,7 @@ export function createModelsService(deps: ModelsServiceDeps): {
|
|||
if (endpoint == null) {
|
||||
return { ok: true, modelsConfig: await loadModels(req) };
|
||||
}
|
||||
const config = req.config ?? (await deps.getAppConfig(getAppConfigOptionsFromUser(req.user)));
|
||||
req.config = config;
|
||||
const config = await resolveConfig(req);
|
||||
const { getEndpointsConfig } = createEndpointsConfigService({
|
||||
getAppConfig: deps.getAppConfig,
|
||||
loadDefaultEndpointsConfig: async () => {
|
||||
|
|
@ -76,16 +137,31 @@ export function createModelsService(deps: ModelsServiceDeps): {
|
|||
if (!Object.prototype.hasOwnProperty.call(endpoints, endpoint)) {
|
||||
return { ok: false, status: 404, error: 'Endpoint not found' };
|
||||
}
|
||||
const configured = await deps.loadConfigModels(req, endpoint);
|
||||
if (Object.prototype.hasOwnProperty.call(configured, endpoint)) {
|
||||
return { ok: true, modelsConfig: { [endpoint]: configured[endpoint] } };
|
||||
const loaded = await loadEndpointModels(req, config, endpoint);
|
||||
return { ok: true, modelsConfig: { [endpoint]: loaded[endpoint] ?? [] } };
|
||||
}
|
||||
|
||||
/**
|
||||
* The part of the full catalog that `resolveModelCatalogKey(endpoint, catalog)` reads for an
|
||||
* agent provider or request endpoint: the entry under the exact name when the catalog has one,
|
||||
* otherwise the entry under the alias or served name it falls back to (`vertexai` reads
|
||||
* `google`, `Ollama` reads `ollama`). No other endpoint is loaded, and nothing is gated on the
|
||||
* endpoint being enabled, as in the full catalog. Built-in defaults come from the same source
|
||||
* as `GET /api/models?endpoint=`. Entries are loaded once per request and endpoint name.
|
||||
*/
|
||||
async function getEndpointModelsConfig(
|
||||
req: ServerRequest,
|
||||
endpoint: string | null | undefined,
|
||||
): Promise<TModelsConfig> {
|
||||
if (typeof endpoint !== 'string' || endpoint.length === 0) {
|
||||
return {};
|
||||
}
|
||||
const models = await (deps.loadDefaultEndpointModels ?? loadDefaultEndpointModels)(
|
||||
req,
|
||||
config,
|
||||
endpoint,
|
||||
);
|
||||
return { ok: true, modelsConfig: { [endpoint]: models[endpoint] ?? [] } };
|
||||
const exact = await loadRequestEntry(req, endpoint);
|
||||
if (hasEntry(exact, endpoint)) {
|
||||
return exact;
|
||||
}
|
||||
const key = resolveModelCatalogKey(endpoint, {});
|
||||
return key === endpoint ? exact : loadRequestEntry(req, key);
|
||||
}
|
||||
|
||||
async function modelController(req: ServerRequest, res: Response): Promise<Response> {
|
||||
|
|
@ -101,5 +177,11 @@ export function createModelsService(deps: ModelsServiceDeps): {
|
|||
}
|
||||
}
|
||||
|
||||
return { modelController, loadModels, getModelsConfig: loadModels, getRequestModelsConfig };
|
||||
return {
|
||||
modelController,
|
||||
loadModels,
|
||||
getModelsConfig: loadModels,
|
||||
getRequestModelsConfig,
|
||||
getEndpointModelsConfig,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import axios from 'axios';
|
||||
import Anthropic from '@anthropic-ai/sdk';
|
||||
import { Time, EModelEndpoint, defaultModels, AuthType } from 'librechat-data-provider';
|
||||
import type { FetchModelsParams } from './models';
|
||||
import type { ServerRequest } from '~/types';
|
||||
import {
|
||||
fetchModels,
|
||||
|
|
@ -1334,3 +1335,169 @@ describe('configured model lists skip provider discovery', () => {
|
|||
expect(mockedAxios.get).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('fetchModels in-flight sharing', () => {
|
||||
type ModelsResponse = { data: { data: Array<Record<string, unknown>> } };
|
||||
|
||||
function deferred<T>() {
|
||||
let resolve!: (value: T) => void;
|
||||
let reject!: (reason: unknown) => void;
|
||||
const promise = new Promise<T>((done, fail) => {
|
||||
resolve = done;
|
||||
reject = fail;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
/** Lets every caller pass its cache lookup and reach the provider request. */
|
||||
const flush = () => new Promise((resolve) => setImmediate(resolve));
|
||||
const listing = (...ids: string[]): ModelsResponse => ({
|
||||
data: { data: ids.map((id) => ({ id })) },
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
mockedAxios.get.mockReset();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
mockedAxios.get.mockReset().mockResolvedValue(listing('model-1', 'model-2'));
|
||||
});
|
||||
|
||||
it('sends one provider request for concurrent callers that miss the shared cache', async () => {
|
||||
const gate = deferred<ModelsResponse>();
|
||||
mockedAxios.get.mockReturnValue(gate.promise);
|
||||
const params = { apiKey: 'key', baseURL: 'https://shared.test/v1', name: 'Shared' };
|
||||
|
||||
const calls = Array.from({ length: 5 }, () => fetchModels(params));
|
||||
await flush();
|
||||
gate.resolve(listing('model-a', 'model-b'));
|
||||
const results = await Promise.all(calls);
|
||||
|
||||
expect(mockedAxios.get).toHaveBeenCalledTimes(1);
|
||||
expect(results).toEqual(Array.from({ length: 5 }, () => ['model-a', 'model-b']));
|
||||
});
|
||||
|
||||
it('keeps each caller storing the token config under its own key', async () => {
|
||||
const gate = deferred<ModelsResponse>();
|
||||
mockedAxios.get.mockReturnValue(gate.promise);
|
||||
const params = { apiKey: 'key', baseURL: 'https://pricing.test/v1', name: 'Pricing' };
|
||||
const priced = {
|
||||
id: 'model-a',
|
||||
pricing: { prompt: '0.002', completion: '0.001' },
|
||||
context_length: 1024,
|
||||
};
|
||||
|
||||
const calls = ['token-a', 'token-b'].map((tokenKey) => fetchModels({ ...params, tokenKey }));
|
||||
await flush();
|
||||
gate.resolve({ data: { data: [priced] } });
|
||||
await expect(Promise.all(calls)).resolves.toEqual([['model-a'], ['model-a']]);
|
||||
|
||||
expect(mockedAxios.get).toHaveBeenCalledTimes(1);
|
||||
for (const tokenKey of ['token-a', 'token-b']) {
|
||||
expect(mockTokenConfigSet).toHaveBeenCalledWith(
|
||||
tokenKey,
|
||||
expect.objectContaining({ 'model-a': expect.objectContaining({ context: 1024 }) }),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it.each<[string, Partial<FetchModelsParams>]>([
|
||||
['another base URL', { baseURL: 'https://other.test/v1' }],
|
||||
['another API key', { apiKey: 'other-key' }],
|
||||
['another native provider', { provider: EModelEndpoint.anthropic }],
|
||||
])('does not share requests that differ by %s', async (_label, other) => {
|
||||
const gate = deferred<ModelsResponse>();
|
||||
mockedAxios.get.mockReturnValue(gate.promise);
|
||||
const params = { apiKey: 'key', baseURL: 'https://keyed.test/v1', name: 'Keyed' };
|
||||
|
||||
const calls = [fetchModels(params), fetchModels({ ...params, ...other })];
|
||||
await flush();
|
||||
gate.resolve(listing('model-a'));
|
||||
await Promise.all(calls);
|
||||
|
||||
expect(mockedAxios.get).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[
|
||||
'headers resolved for a user',
|
||||
{ headers: { Authorization: 'Bearer user-token' }, userObject: { id: 'user-1' } },
|
||||
],
|
||||
['a user query parameter', { userIdQuery: true, user: 'user-1' }],
|
||||
['a skipped cache', { skipCache: true }],
|
||||
])('never shares a request made with %s', async (_label, scoped) => {
|
||||
const gate = deferred<ModelsResponse>();
|
||||
mockedAxios.get.mockReturnValue(gate.promise);
|
||||
const params = { apiKey: 'key', baseURL: 'https://scoped.test/v1', name: 'Scoped', ...scoped };
|
||||
|
||||
const calls = [fetchModels(params), fetchModels(params)];
|
||||
await flush();
|
||||
gate.resolve(listing('model-a'));
|
||||
await Promise.all(calls);
|
||||
|
||||
expect(mockedAxios.get).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('does not remember a failed request for later callers', async () => {
|
||||
const gate = deferred<ModelsResponse>();
|
||||
mockedAxios.get.mockReturnValueOnce(gate.promise);
|
||||
const params = { apiKey: 'key', baseURL: 'https://failing.test/v1', name: 'Failing' };
|
||||
|
||||
const calls = Array.from({ length: 3 }, () => fetchModels(params));
|
||||
await flush();
|
||||
gate.reject(new Error('provider unavailable'));
|
||||
await expect(Promise.all(calls)).resolves.toEqual([[], [], []]);
|
||||
expect(mockedAxios.get).toHaveBeenCalledTimes(1);
|
||||
|
||||
mockedAxios.get.mockResolvedValueOnce(listing('recovered'));
|
||||
await expect(fetchModels(params)).resolves.toEqual(['recovered']);
|
||||
expect(mockedAxios.get).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('does not keep a finished request for later callers', async () => {
|
||||
mockedAxios.get.mockResolvedValue(listing('model-a'));
|
||||
const params = { apiKey: 'key', baseURL: 'https://settled.test/v1', name: 'Settled' };
|
||||
|
||||
await fetchModels(params);
|
||||
await fetchModels(params);
|
||||
|
||||
expect(mockedAxios.get).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('shares one Ollama request, and one fallback request when it fails', async () => {
|
||||
const tags = deferred<{ data: { models: Array<{ name: string }> } }>();
|
||||
const models = deferred<ModelsResponse>();
|
||||
mockedAxios.get.mockImplementation((url: string) =>
|
||||
url.endsWith('/api/tags') ? tags.promise : models.promise,
|
||||
);
|
||||
const params = { apiKey: 'key', baseURL: 'https://ollama.test/v1', name: 'ollama' };
|
||||
|
||||
const calls = [fetchModels(params), fetchModels(params), fetchModels(params)];
|
||||
await flush();
|
||||
expect(mockedAxios.get).toHaveBeenCalledTimes(1);
|
||||
tags.reject(new Error('no tags endpoint'));
|
||||
await flush();
|
||||
expect(mockedAxios.get).toHaveBeenCalledTimes(2);
|
||||
models.resolve(listing('llama3'));
|
||||
|
||||
await expect(Promise.all(calls)).resolves.toEqual([['llama3'], ['llama3'], ['llama3']]);
|
||||
expect(mockedAxios.get).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('shares a successful Ollama request between concurrent callers', async () => {
|
||||
const tags = deferred<{ data: { models: Array<{ name: string }> } }>();
|
||||
mockedAxios.get.mockReturnValue(tags.promise);
|
||||
const params = { apiKey: 'key', baseURL: 'https://ollama-ok.test/v1', name: 'ollama-local' };
|
||||
|
||||
const calls = [fetchModels(params), fetchModels(params)];
|
||||
await flush();
|
||||
tags.resolve({ data: { models: [{ name: 'llama3' }] } });
|
||||
|
||||
await expect(Promise.all(calls)).resolves.toEqual([['llama3'], ['llama3']]);
|
||||
expect(mockedAxios.get).toHaveBeenCalledTimes(1);
|
||||
expect(mockedAxios.get).toHaveBeenCalledWith(
|
||||
'https://ollama-ok.test/api/tags',
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -8,9 +8,9 @@ import {
|
|||
EModelEndpoint,
|
||||
defaultModels,
|
||||
} from 'librechat-data-provider';
|
||||
import type { AxiosRequestConfig, AxiosResponse } from 'axios';
|
||||
import type { TEndpoint } from 'librechat-data-provider';
|
||||
import type { IUser } from '@librechat/data-schemas';
|
||||
import type { AxiosRequestConfig } from 'axios';
|
||||
import {
|
||||
processModelData,
|
||||
extractBaseURL,
|
||||
|
|
@ -27,6 +27,36 @@ import { standardCache, tokenConfigCache } from '~/cache';
|
|||
|
||||
type SSRFSafeAgents = ReturnType<typeof createSSRFSafeAgents>;
|
||||
|
||||
/** Provider requests in flight by cache key, so concurrent cache misses for one key share one. */
|
||||
const ollamaRequests = new Map<string, Promise<string[]>>();
|
||||
const modelsRequests = new Map<string, Promise<AxiosResponse>>();
|
||||
|
||||
/**
|
||||
* Runs `start` unless a request is already in flight under `key`, which is then joined. A request
|
||||
* leaves the map as soon as it settles, so neither a result nor a failure outlives it. An empty
|
||||
* key is never shared.
|
||||
*/
|
||||
function shareInFlight<T>(
|
||||
requests: Map<string, Promise<T>>,
|
||||
key: string,
|
||||
start: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
if (!key) {
|
||||
return start();
|
||||
}
|
||||
const active = requests.get(key);
|
||||
if (active != null) {
|
||||
return active;
|
||||
}
|
||||
const request: Promise<T> = start().finally(() => {
|
||||
if (requests.get(key) === request) {
|
||||
requests.delete(key);
|
||||
}
|
||||
});
|
||||
requests.set(key, request);
|
||||
return request;
|
||||
}
|
||||
|
||||
export interface FetchModelsParams {
|
||||
/** User ID for API requests */
|
||||
user?: string;
|
||||
|
|
@ -226,11 +256,13 @@ export async function fetchModels({
|
|||
if (!isCustomAnthropic && name && name.toLowerCase().startsWith(KnownEndpoints.ollama)) {
|
||||
let ollamaModels: string[] | null = null;
|
||||
try {
|
||||
ollamaModels = await fetchOllamaModels(baseURL ?? '', {
|
||||
headers,
|
||||
ssrfAgents,
|
||||
user: userObject,
|
||||
});
|
||||
ollamaModels = await shareInFlight(ollamaRequests, cacheKey, () =>
|
||||
fetchOllamaModels(baseURL ?? '', {
|
||||
headers,
|
||||
ssrfAgents,
|
||||
user: userObject,
|
||||
}),
|
||||
);
|
||||
} catch (ollamaError) {
|
||||
logAxiosError({
|
||||
message:
|
||||
|
|
@ -297,7 +329,10 @@ export async function fetchModels({
|
|||
}
|
||||
applyAxiosProxyConfig(options, url);
|
||||
applyUserProvidedBaseURLProtection(options, ssrfAgents);
|
||||
const res = await axios.get(url.toString(), options);
|
||||
/** Only the request is shared: each caller still derives and stores its own token config. */
|
||||
const res = await shareInFlight(modelsRequests, cacheKey, async () =>
|
||||
axios.get(url.toString(), options),
|
||||
);
|
||||
|
||||
const input = res.data;
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue