🗂️ fix: Scope Token Config Cache (#13770)

* fix token config tenant cache scope

* fix token config scoped cache backfill

* chore sort token config imports

api/server/controllers/TokenConfigController.js

@@ -14,7 +14,12 @@ async function tokenConfigController(req, res) {
   try {
     const modelsConfig = await getModelsConfig(req);
     const tokenConfigMap = await resolveTokenConfigMap(
-      { appConfig: req.config, modelsConfig, userId: req.user.id },
+      {
+        appConfig: req.config,
+        modelsConfig,
+        userId: req.user.id,
+        tenantId: req.user.tenantId,
+      },
       { getValueKey, getMultiplier, getCacheMultiplier },
     );
     res.json(tokenConfigMap);

packages/api/src/endpoints/config/models.spec.ts

@@ -1,5 +1,6 @@
 import { AuthType, EModelEndpoint } from 'librechat-data-provider';
 import type { ServerRequest } from '~/types';
+import { SCOPED_TOKEN_CONFIG_KEY_PREFIX } from '../keys';
 import { createLoadConfigModels } from './models';
 
 jest.mock('~/utils', () => {
@@ -106,6 +107,34 @@ describe('createLoadConfigModels – user-provided baseURL header guard', () =>
       }),
     );
   });
+
+  it('tenant-scopes the fetched token config cache key for system-defined endpoints', async () => {
+    const loadConfigModels = createLoadConfigModels({
+      getAppConfig: jest.fn().mockResolvedValue(
+        buildAppConfig({
+          baseURL: 'https://admin-trusted.example.com/v1',
+          apiKey: 'sk-system-key',
+        }),
+      ),
+      getUserKeyValues: jest.fn(),
+      fetchModels,
+    });
+
+    const req = {
+      user: { id: 'user-1', tenantId: 'tenant-a' },
+      config: undefined,
+    } as unknown as ServerRequest;
+
+    await loadConfigModels(req);
+
+    expect(fetchModels).toHaveBeenCalledTimes(1);
+    const tokenKey = fetchModels.mock.calls[0][0].tokenKey;
+    expect(fetchModels.mock.calls[0][0].name).toBe('TestProxy');
+    expect(tokenKey.startsWith(SCOPED_TOKEN_CONFIG_KEY_PREFIX)).toBe(true);
+    expect(tokenKey).not.toBe('tenant:tenant-a:TestProxy');
+    expect(tokenKey).not.toContain('tenant-a');
+    expect(tokenKey).not.toContain('TestProxy');
+  });
 });
 
 describe('createLoadConfigModels – in-request fetch coalescing', () => {

packages/api/src/endpoints/config/models.ts

@@ -102,6 +102,7 @@ export function createLoadConfigModels(deps: LoadConfigModelsDeps) {
      *  sharing the fetch can be backfilled with the same config afterward */
     const uniqueKeyToTokenKey: Record<string, string> = {};
     const endpointsMap: Record<string, TEndpoint> = {};
+    const tenantId = req.user?.tenantId;
 
     const resolved: ResolvedEndpoint[] = [];
     const userKeyEndpoints: ResolvedEndpoint[] = [];
@@ -182,7 +183,7 @@ export function createLoadConfigModels(deps: LoadConfigModelsDeps) {
         if (!fetchPromisesMap[uniqueKey]) {
           /** User-scoped when configured headers resolve per user — the
            *  derived token config must not be cached under the shared name */
-          const tokenKey = getTokenConfigKey(endpoint, name, req.user?.id ?? '');
+          const tokenKey = getTokenConfigKey(endpoint, name, req.user?.id ?? '', tenantId);
           uniqueKeyToTokenKey[uniqueKey] = tokenKey;
           fetchPromisesMap[uniqueKey] = fetchModels({
             name,
@@ -227,7 +228,7 @@ export function createLoadConfigModels(deps: LoadConfigModelsDeps) {
               userIdQuery: models.userIdQuery,
               skipCache: true,
               /** Fetched with the user's key/URL — always user-scoped */
-              tokenKey: getTokenConfigKey(endpoint, name, req.user?.id ?? ''),
+              tokenKey: getTokenConfigKey(endpoint, name, req.user?.id ?? '', tenantId),
             });
           uniqueKeyToEndpointsMap[userFetchKey] = uniqueKeyToEndpointsMap[userFetchKey] || [];
           uniqueKeyToEndpointsMap[userFetchKey].push(name);
@@ -271,7 +272,12 @@ export function createLoadConfigModels(deps: LoadConfigModelsDeps) {
         const config = await cache.get(winnerTokenKey);
         if (config != null) {
           for (const name of associatedNames) {
-            const siblingKey = getTokenConfigKey(endpointsMap[name], name, req.user?.id ?? '');
+            const siblingKey = getTokenConfigKey(
+              endpointsMap[name],
+              name,
+              req.user?.id ?? '',
+              tenantId,
+            );
             if (siblingKey !== winnerTokenKey) {
               await cache.set(siblingKey, config);
             }

packages/api/src/endpoints/custom/initialize.spec.ts

@@ -33,7 +33,8 @@ jest.mock('~/app/config', () => ({
   getCustomEndpointConfig: (...args: unknown[]) => mockGetCustomEndpointConfig(...args),
 }));
 
-import { initializeCustom } from './initialize';
+import { getTokenConfigKey, initializeCustom } from './initialize';
+import { SCOPED_TOKEN_CONFIG_KEY_PREFIX } from '../keys';
 
 function createParams(overrides: {
   apiKey?: string;
@@ -219,6 +220,7 @@ describe('initializeCustom – token-config fetch header forwarding', () => {
     baseURL?: string;
     userBaseURL?: string;
     headers?: Record<string, string>;
+    tenantId?: string;
   }): BaseInitializeParams {
     const { apiKey = 'sk-test-key', baseURL = 'https://openrouter.ai/api/v1' } = overrides;
 
@@ -238,7 +240,7 @@ describe('initializeCustom – token-config fetch header forwarding', () => {
 
     return {
       req: {
-        user: { id: 'user-1', email: 'user@example.com' },
+        user: { id: 'user-1', email: 'user@example.com', tenantId: overrides.tenantId },
         body: { key: '2099-01-01' },
         config: {},
       } as unknown as BaseInitializeParams['req'],
@@ -315,6 +317,21 @@ describe('initializeCustom – token-config fetch header forwarding', () => {
     );
   });
 
+  it('tenant-scopes the tokenKey when a tenant id is present', async () => {
+    const params = createTokenConfigParams({
+      apiKey: 'sk-test-key',
+      baseURL: 'https://openrouter.ai/api/v1',
+      tenantId: 'tenant-a',
+    });
+
+    await initializeCustom(params);
+
+    const tokenKey = fetchModels.mock.calls[0][0].tokenKey;
+    expect(fetchModels.mock.calls[0][0].name).toBe('openrouter');
+    expect(tokenKey.startsWith(SCOPED_TOKEN_CONFIG_KEY_PREFIX)).toBe(true);
+    expect(tokenKey).not.toBe('tenant:tenant-a:openrouter');
+  });
+
   it('user-scopes the tokenKey when headers will be forwarded (admin-trusted base URL)', async () => {
     const params = createTokenConfigParams({
       apiKey: 'sk-test-key',
@@ -395,6 +412,47 @@ describe('initializeCustom – token-config fetch header forwarding', () => {
   });
 });
 
+describe('getTokenConfigKey – tenant fallback', () => {
+  const endpointConfig = {
+    apiKey: 'sk-test-key',
+    baseURL: 'https://openrouter.ai/api/v1',
+  };
+
+  it('keeps legacy shared keys when tenant context is unavailable or empty', () => {
+    const tenantIds = [undefined, null, '', '   '] as Array<string | null | undefined>;
+
+    for (const tenantId of tenantIds) {
+      expect(getTokenConfigKey(endpointConfig, 'openrouter', 'user-1', tenantId)).toBe(
+        'openrouter',
+      );
+    }
+  });
+
+  it('keeps legacy user-scoped keys when tenant context is unavailable or empty', () => {
+    const userScopedConfig = {
+      ...endpointConfig,
+      headers: { Authorization: 'Bearer {{LIBRECHAT_OPENID_ID_TOKEN}}' },
+    };
+    const tenantIds = [undefined, null, '', '   '] as Array<string | null | undefined>;
+
+    for (const tenantId of tenantIds) {
+      expect(getTokenConfigKey(userScopedConfig, 'openrouter', 'user-1', tenantId)).toBe(
+        'openrouter:user-1',
+      );
+    }
+  });
+
+  it('adds tenant scope only when tenant context is non-empty', () => {
+    const key = getTokenConfigKey(endpointConfig, 'openrouter', 'user-1', ' tenant-a ');
+
+    expect(key.startsWith(SCOPED_TOKEN_CONFIG_KEY_PREFIX)).toBe(true);
+    expect(key).not.toBe('tenant:tenant-a:openrouter');
+    expect(key).not.toContain('openrouter');
+    expect(key).not.toContain('tenant-a');
+    expect(key).toBe(getTokenConfigKey(endpointConfig, 'openrouter', 'user-1', 'tenant-a'));
+  });
+});
+
 describe('initializeCustom – native Anthropic provider', () => {
   beforeEach(() => {
     jest.clearAllMocks();

packages/api/src/endpoints/custom/initialize.ts

@@ -18,6 +18,7 @@ import { getLLMConfig as getAnthropicLLMConfig } from '~/endpoints/anthropic/llm
 import { extractDefaultParams } from '~/endpoints/openai/llm';
 import { isUserProvided, checkUserKeyExpiry } from '~/utils';
 import { getOpenAIConfig } from '~/endpoints/openai/config';
+import { getScopedTokenConfigKey } from '~/endpoints/keys';
 import { getCustomEndpointConfig } from '~/app/config';
 import { fetchModels } from '~/endpoints/models';
 import { validateEndpointURL } from '~/auth';
@@ -25,24 +26,42 @@ import { tokenConfigCache } from '~/cache';
 
 const { PROXY } = process.env;
 
+function getTenantTokenScope(tenantId?: string | null): string {
+  const normalizedTenantId = typeof tenantId === 'string' ? tenantId.trim() : '';
+  return normalizedTenantId;
+}
+
 /**
  * Cache key for an endpoint's fetched token config. User-scoped when the
  * model fetch can resolve per-user: user-provided key/URL, or header
  * templates forwarded against an admin-trusted base URL — making the
- * response, and therefore the derived token config, user-specific.
+ * response, and therefore the derived token config, user-specific. Otherwise
+ * tenant-scoped when a tenant id is available because custom endpoint config
+ * is resolved per tenant.
  */
 export function getTokenConfigKey(
   endpointConfig: Partial<TEndpoint>,
   endpoint: string,
   userId: string,
+  tenantId?: string | null,
 ): string {
   const hasTokenConfig = endpointConfig.tokenConfig != null;
+  if (hasTokenConfig) {
+    return endpoint;
+  }
+
   const userProvidesKey = isUserProvided(extractEnvVariable(endpointConfig.apiKey ?? ''));
   const userProvidesURL = isUserProvided(extractEnvVariable(endpointConfig.baseURL ?? ''));
   const willForwardUserScopedHeaders = !!endpointConfig?.headers && !userProvidesURL;
-  return !hasTokenConfig && (userProvidesKey || userProvidesURL || willForwardUserScopedHeaders)
-    ? `${endpoint}:${userId}`
-    : endpoint;
+  const tenantScope = getTenantTokenScope(tenantId);
+
+  if (userProvidesKey || userProvidesURL || willForwardUserScopedHeaders) {
+    return tenantScope
+      ? getScopedTokenConfigKey('tenant-user', [tenantScope, endpoint, userId])
+      : `${endpoint}:${userId}`;
+  }
+
+  return tenantScope ? getScopedTokenConfigKey('tenant', [tenantScope, endpoint]) : endpoint;
 }
 
 /**
@@ -224,10 +243,11 @@ export async function initializeCustom({
   let endpointTokenConfig: EndpointTokenConfig | undefined;
 
   const userId = req.user?.id ?? '';
+  const tenantId = req.user?.tenantId;
 
   const cache = tokenConfigCache();
   const hasTokenConfig = endpointConfig.tokenConfig != null;
-  const tokenKey = getTokenConfigKey(endpointConfig, endpoint, userId);
+  const tokenKey = getTokenConfigKey(endpointConfig, endpoint, userId, tenantId);
 
   if (hasTokenConfig) {
     /** A static override is authoritative — use it for the agent's billing

packages/api/src/endpoints/keys.ts

@@ -0,0 +1,19 @@
+import crypto from 'crypto';
+
+/** NUL-delimited scoped keys cannot collide with legacy endpoint-name keys. */
+export const SCOPED_TOKEN_CONFIG_KEY_PREFIX = '\u0000token-config:v2\u0000';
+
+type TokenConfigScope = 'tenant' | 'tenant-user';
+
+export function getScopedTokenConfigKey(scope: TokenConfigScope, parts: string[]): string {
+  const digest = crypto.createHash('sha256').update(JSON.stringify(parts)).digest('hex');
+  return `${SCOPED_TOKEN_CONFIG_KEY_PREFIX}${scope}\u0000${digest}`;
+}
+
+export function getModelCacheTokenConfigKey(cacheKey: string): string {
+  return `${SCOPED_TOKEN_CONFIG_KEY_PREFIX}models\u0000${cacheKey}`;
+}
+
+export function isScopedTokenConfigKey(tokenKey: string): boolean {
+  return tokenKey.startsWith(SCOPED_TOKEN_CONFIG_KEY_PREFIX);
+}

packages/api/src/endpoints/models.spec.ts

@@ -8,16 +8,23 @@ import {
   getBedrockModels,
   getAnthropicModels,
 } from './models';
+import { SCOPED_TOKEN_CONFIG_KEY_PREFIX } from './keys';
 
 jest.mock('axios');
 
 const mockCacheGet = jest.fn().mockResolvedValue(undefined);
 const mockCacheSet = jest.fn().mockResolvedValue(true);
+const mockTokenConfigGet = jest.fn().mockResolvedValue(undefined);
+const mockTokenConfigSet = jest.fn().mockResolvedValue(true);
 jest.mock('~/cache', () => ({
   standardCache: jest.fn().mockImplementation(() => ({
     get: mockCacheGet,
     set: mockCacheSet,
   })),
+  tokenConfigCache: jest.fn().mockImplementation(() => ({
+    get: mockTokenConfigGet,
+    set: mockTokenConfigSet,
+  })),
 }));
 
 jest.mock('~/utils', () => {
@@ -51,6 +58,8 @@ mockedAxios.get.mockResolvedValue({
 beforeEach(() => {
   mockCacheGet.mockReset().mockResolvedValue(undefined);
   mockCacheSet.mockReset().mockResolvedValue(true);
+  mockTokenConfigGet.mockReset().mockResolvedValue(undefined);
+  mockTokenConfigSet.mockReset().mockResolvedValue(true);
 });
 
 describe('fetchModels', () => {
@@ -263,6 +272,7 @@ describe('fetchModels with createTokenConfig true', () => {
   };
 
   beforeEach(() => {
+    mockedAxios.get.mockClear();
     mockedAxios.get.mockResolvedValue({ data });
   });
 
@@ -278,6 +288,59 @@ describe('fetchModels with createTokenConfig true', () => {
     expect(processModelData).toHaveBeenCalled();
     expect(processModelData).toHaveBeenCalledWith(data);
   });
+
+  it('backfills requested token config when model cache hits', async () => {
+    const endpointTokenConfig = {
+      'model-1': { prompt: 2000, completion: 1000, context: 1024 },
+    };
+    mockCacheGet.mockResolvedValue(['model-1']);
+    mockTokenConfigGet.mockResolvedValue(endpointTokenConfig);
+
+    const models = await fetchModels({
+      apiKey: 'testApiKey',
+      baseURL: 'https://api.test.com',
+      name: 'TestAPI',
+      tokenKey: `${SCOPED_TOKEN_CONFIG_KEY_PREFIX}tenant\u0000abc123`,
+    });
+
+    expect(models).toEqual(['model-1']);
+    expect(mockedAxios.get).not.toHaveBeenCalled();
+    expect(
+      mockTokenConfigGet.mock.calls[0][0].startsWith(`${SCOPED_TOKEN_CONFIG_KEY_PREFIX}models`),
+    ).toBe(true);
+    expect(mockTokenConfigSet).toHaveBeenCalledWith(
+      `${SCOPED_TOKEN_CONFIG_KEY_PREFIX}tenant\u0000abc123`,
+      endpointTokenConfig,
+    );
+  });
+
+  it('fetches and writes token config when scoped token key is missing on model cache hit', async () => {
+    mockCacheGet.mockResolvedValue(['cached-model']);
+    mockTokenConfigGet.mockResolvedValue(undefined);
+
+    const models = await fetchModels({
+      apiKey: 'testApiKey',
+      baseURL: 'https://api.test.com',
+      name: 'TestAPI',
+      tokenKey: `${SCOPED_TOKEN_CONFIG_KEY_PREFIX}tenant\u0000def456`,
+    });
+
+    expect(models).toEqual(['model-1', 'model-2']);
+    expect(mockedAxios.get).toHaveBeenCalledWith(
+      expect.stringContaining('https://api.test.com/models'),
+      expect.any(Object),
+    );
+    expect(mockTokenConfigSet).toHaveBeenCalledWith(
+      `${SCOPED_TOKEN_CONFIG_KEY_PREFIX}tenant\u0000def456`,
+      expect.objectContaining({
+        'model-1': expect.objectContaining({ context: 1024 }),
+      }),
+    );
+    const sourceSetCall = mockTokenConfigSet.mock.calls.find(([key]) =>
+      key.startsWith(`${SCOPED_TOKEN_CONFIG_KEY_PREFIX}models`),
+    );
+    expect(sourceSetCall).toBeDefined();
+  });
 });
 
 describe('getOpenAIModels', () => {

packages/api/src/endpoints/models.ts

@@ -20,6 +20,7 @@ import {
   inputSchema,
   applyAxiosProxyConfig,
 } from '~/utils';
+import { getModelCacheTokenConfigKey, isScopedTokenConfigKey } from '~/endpoints/keys';
 import { standardCache, tokenConfigCache } from '~/cache';
 
 export interface FetchModelsParams {
@@ -81,6 +82,19 @@ async function fetchOllamaModels(
   return response.data.models.map((tag) => tag.name);
 }
 
+async function backfillTokenConfigFromModelCache(
+  cacheKey: string,
+  tokenKey: string,
+): Promise<boolean> {
+  const cachedTokenConfig = await tokenConfigCache().get(getModelCacheTokenConfigKey(cacheKey));
+  if (cachedTokenConfig == null) {
+    return false;
+  }
+
+  await tokenConfigCache().set(tokenKey, cachedTokenConfig);
+  return true;
+}
+
 /**
  * Splits a string by commas and trims each resulting value.
  * @param input - The input string to split.
@@ -142,7 +156,16 @@ export async function fetchModels({
   if (modelsCache && cacheKey) {
     const cachedModels = await modelsCache.get(cacheKey);
     if (cachedModels) {
-      return cachedModels as string[];
+      if (createTokenConfig && tokenKey) {
+        const tokenConfigBackfilled = await backfillTokenConfigFromModelCache(cacheKey, tokenKey);
+        if (!tokenConfigBackfilled && isScopedTokenConfigKey(tokenKey)) {
+          models = cachedModels as string[];
+        } else {
+          return cachedModels as string[];
+        }
+      } else {
+        return cachedModels as string[];
+      }
     }
   }
 
@@ -221,7 +244,11 @@ export async function fetchModels({
     const validationResult = inputSchema.safeParse(input);
     if (validationResult.success && createTokenConfig) {
       const endpointTokenConfig = processModelData(input);
-      await tokenConfigCache().set(tokenKey ?? name, endpointTokenConfig);
+      const cache = tokenConfigCache();
+      await cache.set(tokenKey ?? name, endpointTokenConfig);
+      if (modelsCache && cacheKey) {
+        await cache.set(getModelCacheTokenConfigKey(cacheKey), endpointTokenConfig);
+      }
     }
     models = input.data.map((item: { id: string }) => item.id);
   } catch (error) {

packages/api/src/endpoints/tokenConfig.spec.ts

@@ -8,6 +8,7 @@ jest.mock('~/cache', () => ({
 }));
 
 import { resolveTokenConfigMap } from './tokenConfig';
+import { SCOPED_TOKEN_CONFIG_KEY_PREFIX } from './keys';
 
 const deps: TokenomicsDeps = {
   getValueKey: () => 'value-key',
@@ -56,6 +57,60 @@ describe('resolveTokenConfigMap', () => {
     expect(map.MyProxy['custom-model'].prompt).toBe(2);
   });
 
+  it('uses a tenant-scoped cache key for fetched custom endpoint token config', async () => {
+    mockCacheGet.mockImplementation(async (key: string) => {
+      if (key.startsWith(SCOPED_TOKEN_CONFIG_KEY_PREFIX)) {
+        return { 'custom-model': { prompt: 2, completion: 6, context: 16000 } };
+      }
+      if (key === 'MyProxy') {
+        return { 'custom-model': { prompt: 12, completion: 24, context: 111111 } };
+      }
+      return null;
+    });
+    const appConfig = appConfigWith([{ name: 'MyProxy', models: { fetch: true } }]);
+
+    const map = await resolveTokenConfigMap(
+      {
+        appConfig,
+        modelsConfig: { MyProxy: ['custom-model'] },
+        userId: 'user-2',
+        tenantId: 'tenant-b',
+      },
+      deps,
+    );
+
+    const tokenKey = mockCacheGet.mock.calls[0][0];
+    expect(tokenKey.startsWith(SCOPED_TOKEN_CONFIG_KEY_PREFIX)).toBe(true);
+    expect(tokenKey).not.toBe('tenant:tenant-b:MyProxy');
+    expect(tokenKey).not.toContain('tenant-b');
+    expect(tokenKey).not.toContain('MyProxy');
+    expect(map.MyProxy['custom-model'].context).toBe(16000);
+    expect(map.MyProxy['custom-model'].prompt).toBe(2);
+  });
+
+  it('uses the legacy cache key when tenant context is unavailable or empty', async () => {
+    mockCacheGet.mockResolvedValue({ 'custom-model': { prompt: 2, completion: 6, context: 8000 } });
+    const appConfig = appConfigWith([{ name: 'MyProxy', models: { fetch: true } }]);
+    const tenantIds = [undefined, null, '', '   '] as Array<string | null | undefined>;
+
+    for (const tenantId of tenantIds) {
+      mockCacheGet.mockClear();
+      const map = await resolveTokenConfigMap(
+        {
+          appConfig,
+          modelsConfig: { MyProxy: ['custom-model'] },
+          userId: 'user-1',
+          tenantId,
+        },
+        deps,
+      );
+
+      expect(mockCacheGet).toHaveBeenCalledWith('MyProxy');
+      expect(mockCacheGet.mock.calls.every(([key]) => key === 'MyProxy')).toBe(true);
+      expect(map.MyProxy['custom-model'].context).toBe(8000);
+    }
+  });
+
   it('omits pricing when contextCost is disabled', async () => {
     const appConfig = appConfigWith(
       [

packages/api/src/endpoints/tokenConfig.ts

@@ -11,6 +11,7 @@ export interface ResolveTokenConfigParams {
   /** endpoint → model list, from the resolved models config */
   modelsConfig: TModelsConfig;
   userId: string;
+  tenantId?: string | null;
 }
 
 /**
@@ -21,7 +22,7 @@ export interface ResolveTokenConfigParams {
  * Lives in TypeScript so the `/api` controller stays a thin wrapper.
  */
 export async function resolveTokenConfigMap(
-  { appConfig, modelsConfig, userId }: ResolveTokenConfigParams,
+  { appConfig, modelsConfig, userId, tenantId }: ResolveTokenConfigParams,
   deps: TokenomicsDeps,
 ): Promise<TTokenConfigMap> {
   const includePricing = appConfig?.interfaceConfig?.contextCost === true;
@@ -41,7 +42,7 @@ export async function resolveTokenConfigMap(
     }
     /** Model fetches and chat initialization both store under this key —
      *  user-scoped whenever the fetched config can be user-specific */
-    const tokenKey = getTokenConfigKey(endpointConfig, name, userId);
+    const tokenKey = getTokenConfigKey(endpointConfig, name, userId, tenantId);
     const cached = (await cache.get(tokenKey)) as EndpointTokenConfig | undefined;
     if (cached) {
       endpointTokenConfigs[name] = cached;