From 81783b2d52ae6d38b7637e4f8ec5fa2e4409a94d Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 20 Aug 2026 12:18:50 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=B2=20refactor:=20Consolidate=20Claude?= =?UTF-8?q?=20Prompt=20Cache=20and=20Context=20Checks=20(#15008)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🧭 fix: Unify Future Claude Capabilities * 🐛 fix: Cover Claude Capability Edge Cases --- e2e/specs/mock/agents.spec.ts | 11 +++--- .../api/src/endpoints/anthropic/helpers.ts | 22 +---------- .../api/src/endpoints/anthropic/llm.spec.ts | 6 +++ packages/api/src/utils/tokens.spec.ts | 18 +++++++++ packages/api/src/utils/tokens.ts | 25 ++++++------ packages/data-provider/src/bedrock.spec.ts | 14 +++++++ packages/data-provider/src/bedrock.ts | 22 +++++++++++ .../src/parameterSettings.spec.ts | 38 +++++++++++++++---- .../data-provider/src/parameterSettings.ts | 24 +++++++++--- 9 files changed, 130 insertions(+), 50 deletions(-) create mode 100644 packages/data-provider/src/bedrock.spec.ts diff --git a/e2e/specs/mock/agents.spec.ts b/e2e/specs/mock/agents.spec.ts index e935b06dbb..96b129e13f 100644 --- a/e2e/specs/mock/agents.spec.ts +++ b/e2e/specs/mock/agents.spec.ts @@ -20,7 +20,6 @@ const MODEL_PARAMETERS = { topP: 0.8, topK: 12, resendFiles: false, - promptCache: true, thinking: true, thinkingBudget: 2000, web_search: true, @@ -70,7 +69,9 @@ async function fillAnthropicStyleModelParameters(page: Page) { await form.locator('#fileTokenLimit-dynamic-input').fill(`${MODEL_PARAMETERS.fileTokenLimit}`); await setSwitch(form, 'Resend Files', MODEL_PARAMETERS.resendFiles); - await setSwitch(form, 'Use Prompt Caching', MODEL_PARAMETERS.promptCache); + await expect(form.getByRole('switch', { name: 'Use Prompt Caching', exact: true })).toHaveCount( + 0, + ); await setSwitch(form, 'Thinking', MODEL_PARAMETERS.thinking); await setSwitch(form, 'Web Search', MODEL_PARAMETERS.web_search); @@ -107,9 +108,9 @@ async function expectAnthropicStyleModelParameters(page: Page) { 'aria-checked', String(MODEL_PARAMETERS.resendFiles), ); - await expect( - form.getByRole('switch', { name: 'Use Prompt Caching', exact: true }), - ).toHaveAttribute('aria-checked', String(MODEL_PARAMETERS.promptCache)); + await expect(form.getByRole('switch', { name: 'Use Prompt Caching', exact: true })).toHaveCount( + 0, + ); await expect(form.getByRole('switch', { name: 'Thinking', exact: true })).toHaveAttribute( 'aria-checked', String(MODEL_PARAMETERS.thinking), diff --git a/packages/api/src/endpoints/anthropic/helpers.ts b/packages/api/src/endpoints/anthropic/helpers.ts index 6e5c31c3da..6f2692e0e1 100644 --- a/packages/api/src/endpoints/anthropic/helpers.ts +++ b/packages/api/src/endpoints/anthropic/helpers.ts @@ -1,16 +1,14 @@ import { logger } from '@librechat/data-schemas'; import { AnthropicClientOptions } from '@librechat/agents'; import { - EModelEndpoint, ThinkingDisplay, AnthropicEffort, anthropicSettings, - isMythosClassModel, resolveThinkingDisplay, supportsAdaptiveThinking, + supportsPromptCache, requiresExplicitThinkingDisabled, } from 'librechat-data-provider'; -import { matchModelName } from '~/utils/tokens'; const FINE_GRAINED_TOOL_STREAMING_BETA = 'fine-grained-tool-streaming-2025-05-14'; @@ -37,23 +35,7 @@ function appendAnthropicBetaHeader( * @returns {boolean} */ function checkPromptCacheSupport(modelName: string): boolean { - const modelMatch = matchModelName(modelName, EModelEndpoint.anthropic) ?? ''; - if ( - modelMatch.includes('claude-3-5-sonnet-latest') || - modelMatch.includes('claude-3.5-sonnet-latest') - ) { - return false; - } - - return ( - /claude-3[-.]7/.test(modelMatch) || - /claude-3[-.]5-(?:sonnet|haiku)/.test(modelMatch) || - /claude-3-(?:sonnet|haiku|opus)?/.test(modelMatch) || - /claude-(?:sonnet|opus|haiku)-[4-9]/.test(modelMatch) || - /claude-[4-9]-(?:sonnet|opus|haiku)?/.test(modelMatch) || - /claude-4(?:-(?:sonnet|opus|haiku))?/.test(modelMatch) || - isMythosClassModel(modelMatch) - ); + return supportsPromptCache(modelName); } /** diff --git a/packages/api/src/endpoints/anthropic/llm.spec.ts b/packages/api/src/endpoints/anthropic/llm.spec.ts index a664038f37..b3321f6075 100644 --- a/packages/api/src/endpoints/anthropic/llm.spec.ts +++ b/packages/api/src/endpoints/anthropic/llm.spec.ts @@ -1864,6 +1864,12 @@ describe('getLLMConfig', () => { shouldHaveHeaders: false, shouldHavePromptCache: true, }, + { + model: 'claude-sonnet-6', + promptCache: true, + shouldHaveHeaders: false, + shouldHavePromptCache: true, + }, // Models that support prompt cache but have no additional beta headers needed { model: 'claude-3-opus', diff --git a/packages/api/src/utils/tokens.spec.ts b/packages/api/src/utils/tokens.spec.ts index 7890af6c7c..9c8eb9de9e 100644 --- a/packages/api/src/utils/tokens.spec.ts +++ b/packages/api/src/utils/tokens.spec.ts @@ -7,6 +7,12 @@ describe('getModelMaxTokens partial-override fallback', () => { 'custom-model': { prompt: 1, completion: 2, context: 32000, output: 4096 }, }; + it('returns undefined for non-string model values from JavaScript consumers', () => { + for (const model of [undefined, null, 123]) { + expect(getModelMaxTokens(model as unknown as string)).toBeUndefined(); + } + }); + it('uses the override for a listed model', () => { expect(getModelMaxTokens('custom-model', EModelEndpoint.openAI, partialOverride)).toBe(32000); }); @@ -19,6 +25,18 @@ describe('getModelMaxTokens partial-override fallback', () => { }); }); +describe('future Claude context windows', () => { + it('uses the 1M profile for future Sonnet and Opus model IDs', () => { + for (const model of ['claude-sonnet-6', 'claude-opus-6']) { + expect(getModelMaxTokens(model, EModelEndpoint.anthropic)).toBe(1000000); + } + }); + + it('keeps the safe Claude fallback for unsupported model families', () => { + expect(getModelMaxTokens('claude-haiku-4', EModelEndpoint.anthropic)).toBe(100000); + }); +}); + describe('getModelMaxOutputTokens partial-override fallback', () => { const partialOverride: EndpointTokenConfig = { 'custom-model': { prompt: 1, completion: 2, context: 32000, output: 4096 }, diff --git a/packages/api/src/utils/tokens.ts b/packages/api/src/utils/tokens.ts index 8656c643b1..b3c543bb94 100644 --- a/packages/api/src/utils/tokens.ts +++ b/packages/api/src/utils/tokens.ts @@ -1,5 +1,5 @@ import z from 'zod'; -import { EModelEndpoint } from 'librechat-data-provider'; +import { EModelEndpoint, supportsContext1m } from 'librechat-data-provider'; import type { EndpointTokenConfig, TokenConfig } from '~/types'; /** @@ -185,7 +185,7 @@ const anthropicModels = { 'claude-mythos-5': 1000000, }; -const ANTHROPIC_SONNET_4_6_PLUS_CONTEXT = 1000000; +const ANTHROPIC_CONTEXT_1M = 1000000; const ANTHROPIC_SONNET_4_6_PLUS_OUTPUT = 128000; const ANTHROPIC_SONNET_4_6_PLUS_PATTERN = /(?:claude-sonnet[-.]?4[-.]?(?:[6-9]|\d{2})|claude[-.]?4[-.]?(?:[6-9]|\d{2})[-.]?sonnet)(?=$|[^0-9])/; @@ -200,14 +200,11 @@ function usesAnthropicContextMap(endpoint: EModelEndpoint): boolean { ); } -function getAnthropicSonnet46PlusContext( - modelName: string, - endpoint: EModelEndpoint, -): number | undefined { - if (!usesAnthropicContextMap(endpoint) || !ANTHROPIC_SONNET_4_6_PLUS_PATTERN.test(modelName)) { +function getAnthropicContext1m(modelName: string, endpoint: EModelEndpoint): number | undefined { + if (!usesAnthropicContextMap(endpoint) || !supportsContext1m(modelName)) { return undefined; } - return ANTHROPIC_SONNET_4_6_PLUS_CONTEXT; + return ANTHROPIC_CONTEXT_1M; } function getAnthropicSonnet46PlusOutput( @@ -673,6 +670,10 @@ export function getModelMaxTokens( endpoint: EModelEndpoint = EModelEndpoint.openAI, endpointTokenConfig?: EndpointTokenConfig, ): number | undefined { + if (typeof modelName !== 'string') { + return undefined; + } + /** A partial override only covers the models it lists; fall back to the * built-in map for unlisted models instead of dropping to the default * budget (matches buildTokenConfigMap and getMultiplier). */ @@ -682,9 +683,9 @@ export function getModelMaxTokens( return overrideValue; } } - const sonnet46PlusValue = getAnthropicSonnet46PlusContext(modelName, endpoint); - if (sonnet46PlusValue != null) { - return sonnet46PlusValue; + const context1mValue = getAnthropicContext1m(modelName, endpoint); + if (context1mValue != null) { + return context1mValue; } return getModelTokenValue(modelName, maxTokensMap[endpoint as keyof typeof maxTokensMap]); } @@ -753,7 +754,7 @@ export function matchModelName( const matchedPattern = findMatchingPattern(modelName, tokensMap); if ( (matchedPattern === 'claude-sonnet-4' || matchedPattern === 'claude-4') && - getAnthropicSonnet46PlusContext(modelName, endpoint) != null + getAnthropicContext1m(modelName, endpoint) != null ) { return modelName; } diff --git a/packages/data-provider/src/bedrock.spec.ts b/packages/data-provider/src/bedrock.spec.ts new file mode 100644 index 0000000000..05bd4ec1fb --- /dev/null +++ b/packages/data-provider/src/bedrock.spec.ts @@ -0,0 +1,14 @@ +import { supportsContext1m, supportsPromptCache } from './bedrock'; + +describe('Claude capability helpers', () => { + it('recognizes the 1M context window for future Sonnet and Opus model IDs', () => { + expect(supportsContext1m('claude-sonnet-6')).toBe(true); + expect(supportsContext1m('claude-opus-6')).toBe(true); + expect(supportsContext1m('claude-haiku-4')).toBe(false); + }); + + it('uses raw model IDs to resolve prompt-cache support', () => { + expect(supportsPromptCache('claude-sonnet-6')).toBe(true); + expect(supportsPromptCache('claude-3-5-sonnet-latest')).toBe(false); + }); +}); diff --git a/packages/data-provider/src/bedrock.ts b/packages/data-provider/src/bedrock.ts index 12e444f259..114cec9164 100644 --- a/packages/data-provider/src/bedrock.ts +++ b/packages/data-provider/src/bedrock.ts @@ -275,6 +275,28 @@ export function supportsContext1m(model: string): boolean { return false; } +/** + * Checks whether a native Anthropic Claude model supports prompt caching. + * + * This uses the configured model ID directly. Resolving it through a token + * map first can collapse a newly released Claude model to the generic + * `claude-` fallback and incorrectly disable cache control. + */ +export function supportsPromptCache(model: string): boolean { + if (model.includes('claude-3-5-sonnet-latest') || model.includes('claude-3.5-sonnet-latest')) { + return false; + } + + return ( + /claude-3[-.]7/.test(model) || + /claude-3[-.]5-(?:sonnet|haiku)/.test(model) || + /claude-3-(?:sonnet|haiku|opus)?/.test(model) || + /claude-(?:sonnet|opus|haiku)[-.]?(?:[4-9]|\d{2,})/.test(model) || + /claude-(?:[4-9]|\d{2,})(?:[-.](?:sonnet|opus|haiku))?/.test(model) || + s.isMythosClassModel(model) + ); +} + /** * A Bedrock Claude model ID may be prefixed (`anthropic.claude-*`, * `us.anthropic.claude-*`, `global.anthropic.claude-*`) or bare (`claude-*`, diff --git a/packages/data-provider/src/parameterSettings.spec.ts b/packages/data-provider/src/parameterSettings.spec.ts index a8e269525a..a072459bc1 100644 --- a/packages/data-provider/src/parameterSettings.spec.ts +++ b/packages/data-provider/src/parameterSettings.spec.ts @@ -1,9 +1,12 @@ -import { EModelEndpoint } from './types'; -import { applyModelAwareDefaults, paramSettings } from './parameterSettings'; import type { SettingDefinition } from './generate'; +import { applyModelAwareDefaults, paramSettings } from './parameterSettings'; +import { EModelEndpoint } from './types'; const googleParams = paramSettings[EModelEndpoint.google] as SettingDefinition[]; +const anthropicParams = paramSettings[EModelEndpoint.anthropic] as SettingDefinition[]; const maxOut = (params: SettingDefinition[]) => params.find((p) => p.key === 'maxOutputTokens'); +const hasSetting = (params: SettingDefinition[], key: string) => + params.some((param) => param.key === key); describe('applyModelAwareDefaults', () => { it('resolves the Google maxOutputTokens default for current Gemini models', () => { @@ -25,15 +28,34 @@ describe('applyModelAwareDefaults', () => { expect(maxOut(result)?.default).toBe(32768); }); - it('returns settings unchanged for non-Google endpoints', () => { - const result = applyModelAwareDefaults( - googleParams, - EModelEndpoint.anthropic, - 'gemini-2.5-pro', - ); + it('returns settings unchanged for unrelated endpoints', () => { + const result = applyModelAwareDefaults(googleParams, EModelEndpoint.openAI, 'gemini-2.5-pro'); expect(result).toBe(googleParams); }); + it('keeps prompt-cache controls for future Claude models that support caching', () => { + const result = applyModelAwareDefaults( + anthropicParams, + EModelEndpoint.anthropic, + 'claude-sonnet-6', + ); + + expect(hasSetting(result, 'promptCache')).toBe(true); + expect(hasSetting(result, 'promptCacheTtl')).toBe(true); + }); + + it('hides prompt-cache controls for Anthropic models that do not support caching', () => { + const result = applyModelAwareDefaults( + anthropicParams, + EModelEndpoint.anthropic, + 'claude-3-5-sonnet-latest', + ); + + expect(hasSetting(result, 'promptCache')).toBe(false); + expect(hasSetting(result, 'promptCacheTtl')).toBe(false); + expect(hasSetting(result, 'temperature')).toBe(true); + }); + it('returns settings unchanged when no model is provided', () => { expect(applyModelAwareDefaults(googleParams, EModelEndpoint.google, '')).toBe(googleParams); }); diff --git a/packages/data-provider/src/parameterSettings.ts b/packages/data-provider/src/parameterSettings.ts index f2a65b7dff..af2b779119 100644 --- a/packages/data-provider/src/parameterSettings.ts +++ b/packages/data-provider/src/parameterSettings.ts @@ -16,6 +16,7 @@ import { anthropicSettings, } from './types'; import { SettingDefinition, SettingsConfiguration } from './generate'; +import { supportsPromptCache } from './bedrock'; // Base definitions const baseDefinitions: Record = { @@ -1253,18 +1254,31 @@ export const agentParamSettings: Record - setting.key === 'maxOutputTokens' - ? { ...setting, default: googleSettings.maxOutputTokens.reset(model) } - : setting, + + const modelAwareSettings = + endpoint === EModelEndpoint.google + ? settings.map((setting) => + setting.key === 'maxOutputTokens' + ? { ...setting, default: googleSettings.maxOutputTokens.reset(model) } + : setting, + ) + : settings; + + if (endpoint !== EModelEndpoint.anthropic || supportsPromptCache(model)) { + return modelAwareSettings; + } + + return modelAwareSettings.filter( + (setting) => setting.key !== 'promptCache' && setting.key !== 'promptCacheTtl', ); }