diff --git a/packages/api/src/endpoints/anthropic/helpers.ts b/packages/api/src/endpoints/anthropic/helpers.ts index 4de90d8491..612d947a22 100644 --- a/packages/api/src/endpoints/anthropic/helpers.ts +++ b/packages/api/src/endpoints/anthropic/helpers.ts @@ -10,6 +10,26 @@ import { } from 'librechat-data-provider'; import { matchModelName } from '~/utils/tokens'; +const FINE_GRAINED_TOOL_STREAMING_BETA = 'fine-grained-tool-streaming-2025-05-14'; + +function appendAnthropicBetaHeader( + headers: Record | undefined, + beta: string, +): Record { + const nextHeaders = { ...(headers ?? {}) }; + const betaValues = (nextHeaders['anthropic-beta'] ?? '') + .split(',') + .map((value) => value.trim()) + .filter(Boolean); + + if (!betaValues.includes(beta)) { + betaValues.push(beta); + } + + nextHeaders['anthropic-beta'] = betaValues.join(','); + return nextHeaders; +} + /** * @param {string} modelName * @returns {boolean} @@ -158,4 +178,11 @@ function configureReasoning( return updatedOptions; } -export { checkPromptCacheSupport, getClaudeHeaders, configureReasoning, supportsAdaptiveThinking }; +export { + FINE_GRAINED_TOOL_STREAMING_BETA, + appendAnthropicBetaHeader, + checkPromptCacheSupport, + getClaudeHeaders, + configureReasoning, + supportsAdaptiveThinking, +}; diff --git a/packages/api/src/endpoints/anthropic/llm.spec.ts b/packages/api/src/endpoints/anthropic/llm.spec.ts index 73a5a81c3a..9ab4ef11e8 100644 --- a/packages/api/src/endpoints/anthropic/llm.spec.ts +++ b/packages/api/src/endpoints/anthropic/llm.spec.ts @@ -1,5 +1,6 @@ import { AnthropicEffort, ThinkingDisplay } from 'librechat-data-provider'; import type * as t from '~/types'; +import { FINE_GRAINED_TOOL_STREAMING_BETA } from './helpers'; import { getLLMConfig } from './llm'; jest.mock('https-proxy-agent', () => ({ @@ -44,6 +45,17 @@ describe('getLLMConfig', () => { expect(result.llmConfig).toHaveProperty('anthropicApiUrl', 'http://reverse-proxy'); }); + it('should honor dropParams for clientOptions', () => { + const result = getLLMConfig('test-api-key', { + modelOptions: {}, + reverseProxyUrl: 'http://reverse-proxy', + dropParams: ['clientOptions'], + }); + + expect(result.llmConfig).not.toHaveProperty('clientOptions'); + expect(result.llmConfig).toHaveProperty('anthropicApiUrl', 'http://reverse-proxy'); + }); + it('should include topK and topP for non-Claude-3.7 models', () => { const result = getLLMConfig('test-api-key', { modelOptions: { @@ -95,7 +107,9 @@ describe('getLLMConfig', () => { }; const result = getLLMConfig('test-key', { modelOptions }); const clientOptions = result.llmConfig.clientOptions; - expect(clientOptions?.defaultHeaders).toBeUndefined(); + expect(clientOptions?.defaultHeaders).toEqual({ + 'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA, + }); expect(result.llmConfig.promptCache).toBe(true); }); @@ -110,7 +124,9 @@ describe('getLLMConfig', () => { const modelOptions = { model, promptCache: true }; const result = getLLMConfig('test-key', { modelOptions }); const clientOptions = result.llmConfig.clientOptions; - expect(clientOptions?.defaultHeaders).toBeUndefined(); + expect(clientOptions?.defaultHeaders).toEqual({ + 'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA, + }); expect(result.llmConfig.promptCache).toBe(true); }); }); @@ -122,7 +138,9 @@ describe('getLLMConfig', () => { }; const result = getLLMConfig('test-key', { modelOptions }); const clientOptions = result.llmConfig.clientOptions; - expect(clientOptions?.defaultHeaders).toBeUndefined(); + expect(clientOptions?.defaultHeaders).toEqual({ + 'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA, + }); expect(result.llmConfig.promptCache).toBe(true); }); @@ -137,7 +155,9 @@ describe('getLLMConfig', () => { const modelOptions = { model, promptCache: true }; const result = getLLMConfig('test-key', { modelOptions }); const clientOptions = result.llmConfig.clientOptions; - expect(clientOptions?.defaultHeaders).toBeUndefined(); + expect(clientOptions?.defaultHeaders).toEqual({ + 'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA, + }); expect(result.llmConfig.promptCache).toBe(true); }); }); @@ -149,7 +169,9 @@ describe('getLLMConfig', () => { }; const result = getLLMConfig('test-key', { modelOptions }); const clientOptions = result.llmConfig.clientOptions; - expect(clientOptions?.defaultHeaders).toBeUndefined(); + expect(clientOptions?.defaultHeaders).toEqual({ + 'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA, + }); expect(result.llmConfig.promptCache).toBe(true); }); @@ -165,7 +187,9 @@ describe('getLLMConfig', () => { const modelOptions = { model, promptCache: true }; const result = getLLMConfig('test-key', { modelOptions }); const clientOptions = result.llmConfig.clientOptions; - expect(clientOptions?.defaultHeaders).toBeUndefined(); + expect(clientOptions?.defaultHeaders).toEqual({ + 'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA, + }); expect(result.llmConfig.promptCache).toBe(true); }); }); @@ -327,7 +351,7 @@ describe('getLLMConfig', () => { // claude-3-5-sonnet supports prompt caching and should get the max-tokens header and promptCache boolean expect(result.llmConfig.clientOptions?.defaultHeaders).toEqual({ - 'anthropic-beta': 'max-tokens-3-5-sonnet-2024-07-15', + 'anthropic-beta': `max-tokens-3-5-sonnet-2024-07-15,${FINE_GRAINED_TOOL_STREAMING_BETA}`, }); expect(result.llmConfig.promptCache).toBe(true); }); @@ -537,7 +561,7 @@ describe('getLLMConfig', () => { expect(result.llmConfig).not.toHaveProperty('topK'); // Should have appropriate headers for Claude-3.7 with prompt cache expect(result.llmConfig.clientOptions?.defaultHeaders).toEqual({ - 'anthropic-beta': 'token-efficient-tools-2025-02-19,output-128k-2025-02-19', + 'anthropic-beta': `token-efficient-tools-2025-02-19,output-128k-2025-02-19,${FINE_GRAINED_TOOL_STREAMING_BETA}`, }); // Should pass promptCache boolean expect(result.llmConfig.promptCache).toBe(true); @@ -1545,12 +1569,14 @@ describe('getLLMConfig', () => { }); const headers = result.llmConfig.clientOptions?.defaultHeaders; + expect(headers).toBeDefined(); + const betaHeader = (headers as Record)['anthropic-beta']; + expect(betaHeader).toContain(FINE_GRAINED_TOOL_STREAMING_BETA); if (shouldHaveHeaders) { - expect(headers).toBeDefined(); - expect((headers as Record)['anthropic-beta']).toBeDefined(); + expect(betaHeader).not.toBe(FINE_GRAINED_TOOL_STREAMING_BETA); } else { - expect(headers).toBeUndefined(); + expect(betaHeader).toBe(FINE_GRAINED_TOOL_STREAMING_BETA); } if (shouldHavePromptCache) { diff --git a/packages/api/src/endpoints/anthropic/llm.ts b/packages/api/src/endpoints/anthropic/llm.ts index 9d36867047..0c4929621e 100644 --- a/packages/api/src/endpoints/anthropic/llm.ts +++ b/packages/api/src/endpoints/anthropic/llm.ts @@ -13,6 +13,8 @@ import type { AnthropicCredentials, } from '~/types/anthropic'; import { + FINE_GRAINED_TOOL_STREAMING_BETA, + appendAnthropicBetaHeader, supportsAdaptiveThinking, checkPromptCacheSupport, configureReasoning, @@ -24,6 +26,8 @@ import { getVertexDeploymentName, } from './vertex'; +const WEB_SEARCH_BETA = 'web-search-2025-03-05'; + /** * Parses credentials from string or object format * - If a valid JSON string is passed, it parses and returns the object @@ -270,6 +274,9 @@ function getLLMConfig( } /** Handle dropParams - only drop from Anthropic config */ + const shouldDropClientOptions = + Array.isArray(options.dropParams) && options.dropParams.includes('clientOptions'); + if (options.dropParams && Array.isArray(options.dropParams)) { options.dropParams.forEach((param) => { if (param === 'web_search') { @@ -294,18 +301,28 @@ function getLLMConfig( name: 'web_search', }); - if (isAnthropicVertexCredentials(creds)) { + if (isAnthropicVertexCredentials(creds) && !shouldDropClientOptions) { if (!requestOptions.clientOptions) { requestOptions.clientOptions = {}; } - requestOptions.clientOptions.defaultHeaders = { - ...requestOptions.clientOptions.defaultHeaders, - 'anthropic-beta': 'web-search-2025-03-05', - }; + requestOptions.clientOptions.defaultHeaders = appendAnthropicBetaHeader( + requestOptions.clientOptions.defaultHeaders as Record | undefined, + WEB_SEARCH_BETA, + ); } } + if (!shouldDropClientOptions) { + if (!requestOptions.clientOptions) { + requestOptions.clientOptions = {}; + } + requestOptions.clientOptions.defaultHeaders = appendAnthropicBetaHeader( + requestOptions.clientOptions.defaultHeaders as Record | undefined, + FINE_GRAINED_TOOL_STREAMING_BETA, + ); + } + return { tools, llmConfig: removeNullishValues( diff --git a/packages/api/src/endpoints/bedrock/initialize.spec.ts b/packages/api/src/endpoints/bedrock/initialize.spec.ts index 6571e66479..decf9a7df9 100644 --- a/packages/api/src/endpoints/bedrock/initialize.spec.ts +++ b/packages/api/src/endpoints/bedrock/initialize.spec.ts @@ -1,4 +1,9 @@ -import { AuthType, EModelEndpoint } from 'librechat-data-provider'; +import { + AuthType, + EModelEndpoint, + BEDROCK_OUTPUT_128K_BETA, + BEDROCK_FINE_GRAINED_TOOL_STREAMING_BETA, +} from 'librechat-data-provider'; import { initializeBedrock } from './initialize'; import type { BaseInitializeParams, BedrockLLMConfigResult } from '~/types'; import { checkUserKeyExpiry } from '~/utils'; @@ -23,6 +28,7 @@ jest.mock('~/utils', () => ({ })); const mockedCheckUserKeyExpiry = jest.mocked(checkUserKeyExpiry); +const BEDROCK_CLAUDE_4_BETAS = [BEDROCK_OUTPUT_128K_BETA, BEDROCK_FINE_GRAINED_TOOL_STREAMING_BETA]; const createMockParams = ( overrides: Partial<{ @@ -627,7 +633,7 @@ describe('initializeBedrock', () => { expect(amrf.thinking).toEqual({ type: 'adaptive' }); expect(result.llmConfig.maxTokens).toBeUndefined(); - expect(amrf.anthropic_beta).toEqual(expect.arrayContaining(['output-128k-2025-02-19'])); + expect(amrf.anthropic_beta).toEqual(expect.arrayContaining(BEDROCK_CLAUDE_4_BETAS)); }); it('should pass effort via output_config for Opus 4.6', async () => { @@ -645,6 +651,22 @@ describe('initializeBedrock', () => { expect(amrf.output_config).toEqual({ effort: 'medium' }); }); + it('should preserve user-provided anthropic beta values for Opus 4.6', async () => { + const params = createMockParams({ + model_parameters: { + model: 'anthropic.claude-opus-4-6-v1', + additionalModelRequestFields: { + anthropic_beta: ['context-1m-2025-08-07'], + }, + }, + }); + + const result = (await initializeBedrock(params)) as BedrockLLMConfigResult; + const amrf = result.llmConfig.additionalModelRequestFields as Record; + + expect(amrf.anthropic_beta).toEqual(['context-1m-2025-08-07', ...BEDROCK_CLAUDE_4_BETAS]); + }); + it('should respect user-provided maxTokens for Opus 4.6', async () => { const params = createMockParams({ model_parameters: { diff --git a/packages/api/src/endpoints/openai/config.anthropic.spec.ts b/packages/api/src/endpoints/openai/config.anthropic.spec.ts index f1c17ff85a..18448d36e6 100644 --- a/packages/api/src/endpoints/openai/config.anthropic.spec.ts +++ b/packages/api/src/endpoints/openai/config.anthropic.spec.ts @@ -1,4 +1,5 @@ import { getOpenAIConfig } from './config'; +import { FINE_GRAINED_TOOL_STREAMING_BETA } from '../anthropic/helpers'; describe('getOpenAIConfig - Anthropic Compatibility', () => { describe('Anthropic via LiteLLM', () => { @@ -44,7 +45,9 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => { }, configOptions: { baseURL: 'http://host.docker.internal:4000/v1', - defaultHeaders: {}, + defaultHeaders: { + 'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA, + }, }, tools: [], }); @@ -92,7 +95,7 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => { configOptions: { baseURL: 'http://localhost:4000/v1', defaultHeaders: { - 'anthropic-beta': 'token-efficient-tools-2025-02-19,output-128k-2025-02-19', + 'anthropic-beta': `token-efficient-tools-2025-02-19,output-128k-2025-02-19,${FINE_GRAINED_TOOL_STREAMING_BETA}`, }, }, tools: [], @@ -140,7 +143,7 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => { configOptions: { baseURL: 'http://localhost:4000/v1', defaultHeaders: { - 'anthropic-beta': 'token-efficient-tools-2025-02-19,output-128k-2025-02-19', + 'anthropic-beta': `token-efficient-tools-2025-02-19,output-128k-2025-02-19,${FINE_GRAINED_TOOL_STREAMING_BETA}`, }, }, tools: [], @@ -182,7 +185,7 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => { configOptions: { baseURL: 'https://api.anthropic.proxy.com/v1', defaultHeaders: { - 'anthropic-beta': 'max-tokens-3-5-sonnet-2024-07-15', + 'anthropic-beta': `max-tokens-3-5-sonnet-2024-07-15,${FINE_GRAINED_TOOL_STREAMING_BETA}`, }, }, tools: [], @@ -226,6 +229,7 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => { configOptions: { baseURL: 'http://custom.proxy/v1', defaultHeaders: { + 'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA, 'Custom-Header': 'custom-value', Authorization: 'Bearer custom-token', }, @@ -234,6 +238,33 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => { }); }); + it('should merge custom Anthropic beta headers with generated beta headers', () => { + const apiKey = 'sk-custom-beta'; + const endpoint = 'Anthropic (via LiteLLM)'; + const options = { + modelOptions: { + model: 'claude-3.5-sonnet-20240620', + }, + reverseProxyUrl: 'http://custom.proxy/v1', + headers: { + 'anthropic-beta': 'files-api-2025-04-14', + 'Custom-Header': 'custom-value', + }, + customParams: { + defaultParamsEndpoint: 'anthropic', + }, + endpoint: 'Anthropic (via LiteLLM)', + endpointType: 'custom', + }; + + const result = getOpenAIConfig(apiKey, options, endpoint); + + expect(result.configOptions?.defaultHeaders).toEqual({ + 'anthropic-beta': `files-api-2025-04-14,max-tokens-3-5-sonnet-2024-07-15,${FINE_GRAINED_TOOL_STREAMING_BETA}`, + 'Custom-Header': 'custom-value', + }); + }); + it('should handle models that do not match Claude patterns', () => { const apiKey = 'sk-other'; const endpoint = 'Anthropic (via LiteLLM)'; @@ -268,6 +299,9 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => { }, configOptions: { baseURL: 'http://litellm:4000/v1', + defaultHeaders: { + 'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA, + }, }, tools: [], }); @@ -312,6 +346,9 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => { }, configOptions: { baseURL: 'http://proxy.litellm/v1', + defaultHeaders: { + 'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA, + }, }, tools: [], }); @@ -351,6 +388,9 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => { }, configOptions: { baseURL: 'http://litellm/v1', + defaultHeaders: { + 'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA, + }, }, tools: [], }); @@ -390,6 +430,9 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => { }, configOptions: { baseURL: 'http://litellm/v1', + defaultHeaders: { + 'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA, + }, }, tools: [ { @@ -437,6 +480,9 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => { }, configOptions: { baseURL: 'http://litellm/v1', + defaultHeaders: { + 'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA, + }, }, tools: [], }); @@ -485,6 +531,9 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => { }, configOptions: { baseURL: 'http://litellm/v1', + defaultHeaders: { + 'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA, + }, }, tools: [], }); @@ -536,7 +585,7 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => { configOptions: { baseURL: 'http://litellm/v1', defaultHeaders: { - 'anthropic-beta': 'max-tokens-3-5-sonnet-2024-07-15', + 'anthropic-beta': `max-tokens-3-5-sonnet-2024-07-15,${FINE_GRAINED_TOOL_STREAMING_BETA}`, }, }, tools: [], diff --git a/packages/api/src/endpoints/openai/config.ts b/packages/api/src/endpoints/openai/config.ts index 5e8d8236ff..69aed82d80 100644 --- a/packages/api/src/endpoints/openai/config.ts +++ b/packages/api/src/endpoints/openai/config.ts @@ -11,6 +11,26 @@ import { createFetch } from '~/utils/generators'; type Fetch = (input: string | URL | Request, init?: RequestInit) => Promise; +function mergeHeadersPreservingAnthropicBeta( + headers: Record | undefined, + defaultHeaders: Record, +): Record { + const mergedHeaders = Object.assign({}, headers ?? {}, defaultHeaders); + const existingBetaHeader = headers?.['anthropic-beta']; + const defaultBetaHeader = defaultHeaders['anthropic-beta']; + + if (existingBetaHeader && defaultBetaHeader) { + const betaValues = [existingBetaHeader, defaultBetaHeader] + .flatMap((value) => value.split(',')) + .map((value) => value.trim()) + .filter(Boolean); + + mergedHeaders['anthropic-beta'] = Array.from(new Set(betaValues)).join(','); + } + + return mergedHeaders; +} + /** * Generates configuration options for creating a language model (LLM) instance. * @param apiKey - The API key for authentication. @@ -74,7 +94,10 @@ export function getOpenAIConfig( llmConfig = transformed.llmConfig; tools = anthropicResult.tools; if (transformed.configOptions?.defaultHeaders) { - headers = Object.assign(headers ?? {}, transformed.configOptions?.defaultHeaders); + headers = mergeHeadersPreservingAnthropicBeta( + headers, + transformed.configOptions.defaultHeaders as Record, + ); } } else if (isGoogle) { const googleResult = getGoogleConfig( diff --git a/packages/data-provider/specs/bedrock.spec.ts b/packages/data-provider/specs/bedrock.spec.ts index 6fde8cfa32..bbb57310c6 100644 --- a/packages/data-provider/specs/bedrock.spec.ts +++ b/packages/data-provider/specs/bedrock.spec.ts @@ -1,5 +1,6 @@ import { ThinkingDisplay } from '../src/schemas'; import { + BEDROCK_OUTPUT_128K_BETA, supportsAdaptiveThinking, omitsThinkingByDefault, resolveThinkingDisplay, @@ -7,8 +8,11 @@ import { bedrockInputParser, bedrockInputSchema, supportsContext1m, + BEDROCK_FINE_GRAINED_TOOL_STREAMING_BETA, } from '../src/bedrock'; +const BEDROCK_CLAUDE_4_BETAS = [BEDROCK_OUTPUT_128K_BETA, BEDROCK_FINE_GRAINED_TOOL_STREAMING_BETA]; + describe('supportsAdaptiveThinking', () => { test('should return true for claude-opus-4-6', () => { expect(supportsAdaptiveThinking('claude-opus-4-6')).toBe(true); @@ -306,7 +310,7 @@ describe('bedrockInputParser', () => { const additionalFields = result.additionalModelRequestFields as Record; expect(additionalFields.thinking).toBe(true); expect(additionalFields.thinkingBudget).toBe(2000); - expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']); + expect(additionalFields.anthropic_beta).toEqual([BEDROCK_OUTPUT_128K_BETA]); }); test('should match anthropic.claude-sonnet-4 model without context beta header', () => { @@ -317,7 +321,7 @@ describe('bedrockInputParser', () => { const additionalFields = result.additionalModelRequestFields as Record; expect(additionalFields.thinking).toBe(true); expect(additionalFields.thinkingBudget).toBe(2000); - expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']); + expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS); }); test('should match anthropic.claude-opus-5 model with adaptive thinking', () => { @@ -328,7 +332,7 @@ describe('bedrockInputParser', () => { const additionalFields = result.additionalModelRequestFields as Record; expect(additionalFields.thinking).toEqual({ type: 'adaptive', display: 'summarized' }); expect(additionalFields.thinkingBudget).toBeUndefined(); - expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']); + expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS); }); test('should match anthropic.claude-haiku-6 model without context beta header', () => { @@ -339,7 +343,7 @@ describe('bedrockInputParser', () => { const additionalFields = result.additionalModelRequestFields as Record; expect(additionalFields.thinking).toBe(true); expect(additionalFields.thinkingBudget).toBe(2000); - expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']); + expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS); }); test('should match anthropic.claude-4-sonnet model without context beta header', () => { @@ -350,7 +354,7 @@ describe('bedrockInputParser', () => { const additionalFields = result.additionalModelRequestFields as Record; expect(additionalFields.thinking).toBe(true); expect(additionalFields.thinkingBudget).toBe(2000); - expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']); + expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS); }); test('should match anthropic.claude-4.5-sonnet model without context beta header', () => { @@ -361,7 +365,7 @@ describe('bedrockInputParser', () => { const additionalFields = result.additionalModelRequestFields as Record; expect(additionalFields.thinking).toBe(true); expect(additionalFields.thinkingBudget).toBe(2000); - expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']); + expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS); }); test('should match anthropic.claude-sonnet-4-6 with adaptive thinking', () => { @@ -372,7 +376,7 @@ describe('bedrockInputParser', () => { const additionalFields = result.additionalModelRequestFields as Record; expect(additionalFields.thinking).toEqual({ type: 'adaptive' }); expect(additionalFields.thinkingBudget).toBeUndefined(); - expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']); + expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS); }); test('should match us.anthropic.claude-sonnet-4-6 with adaptive thinking', () => { @@ -383,7 +387,7 @@ describe('bedrockInputParser', () => { const additionalFields = result.additionalModelRequestFields as Record; expect(additionalFields.thinking).toEqual({ type: 'adaptive' }); expect(additionalFields.thinkingBudget).toBeUndefined(); - expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']); + expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS); }); test('should match anthropic.claude-4-7-sonnet model with adaptive thinking', () => { @@ -394,7 +398,7 @@ describe('bedrockInputParser', () => { const additionalFields = result.additionalModelRequestFields as Record; expect(additionalFields.thinking).toEqual({ type: 'adaptive' }); expect(additionalFields.thinkingBudget).toBeUndefined(); - expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']); + expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS); }); test('should match anthropic.claude-sonnet-4-20250514-v1:0 with full model ID', () => { @@ -405,7 +409,7 @@ describe('bedrockInputParser', () => { const additionalFields = result.additionalModelRequestFields as Record; expect(additionalFields.thinking).toBe(true); expect(additionalFields.thinkingBudget).toBe(2000); - expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']); + expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS); }); test('should not match non-Claude models', () => { @@ -447,7 +451,7 @@ describe('bedrockInputParser', () => { const additionalFields = result.additionalModelRequestFields as Record; expect(additionalFields.thinking).toBeUndefined(); expect(additionalFields.thinkingBudget).toBeUndefined(); - expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']); + expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS); }); test('should respect custom thinking budget', () => { @@ -461,6 +465,23 @@ describe('bedrockInputParser', () => { expect(additionalFields.thinking).toBe(true); expect(additionalFields.thinkingBudget).toBe(3000); }); + + test('should preserve user-provided anthropic_beta values for Claude 4 models', () => { + const input = { + model: 'anthropic.claude-sonnet-4', + additionalModelRequestFields: { + anthropic_beta: ['context-1m-2025-08-07'], + custom_flag: true, + }, + }; + const result = bedrockInputParser.parse(input) as Record; + const additionalFields = result.additionalModelRequestFields as Record; + expect(additionalFields.anthropic_beta).toEqual([ + 'context-1m-2025-08-07', + ...BEDROCK_CLAUDE_4_BETAS, + ]); + expect(additionalFields.custom_flag).toBe(true); + }); }); describe('Opus 4.6 Adaptive Thinking', () => { @@ -472,7 +493,7 @@ describe('bedrockInputParser', () => { const additionalFields = result.additionalModelRequestFields as Record; expect(additionalFields.thinking).toEqual({ type: 'adaptive' }); expect(additionalFields.thinkingBudget).toBeUndefined(); - expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']); + expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS); }); test('should handle cross-region model ID us.anthropic.claude-opus-4-6-v1', () => { @@ -483,7 +504,7 @@ describe('bedrockInputParser', () => { const additionalFields = result.additionalModelRequestFields as Record; expect(additionalFields.thinking).toEqual({ type: 'adaptive' }); expect(additionalFields.thinkingBudget).toBeUndefined(); - expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']); + expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS); }); test('should handle cross-region model ID global.anthropic.claude-opus-4-6-v1', () => { @@ -493,7 +514,7 @@ describe('bedrockInputParser', () => { const result = bedrockInputParser.parse(input) as Record; const additionalFields = result.additionalModelRequestFields as Record; expect(additionalFields.thinking).toEqual({ type: 'adaptive' }); - expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']); + expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS); }); test('should pass effort parameter via output_config for adaptive models', () => { @@ -697,7 +718,7 @@ describe('bedrockInputParser', () => { const additionalFields = result.additionalModelRequestFields as Record; expect(additionalFields.thinking).toBeUndefined(); expect(additionalFields.thinkingBudget).toBeUndefined(); - expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']); + expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS); }); test('should preserve effort when thinking=false for adaptive models', () => { diff --git a/packages/data-provider/src/bedrock.ts b/packages/data-provider/src/bedrock.ts index dc9e3c2bd3..96a4998079 100644 --- a/packages/data-provider/src/bedrock.ts +++ b/packages/data-provider/src/bedrock.ts @@ -3,6 +3,8 @@ import * as s from './schemas'; const DEFAULT_ENABLED_MAX_TOKENS = 8192; const DEFAULT_THINKING_BUDGET = 2000; +export const BEDROCK_OUTPUT_128K_BETA = 'output-128k-2025-02-19'; +export const BEDROCK_FINE_GRAINED_TOOL_STREAMING_BETA = 'fine-grained-tool-streaming-2025-05-14'; const bedrockReasoningConfigValues = new Set(Object.values(s.BedrockReasoningConfig)); @@ -157,19 +159,47 @@ export function supportsContext1m(model: string): boolean { function getBedrockAnthropicBetaHeaders(model: string): string[] { const betaHeaders: string[] = []; - const isClaudeThinkingModel = - model.includes('anthropic.claude-3-7-sonnet') || + const isClaude4PlusModel = /anthropic\.claude-(?:[4-9](?:\.\d+)?(?:-\d+)?-(?:sonnet|opus|haiku)|(?:sonnet|opus|haiku)-[4-9])/.test( model, ); + const isClaudeThinkingModel = model.includes('anthropic.claude-3-7-sonnet') || isClaude4PlusModel; if (isClaudeThinkingModel) { - betaHeaders.push('output-128k-2025-02-19'); + betaHeaders.push(BEDROCK_OUTPUT_128K_BETA); + } + + if (isClaude4PlusModel) { + betaHeaders.push(BEDROCK_FINE_GRAINED_TOOL_STREAMING_BETA); } return betaHeaders; } +function mergeBedrockAnthropicBetaHeaders(existing: unknown, generated: string[]): string[] { + const existingValues: unknown[] = Array.isArray(existing) + ? existing + : typeof existing === 'string' + ? [existing] + : []; + + const betaHeaders = new Set(); + + [...existingValues, ...generated].forEach((value) => { + if (typeof value !== 'string') { + return; + } + + value + .split(',') + .map((header) => header.trim()) + .filter(Boolean) + .forEach((header) => betaHeaders.add(header)); + }); + + return Array.from(betaHeaders); +} + export const bedrockInputSchema = s.tConversationSchema .pick({ /* LibreChat params; optionType: 'conversation' */ @@ -359,7 +389,13 @@ export const bedrockInputParser = s.tConversationSchema if ((typedData.model as string).includes('anthropic.')) { const betaHeaders = getBedrockAnthropicBetaHeaders(typedData.model as string); if (betaHeaders.length > 0) { - additionalFields.anthropic_beta = betaHeaders; + const existingBetaHeaders = ( + typedData.additionalModelRequestFields as Record | undefined + )?.anthropic_beta; + additionalFields.anthropic_beta = mergeBedrockAnthropicBetaHeaders( + existingBetaHeaders, + betaHeaders, + ); } } } else {