CERBERUS/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2026-08-03 22:32:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

🧵 feat: Enable Anthropic Tool Argument Streaming (#12962)

Browse source 
* fix: Enable Anthropic Tool Argument Streaming

* fix: Honor Anthropic clientOptions drops

* fix: Preserve custom Anthropic beta headers

* fix: Enable Bedrock Anthropic Tool Streaming
This commit is contained in:
Danny Avila 2026-05-06 01:09:14 -04:00 • committed by GitHub
parent 25a4556aee
commit cf0657509c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 265 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

29
packages/api/src/endpoints/anthropic/helpers.ts
View file

@ -10,6 +10,26 @@ import {
} from 'librechat-data-provider';
import { matchModelName } from '~/utils/tokens';
const FINE_GRAINED_TOOL_STREAMING_BETA = 'fine-grained-tool-streaming-2025-05-14';
function appendAnthropicBetaHeader(
headers: Record<string, string> | undefined,
beta: string,
): Record<string, string> {
const nextHeaders = { ...(headers ?? {}) };
const betaValues = (nextHeaders['anthropic-beta'] ?? '')
.split(',')
.map((value) => value.trim())
.filter(Boolean);
if (!betaValues.includes(beta)) {
betaValues.push(beta);
}
nextHeaders['anthropic-beta'] = betaValues.join(',');
return nextHeaders;
}
/**
* @param {string} modelName
* @returns {boolean}
@ -158,4 +178,11 @@ function configureReasoning(
return updatedOptions;
}
export { checkPromptCacheSupport, getClaudeHeaders, configureReasoning, supportsAdaptiveThinking };
export {
FINE_GRAINED_TOOL_STREAMING_BETA,
appendAnthropicBetaHeader,
checkPromptCacheSupport,
getClaudeHeaders,
configureReasoning,
supportsAdaptiveThinking,
};

48
packages/api/src/endpoints/anthropic/llm.spec.ts
View file

@ -1,5 +1,6 @@
import { AnthropicEffort, ThinkingDisplay } from 'librechat-data-provider';
import type * as t from '~/types';
import { FINE_GRAINED_TOOL_STREAMING_BETA } from './helpers';
import { getLLMConfig } from './llm';
jest.mock('https-proxy-agent', () => ({
@ -44,6 +45,17 @@ describe('getLLMConfig', () => {
expect(result.llmConfig).toHaveProperty('anthropicApiUrl', 'http://reverse-proxy');
});
it('should honor dropParams for clientOptions', () => {
const result = getLLMConfig('test-api-key', {
modelOptions: {},
reverseProxyUrl: 'http://reverse-proxy',
dropParams: ['clientOptions'],
});
expect(result.llmConfig).not.toHaveProperty('clientOptions');
expect(result.llmConfig).toHaveProperty('anthropicApiUrl', 'http://reverse-proxy');
});
it('should include topK and topP for non-Claude-3.7 models', () => {
const result = getLLMConfig('test-api-key', {
modelOptions: {
@ -95,7 +107,9 @@ describe('getLLMConfig', () => {
};
const result = getLLMConfig('test-key', { modelOptions });
const clientOptions = result.llmConfig.clientOptions;
expect(clientOptions?.defaultHeaders).toBeUndefined();
expect(clientOptions?.defaultHeaders).toEqual({
'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA,
});
expect(result.llmConfig.promptCache).toBe(true);
});
@ -110,7 +124,9 @@ describe('getLLMConfig', () => {
const modelOptions = { model, promptCache: true };
const result = getLLMConfig('test-key', { modelOptions });
const clientOptions = result.llmConfig.clientOptions;
expect(clientOptions?.defaultHeaders).toBeUndefined();
expect(clientOptions?.defaultHeaders).toEqual({
'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA,
});
expect(result.llmConfig.promptCache).toBe(true);
});
});
@ -122,7 +138,9 @@ describe('getLLMConfig', () => {
};
const result = getLLMConfig('test-key', { modelOptions });
const clientOptions = result.llmConfig.clientOptions;
expect(clientOptions?.defaultHeaders).toBeUndefined();
expect(clientOptions?.defaultHeaders).toEqual({
'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA,
});
expect(result.llmConfig.promptCache).toBe(true);
});
@ -137,7 +155,9 @@ describe('getLLMConfig', () => {
const modelOptions = { model, promptCache: true };
const result = getLLMConfig('test-key', { modelOptions });
const clientOptions = result.llmConfig.clientOptions;
expect(clientOptions?.defaultHeaders).toBeUndefined();
expect(clientOptions?.defaultHeaders).toEqual({
'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA,
});
expect(result.llmConfig.promptCache).toBe(true);
});
});
@ -149,7 +169,9 @@ describe('getLLMConfig', () => {
};
const result = getLLMConfig('test-key', { modelOptions });
const clientOptions = result.llmConfig.clientOptions;
expect(clientOptions?.defaultHeaders).toBeUndefined();
expect(clientOptions?.defaultHeaders).toEqual({
'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA,
});
expect(result.llmConfig.promptCache).toBe(true);
});
@ -165,7 +187,9 @@ describe('getLLMConfig', () => {
const modelOptions = { model, promptCache: true };
const result = getLLMConfig('test-key', { modelOptions });
const clientOptions = result.llmConfig.clientOptions;
expect(clientOptions?.defaultHeaders).toBeUndefined();
expect(clientOptions?.defaultHeaders).toEqual({
'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA,
});
expect(result.llmConfig.promptCache).toBe(true);
});
});
@ -327,7 +351,7 @@ describe('getLLMConfig', () => {
// claude-3-5-sonnet supports prompt caching and should get the max-tokens header and promptCache boolean
expect(result.llmConfig.clientOptions?.defaultHeaders).toEqual({
'anthropic-beta': 'max-tokens-3-5-sonnet-2024-07-15',
'anthropic-beta': `max-tokens-3-5-sonnet-2024-07-15,${FINE_GRAINED_TOOL_STREAMING_BETA}`,
});
expect(result.llmConfig.promptCache).toBe(true);
});
@ -537,7 +561,7 @@ describe('getLLMConfig', () => {
expect(result.llmConfig).not.toHaveProperty('topK');
// Should have appropriate headers for Claude-3.7 with prompt cache
expect(result.llmConfig.clientOptions?.defaultHeaders).toEqual({
'anthropic-beta': 'token-efficient-tools-2025-02-19,output-128k-2025-02-19',
'anthropic-beta': `token-efficient-tools-2025-02-19,output-128k-2025-02-19,${FINE_GRAINED_TOOL_STREAMING_BETA}`,
});
// Should pass promptCache boolean
expect(result.llmConfig.promptCache).toBe(true);
@ -1545,12 +1569,14 @@ describe('getLLMConfig', () => {
});
const headers = result.llmConfig.clientOptions?.defaultHeaders;
expect(headers).toBeDefined();
const betaHeader = (headers as Record<string, string>)['anthropic-beta'];
expect(betaHeader).toContain(FINE_GRAINED_TOOL_STREAMING_BETA);
if (shouldHaveHeaders) {
expect(headers).toBeDefined();
expect((headers as Record<string, string>)['anthropic-beta']).toBeDefined();
expect(betaHeader).not.toBe(FINE_GRAINED_TOOL_STREAMING_BETA);
} else {
expect(headers).toBeUndefined();
expect(betaHeader).toBe(FINE_GRAINED_TOOL_STREAMING_BETA);
}
if (shouldHavePromptCache) {

27
packages/api/src/endpoints/anthropic/llm.ts
View file

@ -13,6 +13,8 @@ import type {
AnthropicCredentials,
} from '~/types/anthropic';
import {
FINE_GRAINED_TOOL_STREAMING_BETA,
appendAnthropicBetaHeader,
supportsAdaptiveThinking,
checkPromptCacheSupport,
configureReasoning,
@ -24,6 +26,8 @@ import {
getVertexDeploymentName,
} from './vertex';
const WEB_SEARCH_BETA = 'web-search-2025-03-05';
/**
* Parses credentials from string or object format
* - If a valid JSON string is passed, it parses and returns the object
@ -270,6 +274,9 @@ function getLLMConfig(
}
/** Handle dropParams - only drop from Anthropic config */
const shouldDropClientOptions =
Array.isArray(options.dropParams) && options.dropParams.includes('clientOptions');
if (options.dropParams && Array.isArray(options.dropParams)) {
options.dropParams.forEach((param) => {
if (param === 'web_search') {
@ -294,18 +301,28 @@ function getLLMConfig(
name: 'web_search',
});
if (isAnthropicVertexCredentials(creds)) {
if (isAnthropicVertexCredentials(creds) && !shouldDropClientOptions) {
if (!requestOptions.clientOptions) {
requestOptions.clientOptions = {};
}
requestOptions.clientOptions.defaultHeaders = {
...requestOptions.clientOptions.defaultHeaders,
'anthropic-beta': 'web-search-2025-03-05',
};
requestOptions.clientOptions.defaultHeaders = appendAnthropicBetaHeader(
requestOptions.clientOptions.defaultHeaders as Record<string, string> | undefined,
WEB_SEARCH_BETA,
);
}
}
if (!shouldDropClientOptions) {
if (!requestOptions.clientOptions) {
requestOptions.clientOptions = {};
}
requestOptions.clientOptions.defaultHeaders = appendAnthropicBetaHeader(
requestOptions.clientOptions.defaultHeaders as Record<string, string> | undefined,
FINE_GRAINED_TOOL_STREAMING_BETA,
);
}
return {
tools,
llmConfig: removeNullishValues(

26
packages/api/src/endpoints/bedrock/initialize.spec.ts
View file

@ -1,4 +1,9 @@
import { AuthType, EModelEndpoint } from 'librechat-data-provider';
import {
AuthType,
EModelEndpoint,
BEDROCK_OUTPUT_128K_BETA,
BEDROCK_FINE_GRAINED_TOOL_STREAMING_BETA,
} from 'librechat-data-provider';
import { initializeBedrock } from './initialize';
import type { BaseInitializeParams, BedrockLLMConfigResult } from '~/types';
import { checkUserKeyExpiry } from '~/utils';
@ -23,6 +28,7 @@ jest.mock('~/utils', () => ({
}));
const mockedCheckUserKeyExpiry = jest.mocked(checkUserKeyExpiry);
const BEDROCK_CLAUDE_4_BETAS = [BEDROCK_OUTPUT_128K_BETA, BEDROCK_FINE_GRAINED_TOOL_STREAMING_BETA];
const createMockParams = (
overrides: Partial<{
@ -627,7 +633,7 @@ describe('initializeBedrock', () => {
expect(amrf.thinking).toEqual({ type: 'adaptive' });
expect(result.llmConfig.maxTokens).toBeUndefined();
expect(amrf.anthropic_beta).toEqual(expect.arrayContaining(['output-128k-2025-02-19']));
expect(amrf.anthropic_beta).toEqual(expect.arrayContaining(BEDROCK_CLAUDE_4_BETAS));
});
it('should pass effort via output_config for Opus 4.6', async () => {
@ -645,6 +651,22 @@ describe('initializeBedrock', () => {
expect(amrf.output_config).toEqual({ effort: 'medium' });
});
it('should preserve user-provided anthropic beta values for Opus 4.6', async () => {
const params = createMockParams({
model_parameters: {
model: 'anthropic.claude-opus-4-6-v1',
additionalModelRequestFields: {
anthropic_beta: ['context-1m-2025-08-07'],
},
},
});
const result = (await initializeBedrock(params)) as BedrockLLMConfigResult;
const amrf = result.llmConfig.additionalModelRequestFields as Record<string, unknown>;
expect(amrf.anthropic_beta).toEqual(['context-1m-2025-08-07', ...BEDROCK_CLAUDE_4_BETAS]);
});
it('should respect user-provided maxTokens for Opus 4.6', async () => {
const params = createMockParams({
model_parameters: {

59
packages/api/src/endpoints/openai/config.anthropic.spec.ts
View file

@ -1,4 +1,5 @@
import { getOpenAIConfig } from './config';
import { FINE_GRAINED_TOOL_STREAMING_BETA } from '../anthropic/helpers';
describe('getOpenAIConfig - Anthropic Compatibility', () => {
describe('Anthropic via LiteLLM', () => {
@ -44,7 +45,9 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => {
},
configOptions: {
baseURL: 'http://host.docker.internal:4000/v1',
defaultHeaders: {},
defaultHeaders: {
'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA,
},
},
tools: [],
});
@ -92,7 +95,7 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => {
configOptions: {
baseURL: 'http://localhost:4000/v1',
defaultHeaders: {
'anthropic-beta': 'token-efficient-tools-2025-02-19,output-128k-2025-02-19',
'anthropic-beta': `token-efficient-tools-2025-02-19,output-128k-2025-02-19,${FINE_GRAINED_TOOL_STREAMING_BETA}`,
},
},
tools: [],
@ -140,7 +143,7 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => {
configOptions: {
baseURL: 'http://localhost:4000/v1',
defaultHeaders: {
'anthropic-beta': 'token-efficient-tools-2025-02-19,output-128k-2025-02-19',
'anthropic-beta': `token-efficient-tools-2025-02-19,output-128k-2025-02-19,${FINE_GRAINED_TOOL_STREAMING_BETA}`,
},
},
tools: [],
@ -182,7 +185,7 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => {
configOptions: {
baseURL: 'https://api.anthropic.proxy.com/v1',
defaultHeaders: {
'anthropic-beta': 'max-tokens-3-5-sonnet-2024-07-15',
'anthropic-beta': `max-tokens-3-5-sonnet-2024-07-15,${FINE_GRAINED_TOOL_STREAMING_BETA}`,
},
},
tools: [],
@ -226,6 +229,7 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => {
configOptions: {
baseURL: 'http://custom.proxy/v1',
defaultHeaders: {
'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA,
'Custom-Header': 'custom-value',
Authorization: 'Bearer custom-token',
},
@ -234,6 +238,33 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => {
});
});
it('should merge custom Anthropic beta headers with generated beta headers', () => {
const apiKey = 'sk-custom-beta';
const endpoint = 'Anthropic (via LiteLLM)';
const options = {
modelOptions: {
model: 'claude-3.5-sonnet-20240620',
},
reverseProxyUrl: 'http://custom.proxy/v1',
headers: {
'anthropic-beta': 'files-api-2025-04-14',
'Custom-Header': 'custom-value',
},
customParams: {
defaultParamsEndpoint: 'anthropic',
},
endpoint: 'Anthropic (via LiteLLM)',
endpointType: 'custom',
};
const result = getOpenAIConfig(apiKey, options, endpoint);
expect(result.configOptions?.defaultHeaders).toEqual({
'anthropic-beta': `files-api-2025-04-14,max-tokens-3-5-sonnet-2024-07-15,${FINE_GRAINED_TOOL_STREAMING_BETA}`,
'Custom-Header': 'custom-value',
});
});
it('should handle models that do not match Claude patterns', () => {
const apiKey = 'sk-other';
const endpoint = 'Anthropic (via LiteLLM)';
@ -268,6 +299,9 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => {
},
configOptions: {
baseURL: 'http://litellm:4000/v1',
defaultHeaders: {
'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA,
},
},
tools: [],
});
@ -312,6 +346,9 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => {
},
configOptions: {
baseURL: 'http://proxy.litellm/v1',
defaultHeaders: {
'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA,
},
},
tools: [],
});
@ -351,6 +388,9 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => {
},
configOptions: {
baseURL: 'http://litellm/v1',
defaultHeaders: {
'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA,
},
},
tools: [],
});
@ -390,6 +430,9 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => {
},
configOptions: {
baseURL: 'http://litellm/v1',
defaultHeaders: {
'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA,
},
},
tools: [
{
@ -437,6 +480,9 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => {
},
configOptions: {
baseURL: 'http://litellm/v1',
defaultHeaders: {
'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA,
},
},
tools: [],
});
@ -485,6 +531,9 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => {
},
configOptions: {
baseURL: 'http://litellm/v1',
defaultHeaders: {
'anthropic-beta': FINE_GRAINED_TOOL_STREAMING_BETA,
},
},
tools: [],
});
@ -536,7 +585,7 @@ describe('getOpenAIConfig - Anthropic Compatibility', () => {
configOptions: {
baseURL: 'http://litellm/v1',
defaultHeaders: {
'anthropic-beta': 'max-tokens-3-5-sonnet-2024-07-15',
'anthropic-beta': `max-tokens-3-5-sonnet-2024-07-15,${FINE_GRAINED_TOOL_STREAMING_BETA}`,
},
},
tools: [],

25
packages/api/src/endpoints/openai/config.ts
View file

@ -11,6 +11,26 @@ import { createFetch } from '~/utils/generators';
type Fetch = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
function mergeHeadersPreservingAnthropicBeta(
headers: Record<string, string> | undefined,
defaultHeaders: Record<string, string>,
): Record<string, string> {
const mergedHeaders = Object.assign({}, headers ?? {}, defaultHeaders);
const existingBetaHeader = headers?.['anthropic-beta'];
const defaultBetaHeader = defaultHeaders['anthropic-beta'];
if (existingBetaHeader && defaultBetaHeader) {
const betaValues = [existingBetaHeader, defaultBetaHeader]
.flatMap((value) => value.split(','))
.map((value) => value.trim())
.filter(Boolean);
mergedHeaders['anthropic-beta'] = Array.from(new Set(betaValues)).join(',');
}
return mergedHeaders;
}
/**
* Generates configuration options for creating a language model (LLM) instance.
* @param apiKey - The API key for authentication.
@ -74,7 +94,10 @@ export function getOpenAIConfig(
llmConfig = transformed.llmConfig;
tools = anthropicResult.tools;
if (transformed.configOptions?.defaultHeaders) {
headers = Object.assign(headers ?? {}, transformed.configOptions?.defaultHeaders);
headers = mergeHeadersPreservingAnthropicBeta(
headers,
transformed.configOptions.defaultHeaders as Record<string, string>,
);
}
} else if (isGoogle) {
const googleResult = getGoogleConfig(

51
packages/data-provider/specs/bedrock.spec.ts
View file

@ -1,5 +1,6 @@
import { ThinkingDisplay } from '../src/schemas';
import {
BEDROCK_OUTPUT_128K_BETA,
supportsAdaptiveThinking,
omitsThinkingByDefault,
resolveThinkingDisplay,
@ -7,8 +8,11 @@ import {
bedrockInputParser,
bedrockInputSchema,
supportsContext1m,
BEDROCK_FINE_GRAINED_TOOL_STREAMING_BETA,
} from '../src/bedrock';
const BEDROCK_CLAUDE_4_BETAS = [BEDROCK_OUTPUT_128K_BETA, BEDROCK_FINE_GRAINED_TOOL_STREAMING_BETA];
describe('supportsAdaptiveThinking', () => {
test('should return true for claude-opus-4-6', () => {
expect(supportsAdaptiveThinking('claude-opus-4-6')).toBe(true);
@ -306,7 +310,7 @@ describe('bedrockInputParser', () => {
const additionalFields = result.additionalModelRequestFields as Record<string, unknown>;
expect(additionalFields.thinking).toBe(true);
expect(additionalFields.thinkingBudget).toBe(2000);
expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']);
expect(additionalFields.anthropic_beta).toEqual([BEDROCK_OUTPUT_128K_BETA]);
});
test('should match anthropic.claude-sonnet-4 model without context beta header', () => {
@ -317,7 +321,7 @@ describe('bedrockInputParser', () => {
const additionalFields = result.additionalModelRequestFields as Record<string, unknown>;
expect(additionalFields.thinking).toBe(true);
expect(additionalFields.thinkingBudget).toBe(2000);
expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']);
expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS);
});
test('should match anthropic.claude-opus-5 model with adaptive thinking', () => {
@ -328,7 +332,7 @@ describe('bedrockInputParser', () => {
const additionalFields = result.additionalModelRequestFields as Record<string, unknown>;
expect(additionalFields.thinking).toEqual({ type: 'adaptive', display: 'summarized' });
expect(additionalFields.thinkingBudget).toBeUndefined();
expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']);
expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS);
});
test('should match anthropic.claude-haiku-6 model without context beta header', () => {
@ -339,7 +343,7 @@ describe('bedrockInputParser', () => {
const additionalFields = result.additionalModelRequestFields as Record<string, unknown>;
expect(additionalFields.thinking).toBe(true);
expect(additionalFields.thinkingBudget).toBe(2000);
expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']);
expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS);
});
test('should match anthropic.claude-4-sonnet model without context beta header', () => {
@ -350,7 +354,7 @@ describe('bedrockInputParser', () => {
const additionalFields = result.additionalModelRequestFields as Record<string, unknown>;
expect(additionalFields.thinking).toBe(true);
expect(additionalFields.thinkingBudget).toBe(2000);
expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']);
expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS);
});
test('should match anthropic.claude-4.5-sonnet model without context beta header', () => {
@ -361,7 +365,7 @@ describe('bedrockInputParser', () => {
const additionalFields = result.additionalModelRequestFields as Record<string, unknown>;
expect(additionalFields.thinking).toBe(true);
expect(additionalFields.thinkingBudget).toBe(2000);
expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']);
expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS);
});
test('should match anthropic.claude-sonnet-4-6 with adaptive thinking', () => {
@ -372,7 +376,7 @@ describe('bedrockInputParser', () => {
const additionalFields = result.additionalModelRequestFields as Record<string, unknown>;
expect(additionalFields.thinking).toEqual({ type: 'adaptive' });
expect(additionalFields.thinkingBudget).toBeUndefined();
expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']);
expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS);
});
test('should match us.anthropic.claude-sonnet-4-6 with adaptive thinking', () => {
@ -383,7 +387,7 @@ describe('bedrockInputParser', () => {
const additionalFields = result.additionalModelRequestFields as Record<string, unknown>;
expect(additionalFields.thinking).toEqual({ type: 'adaptive' });
expect(additionalFields.thinkingBudget).toBeUndefined();
expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']);
expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS);
});
test('should match anthropic.claude-4-7-sonnet model with adaptive thinking', () => {
@ -394,7 +398,7 @@ describe('bedrockInputParser', () => {
const additionalFields = result.additionalModelRequestFields as Record<string, unknown>;
expect(additionalFields.thinking).toEqual({ type: 'adaptive' });
expect(additionalFields.thinkingBudget).toBeUndefined();
expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']);
expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS);
});
test('should match anthropic.claude-sonnet-4-20250514-v1:0 with full model ID', () => {
@ -405,7 +409,7 @@ describe('bedrockInputParser', () => {
const additionalFields = result.additionalModelRequestFields as Record<string, unknown>;
expect(additionalFields.thinking).toBe(true);
expect(additionalFields.thinkingBudget).toBe(2000);
expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']);
expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS);
});
test('should not match non-Claude models', () => {
@ -447,7 +451,7 @@ describe('bedrockInputParser', () => {
const additionalFields = result.additionalModelRequestFields as Record<string, unknown>;
expect(additionalFields.thinking).toBeUndefined();
expect(additionalFields.thinkingBudget).toBeUndefined();
expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']);
expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS);
});
test('should respect custom thinking budget', () => {
@ -461,6 +465,23 @@ describe('bedrockInputParser', () => {
expect(additionalFields.thinking).toBe(true);
expect(additionalFields.thinkingBudget).toBe(3000);
});
test('should preserve user-provided anthropic_beta values for Claude 4 models', () => {
const input = {
model: 'anthropic.claude-sonnet-4',
additionalModelRequestFields: {
anthropic_beta: ['context-1m-2025-08-07'],
custom_flag: true,
},
};
const result = bedrockInputParser.parse(input) as Record<string, unknown>;
const additionalFields = result.additionalModelRequestFields as Record<string, unknown>;
expect(additionalFields.anthropic_beta).toEqual([
'context-1m-2025-08-07',
...BEDROCK_CLAUDE_4_BETAS,
]);
expect(additionalFields.custom_flag).toBe(true);
});
});
describe('Opus 4.6 Adaptive Thinking', () => {
@ -472,7 +493,7 @@ describe('bedrockInputParser', () => {
const additionalFields = result.additionalModelRequestFields as Record<string, unknown>;
expect(additionalFields.thinking).toEqual({ type: 'adaptive' });
expect(additionalFields.thinkingBudget).toBeUndefined();
expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']);
expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS);
});
test('should handle cross-region model ID us.anthropic.claude-opus-4-6-v1', () => {
@ -483,7 +504,7 @@ describe('bedrockInputParser', () => {
const additionalFields = result.additionalModelRequestFields as Record<string, unknown>;
expect(additionalFields.thinking).toEqual({ type: 'adaptive' });
expect(additionalFields.thinkingBudget).toBeUndefined();
expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']);
expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS);
});
test('should handle cross-region model ID global.anthropic.claude-opus-4-6-v1', () => {
@ -493,7 +514,7 @@ describe('bedrockInputParser', () => {
const result = bedrockInputParser.parse(input) as Record<string, unknown>;
const additionalFields = result.additionalModelRequestFields as Record<string, unknown>;
expect(additionalFields.thinking).toEqual({ type: 'adaptive' });
expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']);
expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS);
});
test('should pass effort parameter via output_config for adaptive models', () => {
@ -697,7 +718,7 @@ describe('bedrockInputParser', () => {
const additionalFields = result.additionalModelRequestFields as Record<string, unknown>;
expect(additionalFields.thinking).toBeUndefined();
expect(additionalFields.thinkingBudget).toBeUndefined();
expect(additionalFields.anthropic_beta).toEqual(['output-128k-2025-02-19']);
expect(additionalFields.anthropic_beta).toEqual(BEDROCK_CLAUDE_4_BETAS);
});
test('should preserve effort when thinking=false for adaptive models', () => {

44
packages/data-provider/src/bedrock.ts
View file

@ -3,6 +3,8 @@ import * as s from './schemas';
const DEFAULT_ENABLED_MAX_TOKENS = 8192;
const DEFAULT_THINKING_BUDGET = 2000;
export const BEDROCK_OUTPUT_128K_BETA = 'output-128k-2025-02-19';
export const BEDROCK_FINE_GRAINED_TOOL_STREAMING_BETA = 'fine-grained-tool-streaming-2025-05-14';
const bedrockReasoningConfigValues = new Set<string>(Object.values(s.BedrockReasoningConfig));
@ -157,19 +159,47 @@ export function supportsContext1m(model: string): boolean {
function getBedrockAnthropicBetaHeaders(model: string): string[] {
const betaHeaders: string[] = [];
const isClaudeThinkingModel =
model.includes('anthropic.claude-3-7-sonnet') ||
const isClaude4PlusModel =
/anthropic\.claude-(?:[4-9](?:\.\d+)?(?:-\d+)?-(?:sonnet|opus|haiku)|(?:sonnet|opus|haiku)-[4-9])/.test(
model,
);
const isClaudeThinkingModel = model.includes('anthropic.claude-3-7-sonnet') || isClaude4PlusModel;
if (isClaudeThinkingModel) {
betaHeaders.push('output-128k-2025-02-19');
betaHeaders.push(BEDROCK_OUTPUT_128K_BETA);
}
if (isClaude4PlusModel) {
betaHeaders.push(BEDROCK_FINE_GRAINED_TOOL_STREAMING_BETA);
}
return betaHeaders;
}
function mergeBedrockAnthropicBetaHeaders(existing: unknown, generated: string[]): string[] {
const existingValues: unknown[] = Array.isArray(existing)
? existing
: typeof existing === 'string'
? [existing]
: [];
const betaHeaders = new Set<string>();
[...existingValues, ...generated].forEach((value) => {
if (typeof value !== 'string') {
return;
}
value
.split(',')
.map((header) => header.trim())
.filter(Boolean)
.forEach((header) => betaHeaders.add(header));
});
return Array.from(betaHeaders);
}
export const bedrockInputSchema = s.tConversationSchema
.pick({
/* LibreChat params; optionType: 'conversation' */
@ -359,7 +389,13 @@ export const bedrockInputParser = s.tConversationSchema
if ((typedData.model as string).includes('anthropic.')) {
const betaHeaders = getBedrockAnthropicBetaHeaders(typedData.model as string);
if (betaHeaders.length > 0) {
additionalFields.anthropic_beta = betaHeaders;
const existingBetaHeaders = (
typedData.additionalModelRequestFields as Record<string, unknown> | undefined
)?.anthropic_beta;
additionalFields.anthropic_beta = mergeBedrockAnthropicBetaHeaders(
existingBetaHeaders,
betaHeaders,
);
}
}
} else {