CERBERUS/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2026-07-11 08:43:48 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor: adjust token counting for Claude model to account for API discrepancies

Browse source 
Implemented a correction factor for token counting when using the Claude model, addressing discrepancies between Anthropic's API and local tokenizer results. This change ensures accurate token counts by applying a scaling factor, improving the reliability of token-related functionalities.
This commit is contained in:
Danny Avila 2026-03-13 21:33:38 -04:00
parent b87f9c7154
commit 3370be68d9
No known key found for this signature in database
GPG key ID: BF31EEB2C5CA0956
1 changed files with 10 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

11
packages/api/src/agents/client.ts
View file

@ -27,10 +27,19 @@ export function payloadParser({ req, endpoint }: { req: ServerRequest; endpoint:
return req.body?.endpointOption?.model_parameters;
}
/**
* Anthropic's API consistently reports ~10% more tokens than the local
* claude tokenizer due to internal message framing and content encoding.
* Verified empirically across content types via the count_tokens endpoint.
*/
const CLAUDE_TOKEN_CORRECTION = 1.1;
export function createTokenCounter(encoding: Parameters<typeof Tokenizer.getTokenCount>[1]) {
const isClaude = encoding === 'claude';
return function (message: BaseMessage) {
const countTokens = (text: string) => Tokenizer.getTokenCount(text, encoding);
return getTokenCountForMessage(message, countTokens);
const count = getTokenCountForMessage(message, countTokens);
return isClaude ? Math.ceil(count * CLAUDE_TOKEN_CORRECTION) : count;
};
}