mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-08-04 14:57:42 +00:00
🩹 fix: Bill Anthropic Prompt-Cache Tokens Once (#13798)
The installed @librechat/agents folds cache_creation + cache_read into Anthropic usage_metadata.input_tokens (cache-inclusive), but cacheSubsetProviders omitted anthropic, so splitUsage() took the additive branch and billed cache tokens twice — at the full input rate and again at the cache write/read rate. Verified live: a cache-read-heavy Sonnet call was overcharged 10.7x. Add Providers.ANTHROPIC to cacheSubsetProviders (single source of truth for backend billing and client usage normalization). Bedrock stays additive: its Converse path passes AWS inputTokens through unmodified. Update the Anthropic regression tests to production-accurate cache-inclusive fixtures. Fixes #13795
This commit is contained in:
Danny Avila
GitHub
parent
054fa4bfa7
commit
4cb35945dc
8 changed files with 115 additions and 45 deletions
|
|
@ -28,9 +28,9 @@ const inflatedSnapshot = (over?: Partial<TContextUsageEvent>): TContextUsageEven
|
|||
});
|
||||
|
||||
const primaryUsage = (over?: Partial<TTokenUsageEvent>): TTokenUsageEvent => ({
|
||||
input_tokens: 53702,
|
||||
input_tokens: 55773, // Anthropic input_tokens is cache-inclusive (53702 fresh + 2071 read)
|
||||
output_tokens: 3780,
|
||||
total_tokens: 57482,
|
||||
total_tokens: 59553,
|
||||
input_token_details: { cache_read: 2071, cache_creation: 0 },
|
||||
provider: 'anthropic',
|
||||
runId: 'run-1',
|
||||
|
|
|
|||
|
|
@ -282,9 +282,9 @@ describe('estimateTokens', () => {
|
|||
});
|
||||
|
||||
describe('normalizeUsageUnits', () => {
|
||||
it('keeps cache additive for Anthropic even when cache <= input', () => {
|
||||
/** Magnitude heuristic would wrongly treat this as inclusive and drop
|
||||
* cache from input; the provider says additive (input is uncached-only) */
|
||||
it('subtracts cache from input for Anthropic (input_tokens is cache-inclusive)', () => {
|
||||
/** The agents SDK folds cache into Anthropic input_tokens, so cache is a
|
||||
* subset of input and must be subtracted: 900 − 100 read = 800. */
|
||||
expect(
|
||||
normalizeUsageUnits({
|
||||
input_tokens: 900,
|
||||
|
|
@ -292,6 +292,17 @@ describe('normalizeUsageUnits', () => {
|
|||
provider: Providers.ANTHROPIC,
|
||||
input_token_details: { cache_read: 100 },
|
||||
}),
|
||||
).toEqual({ input: 800, output: 100, cacheWrite: 0, cacheRead: 100 });
|
||||
});
|
||||
|
||||
it('keeps cache additive for Bedrock (Converse input_tokens excludes cache)', () => {
|
||||
expect(
|
||||
normalizeUsageUnits({
|
||||
input_tokens: 900,
|
||||
output_tokens: 100,
|
||||
provider: Providers.BEDROCK,
|
||||
input_token_details: { cache_read: 100 },
|
||||
}),
|
||||
).toEqual({ input: 900, output: 100, cacheWrite: 0, cacheRead: 100 });
|
||||
});
|
||||
|
||||
|
|
@ -329,14 +340,28 @@ describe('normalizeUsageUnits', () => {
|
|||
).toBe(500);
|
||||
});
|
||||
|
||||
it('does not mistake additive cache for missing completion tokens', () => {
|
||||
/** Anthropic total includes cache; without the cache adjustment the repair
|
||||
* would falsely inflate completion */
|
||||
it('does not mistake additive cache for missing completion tokens (Bedrock)', () => {
|
||||
/** Bedrock keeps cache separate, so total = input + output + cache (1700).
|
||||
* Without the cache adjustment the repair would falsely inflate completion. */
|
||||
expect(
|
||||
normalizeUsageUnits({
|
||||
input_tokens: 1000,
|
||||
output_tokens: 200,
|
||||
total_tokens: 1700,
|
||||
provider: Providers.BEDROCK,
|
||||
input_token_details: { cache_creation: 300, cache_read: 200 },
|
||||
}).output,
|
||||
).toBe(200);
|
||||
});
|
||||
|
||||
it('does not inflate completion for cache-inclusive Anthropic totals', () => {
|
||||
/** Anthropic total already includes cache (input 1000 covers the 500 cache),
|
||||
* so total 1200 leaves output at 200 — no false repair. */
|
||||
expect(
|
||||
normalizeUsageUnits({
|
||||
input_tokens: 1000,
|
||||
output_tokens: 200,
|
||||
total_tokens: 1200,
|
||||
provider: Providers.ANTHROPIC,
|
||||
input_token_details: { cache_creation: 300, cache_read: 200 },
|
||||
}).output,
|
||||
|
|
|
|||
|
|
@ -223,12 +223,17 @@ describe('recordCollectedUsage — bulk path parity', () => {
|
|||
});
|
||||
|
||||
describe('cache token handling - Anthropic format', () => {
|
||||
it('should route Anthropic cache entries to structured path — same input_tokens as legacy', async () => {
|
||||
it('routes Anthropic cache entries to structured path and subtracts cache from input', async () => {
|
||||
/** Production Anthropic wire: input_tokens is cache-inclusive (140 =
|
||||
* 100 fresh + 25 write + 15 read) and provider is tagged. The billed
|
||||
* input must be the fresh 100, not 140 — otherwise cache is charged
|
||||
* twice (full input rate + cache rate). Regression for LibreChat#13795. */
|
||||
const collectedUsage: UsageMetadata[] = [
|
||||
{
|
||||
input_tokens: 100,
|
||||
input_tokens: 140,
|
||||
output_tokens: 50,
|
||||
model: 'claude-3',
|
||||
provider: 'anthropic',
|
||||
cache_creation_input_tokens: 25,
|
||||
cache_read_input_tokens: 15,
|
||||
},
|
||||
|
|
@ -236,7 +241,7 @@ describe('recordCollectedUsage — bulk path parity', () => {
|
|||
|
||||
const result = await recordCollectedUsage(deps, { ...baseParams, collectedUsage });
|
||||
|
||||
expect(result?.input_tokens).toBe(140); // 100 + 25 + 15
|
||||
expect(result?.input_tokens).toBe(140);
|
||||
expect(mockInsertMany).toHaveBeenCalledTimes(1);
|
||||
expect(mockSpendStructuredTokens).not.toHaveBeenCalled();
|
||||
|
||||
|
|
@ -339,34 +344,48 @@ describe('recordCollectedUsage — bulk path parity', () => {
|
|||
});
|
||||
|
||||
it('should handle cache tokens with multiple tool calls — same totals as legacy', async () => {
|
||||
/** Cache-inclusive Anthropic wire (input_tokens = fresh + write + read).
|
||||
* The billed input on each call is the fresh portion only. */
|
||||
const collectedUsage: UsageMetadata[] = [
|
||||
{
|
||||
input_tokens: 788,
|
||||
input_tokens: 31596, // 788 fresh + 30808 write
|
||||
output_tokens: 163,
|
||||
model: 'claude-opus',
|
||||
provider: 'anthropic',
|
||||
input_token_details: { cache_read: 0, cache_creation: 30808 },
|
||||
},
|
||||
{
|
||||
input_tokens: 3802,
|
||||
input_tokens: 35378, // 3802 fresh + 30808 read + 768 write
|
||||
output_tokens: 149,
|
||||
model: 'claude-opus',
|
||||
provider: 'anthropic',
|
||||
input_token_details: { cache_read: 30808, cache_creation: 768 },
|
||||
},
|
||||
{
|
||||
input_tokens: 26808,
|
||||
input_tokens: 58384, // 26808 fresh + 31576 read
|
||||
output_tokens: 225,
|
||||
model: 'claude-opus',
|
||||
provider: 'anthropic',
|
||||
input_token_details: { cache_read: 31576, cache_creation: 0 },
|
||||
},
|
||||
];
|
||||
|
||||
const result = await recordCollectedUsage(deps, { ...baseParams, collectedUsage });
|
||||
|
||||
expect(result?.input_tokens).toBe(31596); // 788 + 30808 + 0
|
||||
expect(result?.input_tokens).toBe(31596); // total prompt of first call
|
||||
expect(result?.output_tokens).toBe(537); // 163 + 149 + 225
|
||||
expect(mockInsertMany).toHaveBeenCalledTimes(1);
|
||||
expect(mockSpendStructuredTokens).not.toHaveBeenCalled();
|
||||
expect(mockSpendTokens).not.toHaveBeenCalled();
|
||||
|
||||
/** Each prompt doc bills only the fresh portion — cache is never folded
|
||||
* into inputTokens on top of its own write/read charge. */
|
||||
const promptDocs = mockInsertMany.mock.calls[0][0].filter(
|
||||
(d: { tokenType: string }) => d.tokenType === 'prompt',
|
||||
);
|
||||
expect(promptDocs.map((d: { inputTokens: number }) => d.inputTokens)).toEqual([
|
||||
-788, -3802, -26808,
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -278,13 +278,15 @@ describe('recordCollectedUsage', () => {
|
|||
});
|
||||
|
||||
expect(mockSpendStructuredTokens).toHaveBeenCalledTimes(1);
|
||||
/** Anthropic input_tokens is cache-inclusive, so cache is subtracted out
|
||||
* of the billed input: 200 − 60 − 30 = 110 (not double-charged). */
|
||||
expect(mockSpendStructuredTokens).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
context: 'subagent',
|
||||
model: 'claude-haiku-4-5',
|
||||
}),
|
||||
{
|
||||
promptTokens: { input: 200, write: 60, read: 30 },
|
||||
promptTokens: { input: 110, write: 60, read: 30 },
|
||||
completionTokens: 80,
|
||||
},
|
||||
);
|
||||
|
|
@ -479,11 +481,13 @@ describe('recordCollectedUsage', () => {
|
|||
|
||||
describe('cache token handling - Anthropic format', () => {
|
||||
it('should use spendStructuredTokens for cache tokens (cache_*_input_tokens)', async () => {
|
||||
/** Cache-inclusive input_tokens (140 = 100 fresh + 25 write + 15 read). */
|
||||
const collectedUsage: UsageMetadata[] = [
|
||||
{
|
||||
input_tokens: 100,
|
||||
input_tokens: 140,
|
||||
output_tokens: 50,
|
||||
model: 'claude-3',
|
||||
provider: 'anthropic',
|
||||
cache_creation_input_tokens: 25,
|
||||
cache_read_input_tokens: 15,
|
||||
},
|
||||
|
|
@ -503,7 +507,7 @@ describe('recordCollectedUsage', () => {
|
|||
completionTokens: 50,
|
||||
},
|
||||
);
|
||||
expect(result?.input_tokens).toBe(140); // 100 + 25 + 15
|
||||
expect(result?.input_tokens).toBe(140);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -708,23 +712,27 @@ describe('recordCollectedUsage', () => {
|
|||
});
|
||||
|
||||
it('should handle cache tokens with multiple tool calls', async () => {
|
||||
/** Cache-inclusive Anthropic wire: input_tokens = fresh + write + read. */
|
||||
const collectedUsage: UsageMetadata[] = [
|
||||
{
|
||||
input_tokens: 788,
|
||||
input_tokens: 31596, // 788 fresh + 30808 write
|
||||
output_tokens: 163,
|
||||
model: 'claude-opus',
|
||||
provider: 'anthropic',
|
||||
input_token_details: { cache_read: 0, cache_creation: 30808 },
|
||||
},
|
||||
{
|
||||
input_tokens: 3802,
|
||||
input_tokens: 35378, // 3802 fresh + 30808 read + 768 write
|
||||
output_tokens: 149,
|
||||
model: 'claude-opus',
|
||||
provider: 'anthropic',
|
||||
input_token_details: { cache_read: 30808, cache_creation: 768 },
|
||||
},
|
||||
{
|
||||
input_tokens: 26808,
|
||||
input_tokens: 58384, // 26808 fresh + 31576 read
|
||||
output_tokens: 225,
|
||||
model: 'claude-opus',
|
||||
provider: 'anthropic',
|
||||
input_token_details: { cache_read: 31576, cache_creation: 0 },
|
||||
},
|
||||
];
|
||||
|
|
@ -734,7 +742,7 @@ describe('recordCollectedUsage', () => {
|
|||
collectedUsage,
|
||||
});
|
||||
|
||||
// input_tokens = 788 + 30808 + 0 = 31596
|
||||
// input_tokens = total prompt of first call (cache-inclusive)
|
||||
expect(result?.input_tokens).toBe(31596);
|
||||
// output_tokens = 163 + 149 + 225 = 537
|
||||
expect(result?.output_tokens).toBe(537);
|
||||
|
|
@ -1585,10 +1593,13 @@ describe('computeUsageCostUSD', () => {
|
|||
expect(cost).toBeCloseTo((300000 * 8 + 1000 * 40) / 1e6);
|
||||
});
|
||||
|
||||
it('prices additive cache tokens (Anthropic) at cache rates', () => {
|
||||
it('prices cache-inclusive Anthropic input by subtracting cache, then cache at its own rates', () => {
|
||||
/** input_tokens is cache-inclusive (13000 = 1000 fresh + 2000 write + 10000
|
||||
* read). The fresh 1000 is billed at the input rate, the cache at its own
|
||||
* rates — never the cache portion at the input rate too (LibreChat#13795). */
|
||||
const cost = computeUsageCostUSD(
|
||||
{
|
||||
input_tokens: 1000,
|
||||
input_tokens: 13000,
|
||||
output_tokens: 500,
|
||||
model: 'claude-haiku-4-5',
|
||||
provider: 'anthropic',
|
||||
|
|
@ -1656,12 +1667,12 @@ describe('aggregateEmittedUsage', () => {
|
|||
});
|
||||
|
||||
it('normalizes mixed-provider calls per their own provider before summing', () => {
|
||||
/** anthropic is additive (cache separate from input), openAI is subset */
|
||||
/** bedrock is additive (cache separate from input), openAI is subset */
|
||||
const rollup = aggregateEmittedUsage([
|
||||
{
|
||||
input_tokens: 100,
|
||||
output_tokens: 20,
|
||||
provider: 'anthropic',
|
||||
provider: 'bedrock',
|
||||
input_token_details: { cache_read: 40 },
|
||||
cost: 0.01,
|
||||
},
|
||||
|
|
@ -1674,7 +1685,7 @@ describe('aggregateEmittedUsage', () => {
|
|||
cost: 0.02,
|
||||
},
|
||||
]);
|
||||
/** anthropic input stays 100 (additive); openAI input 90−30=60 → 160 */
|
||||
/** bedrock input stays 100 (additive); openAI input 90−30=60 → 160 */
|
||||
expect(rollup?.input).toBe(160);
|
||||
expect(rollup?.output).toBe(25);
|
||||
expect(rollup?.cacheRead).toBe(70);
|
||||
|
|
@ -1758,8 +1769,9 @@ describe('buildPersistedContextUsage', () => {
|
|||
it('reconciles the inflated estimate to the final call’s real prompt tokens', () => {
|
||||
/** Real web-search + summarization turn: calibration pinned at 5 inflated
|
||||
* messageTokens to 187471 (used 213375), but the answer call's true prompt was
|
||||
* 53702 + 2071 cache = 55773. The persisted blob must show the real context so
|
||||
* a reload isn't stuck several× too high. */
|
||||
* 55773 (Anthropic input_tokens is cache-inclusive; 2071 of it is cache read).
|
||||
* The persisted blob must show the real context so a reload isn't stuck
|
||||
* several× too high. */
|
||||
const inflated: TContextUsageEvent = {
|
||||
runId: 'run-1',
|
||||
breakdown: {
|
||||
|
|
@ -1780,7 +1792,7 @@ describe('buildPersistedContextUsage', () => {
|
|||
};
|
||||
const events: TTokenUsageEvent[] = [
|
||||
{
|
||||
input_tokens: 53702,
|
||||
input_tokens: 55773, // cache-inclusive: 53702 fresh + 2071 read
|
||||
output_tokens: 3780,
|
||||
input_token_details: { cache_read: 2071, cache_creation: 0 },
|
||||
provider: 'anthropic',
|
||||
|
|
|
|||
|
|
@ -47,22 +47,23 @@ type SpendStructuredTokensFn = (
|
|||
* path emits `output_tokens = candidatesTokenCount` and drops `thoughtsTokenCount`,
|
||||
* so `total - input > output`. The gap is recovered as `total - input`.
|
||||
*
|
||||
* **Bedrock / Anthropic cache inflation:** additive providers keep cache tokens
|
||||
* **Bedrock cache inflation:** additive providers keep cache tokens
|
||||
* separate from `input_tokens`, making
|
||||
* `total = input + output + cache_read + cache_creation`. Without adjustment
|
||||
* the Vertex recovery fires on every cached step and returns
|
||||
* `output + cache_read + cache_creation` instead of `output`, inflating
|
||||
* completion counts by orders of magnitude. The fix subtracts the cache
|
||||
* adjustment before the gap test — but only for additive providers; subset
|
||||
* providers (Google, OpenAI, …) already include cache inside `input_tokens`
|
||||
* so their `cacheAdjustment` is zero and the Vertex recovery is unaffected.
|
||||
* providers (Anthropic, Google, OpenAI, …) already include cache inside
|
||||
* `input_tokens` so their `cacheAdjustment` is zero and the Vertex recovery
|
||||
* is unaffected.
|
||||
*/
|
||||
function resolveCompletionTokens(usage: UsageMetadata): number {
|
||||
const output = Number(usage.output_tokens) || 0;
|
||||
const total = Number(usage.total_tokens) || 0;
|
||||
const input = Number(usage.input_tokens) || 0;
|
||||
|
||||
// For additive providers (Bedrock, Anthropic), cache tokens are separate
|
||||
// For additive providers (Bedrock), cache tokens are separate
|
||||
// from input_tokens and are included in total_tokens, widening the gap
|
||||
// independently of any missing thinking tokens. Subtract them so the gap
|
||||
// check only fires when output_tokens genuinely undercounts (Vertex case).
|
||||
|
|
|
|||
|
|
@ -83,8 +83,11 @@ export const isOpenAILikeProvider = (provider?: string | null): boolean => {
|
|||
* Providers whose `usage_metadata.input_tokens` ALREADY INCLUDES cached tokens
|
||||
* (`input_token_details.cache_*` is a subset, not an additional charge):
|
||||
* Google/Vertex (`promptTokenCount`), OpenAI/Azure (`prompt_tokens`), and the
|
||||
* OpenAI-compatible family. Anthropic/Bedrock keep cache values separate and
|
||||
* additive. Single source of truth shared by the backend billing path
|
||||
* OpenAI-compatible family. `@librechat/agents`' `getAnthropicUsageMetadata`
|
||||
* folds `cache_creation` + `cache_read` into `input_tokens`, so Anthropic is a
|
||||
* subset provider too; without this the cache portion is billed twice. Bedrock
|
||||
* stays additive — its Converse path passes AWS `inputTokens` through unmodified.
|
||||
* Single source of truth shared by the backend billing path
|
||||
* (`packages/api/src/agents/usage.ts`) and the client usage normalization.
|
||||
*/
|
||||
export const cacheSubsetProviders = new Set<string>([
|
||||
|
|
@ -96,6 +99,7 @@ export const cacheSubsetProviders = new Set<string>([
|
|||
Providers.DEEPSEEK,
|
||||
Providers.OPENROUTER,
|
||||
Providers.MOONSHOT,
|
||||
Providers.ANTHROPIC,
|
||||
]);
|
||||
|
||||
export const inputTokensIncludesCache = (provider?: string | null): boolean => {
|
||||
|
|
|
|||
|
|
@ -2,10 +2,19 @@ import type { TContextUsageEvent, TTokenUsageEvent } from './runs';
|
|||
import { promptTokensFromUsage, reconcileContextUsage } from './runs';
|
||||
|
||||
describe('promptTokensFromUsage', () => {
|
||||
it('adds cache reads/writes for additive providers (Anthropic)', () => {
|
||||
it('adds cache reads/writes for additive providers (Bedrock)', () => {
|
||||
const event: TTokenUsageEvent = {
|
||||
input_tokens: 53702,
|
||||
input_token_details: { cache_read: 2071, cache_creation: 0 },
|
||||
provider: 'bedrock',
|
||||
};
|
||||
expect(promptTokensFromUsage(event)).toBe(55773);
|
||||
});
|
||||
|
||||
it('treats input_tokens as the full prompt for Anthropic (cache-inclusive)', () => {
|
||||
const event: TTokenUsageEvent = {
|
||||
input_tokens: 55773,
|
||||
input_token_details: { cache_read: 2071, cache_creation: 0 },
|
||||
provider: 'anthropic',
|
||||
};
|
||||
expect(promptTokensFromUsage(event)).toBe(55773);
|
||||
|
|
@ -101,7 +110,7 @@ describe('reconcileContextUsage', () => {
|
|||
remainingContextTokens: 202815,
|
||||
};
|
||||
const usage: TTokenUsageEvent = {
|
||||
input_tokens: 7804,
|
||||
input_tokens: 9875, // cache-inclusive (Anthropic): 7804 fresh + 2071 read
|
||||
input_token_details: { cache_read: 2071, cache_creation: 0 },
|
||||
provider: 'anthropic',
|
||||
};
|
||||
|
|
|
|||
|
|
@ -130,13 +130,13 @@ export type TTokenUsageEvent = {
|
|||
|
||||
/**
|
||||
* Full prompt token count for one completed model call — the EXACT context the
|
||||
* model saw, provider-aware: additive providers (Anthropic/Bedrock) report
|
||||
* `input_tokens` excluding cache, so cache reads/writes are added back; subset
|
||||
* providers (OpenAI/…) already fold cache into `input_tokens`. When the provider
|
||||
* is absent (custom/OpenAI-compatible payloads), fall back to the same magnitude
|
||||
* heuristic `normalizeUsageUnits` uses — cache ≤ input means it's already
|
||||
* included — so cached events aren't re-inflated. The ground truth the gauge
|
||||
* reconciles its calibrated estimate to.
|
||||
* model saw, provider-aware: additive providers (Bedrock) report `input_tokens`
|
||||
* excluding cache, so cache reads/writes are added back; subset providers
|
||||
* (Anthropic, OpenAI, …) already fold cache into `input_tokens`. When the
|
||||
* provider is absent (custom/OpenAI-compatible payloads), fall back to the same
|
||||
* magnitude heuristic `normalizeUsageUnits` uses — cache ≤ input means it's
|
||||
* already included — so cached events aren't re-inflated. The ground truth the
|
||||
* gauge reconciles its calibrated estimate to.
|
||||
*/
|
||||
export const promptTokensFromUsage = (event: TTokenUsageEvent): number => {
|
||||
const input = event.input_tokens ?? 0;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue