diff --git a/client/src/hooks/SSE/__tests__/useUsageHandler.spec.tsx b/client/src/hooks/SSE/__tests__/useUsageHandler.spec.tsx index 59f3bf2b85..41ba9e3511 100644 --- a/client/src/hooks/SSE/__tests__/useUsageHandler.spec.tsx +++ b/client/src/hooks/SSE/__tests__/useUsageHandler.spec.tsx @@ -28,9 +28,9 @@ const inflatedSnapshot = (over?: Partial): TContextUsageEven }); const primaryUsage = (over?: Partial): TTokenUsageEvent => ({ - input_tokens: 53702, + input_tokens: 55773, // Anthropic input_tokens is cache-inclusive (53702 fresh + 2071 read) output_tokens: 3780, - total_tokens: 57482, + total_tokens: 59553, input_token_details: { cache_read: 2071, cache_creation: 0 }, provider: 'anthropic', runId: 'run-1', diff --git a/client/src/utils/tokens.spec.ts b/client/src/utils/tokens.spec.ts index 259eea56e4..725fdab224 100644 --- a/client/src/utils/tokens.spec.ts +++ b/client/src/utils/tokens.spec.ts @@ -282,9 +282,9 @@ describe('estimateTokens', () => { }); describe('normalizeUsageUnits', () => { - it('keeps cache additive for Anthropic even when cache <= input', () => { - /** Magnitude heuristic would wrongly treat this as inclusive and drop - * cache from input; the provider says additive (input is uncached-only) */ + it('subtracts cache from input for Anthropic (input_tokens is cache-inclusive)', () => { + /** The agents SDK folds cache into Anthropic input_tokens, so cache is a + * subset of input and must be subtracted: 900 − 100 read = 800. */ expect( normalizeUsageUnits({ input_tokens: 900, @@ -292,6 +292,17 @@ describe('normalizeUsageUnits', () => { provider: Providers.ANTHROPIC, input_token_details: { cache_read: 100 }, }), + ).toEqual({ input: 800, output: 100, cacheWrite: 0, cacheRead: 100 }); + }); + + it('keeps cache additive for Bedrock (Converse input_tokens excludes cache)', () => { + expect( + normalizeUsageUnits({ + input_tokens: 900, + output_tokens: 100, + provider: Providers.BEDROCK, + input_token_details: { cache_read: 100 }, + }), ).toEqual({ input: 900, output: 100, cacheWrite: 0, cacheRead: 100 }); }); @@ -329,14 +340,28 @@ describe('normalizeUsageUnits', () => { ).toBe(500); }); - it('does not mistake additive cache for missing completion tokens', () => { - /** Anthropic total includes cache; without the cache adjustment the repair - * would falsely inflate completion */ + it('does not mistake additive cache for missing completion tokens (Bedrock)', () => { + /** Bedrock keeps cache separate, so total = input + output + cache (1700). + * Without the cache adjustment the repair would falsely inflate completion. */ expect( normalizeUsageUnits({ input_tokens: 1000, output_tokens: 200, total_tokens: 1700, + provider: Providers.BEDROCK, + input_token_details: { cache_creation: 300, cache_read: 200 }, + }).output, + ).toBe(200); + }); + + it('does not inflate completion for cache-inclusive Anthropic totals', () => { + /** Anthropic total already includes cache (input 1000 covers the 500 cache), + * so total 1200 leaves output at 200 — no false repair. */ + expect( + normalizeUsageUnits({ + input_tokens: 1000, + output_tokens: 200, + total_tokens: 1200, provider: Providers.ANTHROPIC, input_token_details: { cache_creation: 300, cache_read: 200 }, }).output, diff --git a/packages/api/src/agents/usage.bulk-parity.spec.ts b/packages/api/src/agents/usage.bulk-parity.spec.ts index a18565ba0d..5520b66794 100644 --- a/packages/api/src/agents/usage.bulk-parity.spec.ts +++ b/packages/api/src/agents/usage.bulk-parity.spec.ts @@ -223,12 +223,17 @@ describe('recordCollectedUsage — bulk path parity', () => { }); describe('cache token handling - Anthropic format', () => { - it('should route Anthropic cache entries to structured path — same input_tokens as legacy', async () => { + it('routes Anthropic cache entries to structured path and subtracts cache from input', async () => { + /** Production Anthropic wire: input_tokens is cache-inclusive (140 = + * 100 fresh + 25 write + 15 read) and provider is tagged. The billed + * input must be the fresh 100, not 140 — otherwise cache is charged + * twice (full input rate + cache rate). Regression for LibreChat#13795. */ const collectedUsage: UsageMetadata[] = [ { - input_tokens: 100, + input_tokens: 140, output_tokens: 50, model: 'claude-3', + provider: 'anthropic', cache_creation_input_tokens: 25, cache_read_input_tokens: 15, }, @@ -236,7 +241,7 @@ describe('recordCollectedUsage — bulk path parity', () => { const result = await recordCollectedUsage(deps, { ...baseParams, collectedUsage }); - expect(result?.input_tokens).toBe(140); // 100 + 25 + 15 + expect(result?.input_tokens).toBe(140); expect(mockInsertMany).toHaveBeenCalledTimes(1); expect(mockSpendStructuredTokens).not.toHaveBeenCalled(); @@ -339,34 +344,48 @@ describe('recordCollectedUsage — bulk path parity', () => { }); it('should handle cache tokens with multiple tool calls — same totals as legacy', async () => { + /** Cache-inclusive Anthropic wire (input_tokens = fresh + write + read). + * The billed input on each call is the fresh portion only. */ const collectedUsage: UsageMetadata[] = [ { - input_tokens: 788, + input_tokens: 31596, // 788 fresh + 30808 write output_tokens: 163, model: 'claude-opus', + provider: 'anthropic', input_token_details: { cache_read: 0, cache_creation: 30808 }, }, { - input_tokens: 3802, + input_tokens: 35378, // 3802 fresh + 30808 read + 768 write output_tokens: 149, model: 'claude-opus', + provider: 'anthropic', input_token_details: { cache_read: 30808, cache_creation: 768 }, }, { - input_tokens: 26808, + input_tokens: 58384, // 26808 fresh + 31576 read output_tokens: 225, model: 'claude-opus', + provider: 'anthropic', input_token_details: { cache_read: 31576, cache_creation: 0 }, }, ]; const result = await recordCollectedUsage(deps, { ...baseParams, collectedUsage }); - expect(result?.input_tokens).toBe(31596); // 788 + 30808 + 0 + expect(result?.input_tokens).toBe(31596); // total prompt of first call expect(result?.output_tokens).toBe(537); // 163 + 149 + 225 expect(mockInsertMany).toHaveBeenCalledTimes(1); expect(mockSpendStructuredTokens).not.toHaveBeenCalled(); expect(mockSpendTokens).not.toHaveBeenCalled(); + + /** Each prompt doc bills only the fresh portion — cache is never folded + * into inputTokens on top of its own write/read charge. */ + const promptDocs = mockInsertMany.mock.calls[0][0].filter( + (d: { tokenType: string }) => d.tokenType === 'prompt', + ); + expect(promptDocs.map((d: { inputTokens: number }) => d.inputTokens)).toEqual([ + -788, -3802, -26808, + ]); }); }); diff --git a/packages/api/src/agents/usage.spec.ts b/packages/api/src/agents/usage.spec.ts index 6c293f7768..a9a3315146 100644 --- a/packages/api/src/agents/usage.spec.ts +++ b/packages/api/src/agents/usage.spec.ts @@ -278,13 +278,15 @@ describe('recordCollectedUsage', () => { }); expect(mockSpendStructuredTokens).toHaveBeenCalledTimes(1); + /** Anthropic input_tokens is cache-inclusive, so cache is subtracted out + * of the billed input: 200 − 60 − 30 = 110 (not double-charged). */ expect(mockSpendStructuredTokens).toHaveBeenCalledWith( expect.objectContaining({ context: 'subagent', model: 'claude-haiku-4-5', }), { - promptTokens: { input: 200, write: 60, read: 30 }, + promptTokens: { input: 110, write: 60, read: 30 }, completionTokens: 80, }, ); @@ -479,11 +481,13 @@ describe('recordCollectedUsage', () => { describe('cache token handling - Anthropic format', () => { it('should use spendStructuredTokens for cache tokens (cache_*_input_tokens)', async () => { + /** Cache-inclusive input_tokens (140 = 100 fresh + 25 write + 15 read). */ const collectedUsage: UsageMetadata[] = [ { - input_tokens: 100, + input_tokens: 140, output_tokens: 50, model: 'claude-3', + provider: 'anthropic', cache_creation_input_tokens: 25, cache_read_input_tokens: 15, }, @@ -503,7 +507,7 @@ describe('recordCollectedUsage', () => { completionTokens: 50, }, ); - expect(result?.input_tokens).toBe(140); // 100 + 25 + 15 + expect(result?.input_tokens).toBe(140); }); }); @@ -708,23 +712,27 @@ describe('recordCollectedUsage', () => { }); it('should handle cache tokens with multiple tool calls', async () => { + /** Cache-inclusive Anthropic wire: input_tokens = fresh + write + read. */ const collectedUsage: UsageMetadata[] = [ { - input_tokens: 788, + input_tokens: 31596, // 788 fresh + 30808 write output_tokens: 163, model: 'claude-opus', + provider: 'anthropic', input_token_details: { cache_read: 0, cache_creation: 30808 }, }, { - input_tokens: 3802, + input_tokens: 35378, // 3802 fresh + 30808 read + 768 write output_tokens: 149, model: 'claude-opus', + provider: 'anthropic', input_token_details: { cache_read: 30808, cache_creation: 768 }, }, { - input_tokens: 26808, + input_tokens: 58384, // 26808 fresh + 31576 read output_tokens: 225, model: 'claude-opus', + provider: 'anthropic', input_token_details: { cache_read: 31576, cache_creation: 0 }, }, ]; @@ -734,7 +742,7 @@ describe('recordCollectedUsage', () => { collectedUsage, }); - // input_tokens = 788 + 30808 + 0 = 31596 + // input_tokens = total prompt of first call (cache-inclusive) expect(result?.input_tokens).toBe(31596); // output_tokens = 163 + 149 + 225 = 537 expect(result?.output_tokens).toBe(537); @@ -1585,10 +1593,13 @@ describe('computeUsageCostUSD', () => { expect(cost).toBeCloseTo((300000 * 8 + 1000 * 40) / 1e6); }); - it('prices additive cache tokens (Anthropic) at cache rates', () => { + it('prices cache-inclusive Anthropic input by subtracting cache, then cache at its own rates', () => { + /** input_tokens is cache-inclusive (13000 = 1000 fresh + 2000 write + 10000 + * read). The fresh 1000 is billed at the input rate, the cache at its own + * rates — never the cache portion at the input rate too (LibreChat#13795). */ const cost = computeUsageCostUSD( { - input_tokens: 1000, + input_tokens: 13000, output_tokens: 500, model: 'claude-haiku-4-5', provider: 'anthropic', @@ -1656,12 +1667,12 @@ describe('aggregateEmittedUsage', () => { }); it('normalizes mixed-provider calls per their own provider before summing', () => { - /** anthropic is additive (cache separate from input), openAI is subset */ + /** bedrock is additive (cache separate from input), openAI is subset */ const rollup = aggregateEmittedUsage([ { input_tokens: 100, output_tokens: 20, - provider: 'anthropic', + provider: 'bedrock', input_token_details: { cache_read: 40 }, cost: 0.01, }, @@ -1674,7 +1685,7 @@ describe('aggregateEmittedUsage', () => { cost: 0.02, }, ]); - /** anthropic input stays 100 (additive); openAI input 90−30=60 → 160 */ + /** bedrock input stays 100 (additive); openAI input 90−30=60 → 160 */ expect(rollup?.input).toBe(160); expect(rollup?.output).toBe(25); expect(rollup?.cacheRead).toBe(70); @@ -1758,8 +1769,9 @@ describe('buildPersistedContextUsage', () => { it('reconciles the inflated estimate to the final call’s real prompt tokens', () => { /** Real web-search + summarization turn: calibration pinned at 5 inflated * messageTokens to 187471 (used 213375), but the answer call's true prompt was - * 53702 + 2071 cache = 55773. The persisted blob must show the real context so - * a reload isn't stuck several× too high. */ + * 55773 (Anthropic input_tokens is cache-inclusive; 2071 of it is cache read). + * The persisted blob must show the real context so a reload isn't stuck + * several× too high. */ const inflated: TContextUsageEvent = { runId: 'run-1', breakdown: { @@ -1780,7 +1792,7 @@ describe('buildPersistedContextUsage', () => { }; const events: TTokenUsageEvent[] = [ { - input_tokens: 53702, + input_tokens: 55773, // cache-inclusive: 53702 fresh + 2071 read output_tokens: 3780, input_token_details: { cache_read: 2071, cache_creation: 0 }, provider: 'anthropic', diff --git a/packages/api/src/agents/usage.ts b/packages/api/src/agents/usage.ts index 31e8498867..6e6b1c14f3 100644 --- a/packages/api/src/agents/usage.ts +++ b/packages/api/src/agents/usage.ts @@ -47,22 +47,23 @@ type SpendStructuredTokensFn = ( * path emits `output_tokens = candidatesTokenCount` and drops `thoughtsTokenCount`, * so `total - input > output`. The gap is recovered as `total - input`. * - * **Bedrock / Anthropic cache inflation:** additive providers keep cache tokens + * **Bedrock cache inflation:** additive providers keep cache tokens * separate from `input_tokens`, making * `total = input + output + cache_read + cache_creation`. Without adjustment * the Vertex recovery fires on every cached step and returns * `output + cache_read + cache_creation` instead of `output`, inflating * completion counts by orders of magnitude. The fix subtracts the cache * adjustment before the gap test — but only for additive providers; subset - * providers (Google, OpenAI, …) already include cache inside `input_tokens` - * so their `cacheAdjustment` is zero and the Vertex recovery is unaffected. + * providers (Anthropic, Google, OpenAI, …) already include cache inside + * `input_tokens` so their `cacheAdjustment` is zero and the Vertex recovery + * is unaffected. */ function resolveCompletionTokens(usage: UsageMetadata): number { const output = Number(usage.output_tokens) || 0; const total = Number(usage.total_tokens) || 0; const input = Number(usage.input_tokens) || 0; - // For additive providers (Bedrock, Anthropic), cache tokens are separate + // For additive providers (Bedrock), cache tokens are separate // from input_tokens and are included in total_tokens, widening the gap // independently of any missing thinking tokens. Subtract them so the gap // check only fires when output_tokens genuinely undercounts (Vertex case). diff --git a/packages/data-provider/src/schemas.ts b/packages/data-provider/src/schemas.ts index e7b1186a0c..382199c97b 100644 --- a/packages/data-provider/src/schemas.ts +++ b/packages/data-provider/src/schemas.ts @@ -83,8 +83,11 @@ export const isOpenAILikeProvider = (provider?: string | null): boolean => { * Providers whose `usage_metadata.input_tokens` ALREADY INCLUDES cached tokens * (`input_token_details.cache_*` is a subset, not an additional charge): * Google/Vertex (`promptTokenCount`), OpenAI/Azure (`prompt_tokens`), and the - * OpenAI-compatible family. Anthropic/Bedrock keep cache values separate and - * additive. Single source of truth shared by the backend billing path + * OpenAI-compatible family. `@librechat/agents`' `getAnthropicUsageMetadata` + * folds `cache_creation` + `cache_read` into `input_tokens`, so Anthropic is a + * subset provider too; without this the cache portion is billed twice. Bedrock + * stays additive — its Converse path passes AWS `inputTokens` through unmodified. + * Single source of truth shared by the backend billing path * (`packages/api/src/agents/usage.ts`) and the client usage normalization. */ export const cacheSubsetProviders = new Set([ @@ -96,6 +99,7 @@ export const cacheSubsetProviders = new Set([ Providers.DEEPSEEK, Providers.OPENROUTER, Providers.MOONSHOT, + Providers.ANTHROPIC, ]); export const inputTokensIncludesCache = (provider?: string | null): boolean => { diff --git a/packages/data-provider/src/types/runs.spec.ts b/packages/data-provider/src/types/runs.spec.ts index 28593d95f3..f4058d1c7b 100644 --- a/packages/data-provider/src/types/runs.spec.ts +++ b/packages/data-provider/src/types/runs.spec.ts @@ -2,10 +2,19 @@ import type { TContextUsageEvent, TTokenUsageEvent } from './runs'; import { promptTokensFromUsage, reconcileContextUsage } from './runs'; describe('promptTokensFromUsage', () => { - it('adds cache reads/writes for additive providers (Anthropic)', () => { + it('adds cache reads/writes for additive providers (Bedrock)', () => { const event: TTokenUsageEvent = { input_tokens: 53702, input_token_details: { cache_read: 2071, cache_creation: 0 }, + provider: 'bedrock', + }; + expect(promptTokensFromUsage(event)).toBe(55773); + }); + + it('treats input_tokens as the full prompt for Anthropic (cache-inclusive)', () => { + const event: TTokenUsageEvent = { + input_tokens: 55773, + input_token_details: { cache_read: 2071, cache_creation: 0 }, provider: 'anthropic', }; expect(promptTokensFromUsage(event)).toBe(55773); @@ -101,7 +110,7 @@ describe('reconcileContextUsage', () => { remainingContextTokens: 202815, }; const usage: TTokenUsageEvent = { - input_tokens: 7804, + input_tokens: 9875, // cache-inclusive (Anthropic): 7804 fresh + 2071 read input_token_details: { cache_read: 2071, cache_creation: 0 }, provider: 'anthropic', }; diff --git a/packages/data-provider/src/types/runs.ts b/packages/data-provider/src/types/runs.ts index 280cd11cb2..1fdc07cd81 100644 --- a/packages/data-provider/src/types/runs.ts +++ b/packages/data-provider/src/types/runs.ts @@ -130,13 +130,13 @@ export type TTokenUsageEvent = { /** * Full prompt token count for one completed model call — the EXACT context the - * model saw, provider-aware: additive providers (Anthropic/Bedrock) report - * `input_tokens` excluding cache, so cache reads/writes are added back; subset - * providers (OpenAI/…) already fold cache into `input_tokens`. When the provider - * is absent (custom/OpenAI-compatible payloads), fall back to the same magnitude - * heuristic `normalizeUsageUnits` uses — cache ≤ input means it's already - * included — so cached events aren't re-inflated. The ground truth the gauge - * reconciles its calibrated estimate to. + * model saw, provider-aware: additive providers (Bedrock) report `input_tokens` + * excluding cache, so cache reads/writes are added back; subset providers + * (Anthropic, OpenAI, …) already fold cache into `input_tokens`. When the + * provider is absent (custom/OpenAI-compatible payloads), fall back to the same + * magnitude heuristic `normalizeUsageUnits` uses — cache ≤ input means it's + * already included — so cached events aren't re-inflated. The ground truth the + * gauge reconciles its calibrated estimate to. */ export const promptTokensFromUsage = (event: TTokenUsageEvent): number => { const input = event.input_tokens ?? 0;