From d27a3b9a6dce5834d1803983460ec94c2a2a80b4 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 13 Jun 2026 11:43:14 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=A9=B9=20fix:=20Align=20Client=20Usage=20?= =?UTF-8?q?Accounting=20with=20Backend=20Cost=20Semantics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - classify cache tokens by provider (shared inputTokensIncludesCache from data-provider, consumed by both the backend billing path and the client) instead of a magnitude heuristic, so Anthropic/Bedrock turns where cache is smaller than uncached input no longer under-bill input - mirror resolveCompletionTokens on the client so Vertex-style hidden thinking tokens are reflected in the Output row and session cost - prefer endpoint pricing over adapter-provider pricing so a custom endpoint can price a known model name without built-in rates shadowing it - carry static cacheRead/cacheWrite overrides through the tokenConfig schema and buildTokenConfigMap --- client/src/hooks/Chat/useTokenUsage.ts | 7 ++- client/src/hooks/SSE/useUsageHandler.ts | 10 ++-- client/src/utils/tokens.spec.ts | 59 +++++++++++++++++++++- client/src/utils/tokens.ts | 35 +++++++++---- packages/api/src/agents/usage.ts | 29 +---------- packages/api/src/endpoints/pricing.spec.ts | 28 ++++++++++ packages/api/src/endpoints/pricing.ts | 8 +++ packages/data-provider/src/config.ts | 2 + packages/data-provider/src/schemas.ts | 23 +++++++++ 9 files changed, 156 insertions(+), 45 deletions(-) diff --git a/client/src/hooks/Chat/useTokenUsage.ts b/client/src/hooks/Chat/useTokenUsage.ts index 6761559711..e9f91d68bc 100644 --- a/client/src/hooks/Chat/useTokenUsage.ts +++ b/client/src/hooks/Chat/useTokenUsage.ts @@ -67,9 +67,12 @@ export default function useTokenUsage({ } let total = 0; for (const bucket of Object.values(usageTotals.byRate)) { + /** Endpoint-specific config wins (a custom endpoint may price a known + * model name differently); the adapter provider is the fallback, which + * is what resolves agent runs keyed under the underlying provider */ const rates = - (bucket.provider != null ? tokenConfig?.[bucket.provider]?.[bucket.model] : undefined) ?? - tokenConfig?.[bucket.endpoint]?.[bucket.model]; + tokenConfig?.[bucket.endpoint]?.[bucket.model] ?? + (bucket.provider != null ? tokenConfig?.[bucket.provider]?.[bucket.model] : undefined); total += costFromUnits(bucket, rates); } return total; diff --git a/client/src/hooks/SSE/useUsageHandler.ts b/client/src/hooks/SSE/useUsageHandler.ts index 73b418fa9f..a49ba384d8 100644 --- a/client/src/hooks/SSE/useUsageHandler.ts +++ b/client/src/hooks/SSE/useUsageHandler.ts @@ -115,11 +115,13 @@ export default function useUsageHandler(): UsageHandlers { const totalsAtom = usageTotalsFamily(convoKey); const prev = jotai.get(totalsAtom); const bucket = prev.byRate[bucketKey]; + /** Display the same normalized units that drive billing: input is the + * uncached portion, output includes repaired completion tokens */ jotai.set(totalsAtom, { - input: prev.input + (data.input_tokens ?? 0), - output: prev.output + (data.output_tokens ?? 0), - cacheWrite: prev.cacheWrite + (data.input_token_details?.cache_creation ?? 0), - cacheRead: prev.cacheRead + (data.input_token_details?.cache_read ?? 0), + input: prev.input + units.input, + output: prev.output + units.output, + cacheWrite: prev.cacheWrite + units.cacheWrite, + cacheRead: prev.cacheRead + units.cacheRead, eventCount: prev.eventCount + 1, byRate: { ...prev.byRate, diff --git a/client/src/utils/tokens.spec.ts b/client/src/utils/tokens.spec.ts index b6135f9325..f9408a513a 100644 --- a/client/src/utils/tokens.spec.ts +++ b/client/src/utils/tokens.spec.ts @@ -1,4 +1,4 @@ -import { Constants } from 'librechat-data-provider'; +import { Constants, Providers } from 'librechat-data-provider'; import type { TMessage } from 'librechat-data-provider'; import { buildIndex, @@ -148,6 +148,63 @@ describe('calcUsageCost', () => { expect(calcUsageCost({ input_tokens: 100 }, { context: 1000 })).toBe(0); }); + it('classifies cache additively for Anthropic even when cache <= input', () => { + /** The magnitude heuristic would wrongly treat this as inclusive and drop + * cache from input; the provider says additive */ + const event = { + input_tokens: 900, + output_tokens: 100, + provider: Providers.ANTHROPIC, + input_token_details: { cache_read: 100 }, + }; + expect(normalizeUsageUnits(event)).toEqual({ + input: 900, + output: 100, + cacheWrite: 0, + cacheRead: 100, + }); + expect(calcUsageCost(event, rates)).toBeCloseTo((900 * 3 + 100 * 0.3 + 100 * 15) / 1e6); + }); + + it('keeps subset semantics for OpenAI regardless of magnitude', () => { + const event = { + input_tokens: 900, + output_tokens: 100, + provider: Providers.OPENAI, + input_token_details: { cache_read: 100 }, + }; + expect(normalizeUsageUnits(event)).toEqual({ + input: 800, + output: 100, + cacheWrite: 0, + cacheRead: 100, + }); + }); + + it('repairs under-reported completion tokens (Vertex thinking)', () => { + const event = { + input_tokens: 1000, + output_tokens: 200, + total_tokens: 1500, + provider: Providers.VERTEXAI, + }; + /** total - input = 500 recovers the dropped thinking tokens */ + expect(normalizeUsageUnits(event).output).toBe(500); + }); + + it('does not mistake additive cache for missing completion tokens', () => { + /** Anthropic total includes cache; without the cache adjustment the repair + * would falsely inflate completion */ + const event = { + input_tokens: 1000, + output_tokens: 200, + total_tokens: 1700, + provider: Providers.ANTHROPIC, + input_token_details: { cache_creation: 300, cache_read: 200 }, + }; + expect(normalizeUsageUnits(event).output).toBe(200); + }); + it('prices summed normalized units identically to per-event costs', () => { const anthropicEvent = { input_tokens: 1000, diff --git a/client/src/utils/tokens.ts b/client/src/utils/tokens.ts index 69574ad4ed..893ffa3c6a 100644 --- a/client/src/utils/tokens.ts +++ b/client/src/utils/tokens.ts @@ -1,4 +1,4 @@ -import { Tools, Constants } from 'librechat-data-provider'; +import { Tools, Constants, inputTokensIncludesCache } from 'librechat-data-provider'; import type { TMessage, TTokenUsageEvent, TModelTokenomics } from 'librechat-data-provider'; export interface TokenEntry { @@ -242,20 +242,35 @@ export interface CostUnits { } /** - * Normalizes one call's usage into billable units. Mirrors the - * cache-detection heuristic used in token accounting: cache counts are - * additive when they exceed base input (Anthropic), otherwise cache reads - * are included in input tokens (OpenAI) — applied per event so units stay - * correct when summed across calls. + * Normalizes one call's usage into billable units, mirroring the backend's + * authoritative `splitUsage`/`resolveCompletionTokens` + * (packages/api/src/agents/usage.ts): + * - cache classification is by provider, not magnitude — Anthropic/Bedrock + * keep cache additive (input is uncached-only); subset providers fold + * cache into `input_tokens`. Falls back to a magnitude heuristic only when + * the provider is unknown. + * - completion is repaired for providers (e.g. Vertex) that under-report + * `output_tokens` but carry the gap in `total_tokens`. + * Applied per event so units stay correct when summed across calls. */ export function normalizeUsageUnits(usage: TTokenUsageEvent): CostUnits { - const input = usage.input_tokens ?? 0; - const output = usage.output_tokens ?? 0; + const rawInput = usage.input_tokens ?? 0; + const rawOutput = usage.output_tokens ?? 0; + const total = usage.total_tokens ?? 0; const cacheWrite = usage.input_token_details?.cache_creation ?? 0; const cacheRead = usage.input_token_details?.cache_read ?? 0; - const cacheIsAdditive = cacheWrite + cacheRead > input; + + const includesCache = + usage.provider != null + ? inputTokensIncludesCache(usage.provider) + : cacheWrite + cacheRead <= rawInput; + + const cacheAdjustment = includesCache ? 0 : cacheRead + cacheWrite; + const output = + total > rawInput + rawOutput + cacheAdjustment ? total - rawInput - cacheAdjustment : rawOutput; + return { - input: cacheIsAdditive ? input : Math.max(0, input - cacheRead - cacheWrite), + input: includesCache ? Math.max(0, rawInput - cacheRead - cacheWrite) : rawInput, output, cacheWrite, cacheRead, diff --git a/packages/api/src/agents/usage.ts b/packages/api/src/agents/usage.ts index cc15168547..bfc0b88a78 100644 --- a/packages/api/src/agents/usage.ts +++ b/packages/api/src/agents/usage.ts @@ -1,5 +1,5 @@ import { logger } from '@librechat/data-schemas'; -import { Providers } from 'librechat-data-provider'; +import { inputTokensIncludesCache } from 'librechat-data-provider'; import type { TCustomConfig, TTransactionsConfig } from 'librechat-data-provider'; import type { StructuredTokenUsage, @@ -23,33 +23,6 @@ type SpendStructuredTokensFn = ( tokenUsage: StructuredTokenUsage, ) => Promise; -/** - * Providers whose `usage_metadata.input_tokens` ALREADY INCLUDES cached tokens - * (i.e. `input_token_details.cache_*` is a subset, not an additional charge): - * - * - Google / Vertex AI: `input_tokens` = `promptTokenCount` (includes `cachedContentTokenCount`) - * - OpenAI / Azure OpenAI: `input_tokens` = `prompt_tokens` (includes `prompt_tokens_details.cached_tokens`) - * - xAI, DeepSeek, OpenRouter, Moonshot: extend `ChatOpenAI`, same semantics - * - * Anthropic and Bedrock keep cache values separate from `input_tokens`, so they - * must be added back to compute the total prompt size — that's the historical - * additive default. Providers not listed here fall through to additive. - */ -const SUBSET_PROVIDERS: ReadonlySet = new Set([ - Providers.OPENAI, - Providers.AZURE, - Providers.GOOGLE, - Providers.VERTEXAI, - Providers.XAI, - Providers.DEEPSEEK, - Providers.OPENROUTER, - Providers.MOONSHOT, -]); - -function inputTokensIncludesCache(provider?: string): boolean { - return provider != null && SUBSET_PROVIDERS.has(provider); -} - /** * Resolves `completionTokens` for billing, repairing providers whose * `usage_metadata.output_tokens` undercounts. diff --git a/packages/api/src/endpoints/pricing.spec.ts b/packages/api/src/endpoints/pricing.spec.ts index 6b4caa4f68..388b71a509 100644 --- a/packages/api/src/endpoints/pricing.spec.ts +++ b/packages/api/src/endpoints/pricing.spec.ts @@ -84,6 +84,34 @@ describe('buildTokenConfigMap', () => { expect(map.MyProxy['gpt-4o-mini'].prompt).toBeGreaterThan(0); }); + it('carries static cache rates from the override', () => { + const override: EndpointTokenConfig = { + 'cached-model': { + prompt: 3, + completion: 15, + context: 200000, + cacheRead: 0.3, + cacheWrite: 3.75, + } as EndpointTokenConfig[string], + }; + const map = buildTokenConfigMap( + { + modelsConfig: { MyProxy: ['cached-model'] }, + endpointTokenConfigs: { MyProxy: override }, + includePricing: true, + }, + deps, + ); + + expect(map.MyProxy['cached-model']).toEqual({ + context: 200000, + prompt: 3, + completion: 15, + cacheRead: 0.3, + cacheWrite: 3.75, + }); + }); + it('skips endpoints with empty model lists', () => { const map = buildTokenConfigMap({ modelsConfig: { [EModelEndpoint.agents]: [] } }, deps); expect(map[EModelEndpoint.agents]).toBeUndefined(); diff --git a/packages/api/src/endpoints/pricing.ts b/packages/api/src/endpoints/pricing.ts index 48bb5f624e..eb346ed374 100644 --- a/packages/api/src/endpoints/pricing.ts +++ b/packages/api/src/endpoints/pricing.ts @@ -54,6 +54,14 @@ export function buildTokenConfigMap( if (overrideRates?.prompt != null || overrideRates?.completion != null) { tokenomics.prompt = overrideRates.prompt; tokenomics.completion = overrideRates.completion; + /** Carry admin-configured cache rates; client falls back to the + * prompt rate for whichever cache rate is absent */ + if (overrideRates.cacheWrite != null) { + tokenomics.cacheWrite = overrideRates.cacheWrite; + } + if (overrideRates.cacheRead != null) { + tokenomics.cacheRead = overrideRates.cacheRead; + } } else { const valueKey = deps.getValueKey(model, endpoint); tokenomics.prompt = deps.getMultiplier({ diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 11d2fe7890..64bc73e291 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -853,6 +853,8 @@ export const endpointSchema = baseEndpointSchema.merge( prompt: z.number(), completion: z.number(), context: z.number(), + cacheRead: z.number().optional(), + cacheWrite: z.number().optional(), }), ) .optional(), diff --git a/packages/data-provider/src/schemas.ts b/packages/data-provider/src/schemas.ts index 42dd6927dc..e7b1186a0c 100644 --- a/packages/data-provider/src/schemas.ts +++ b/packages/data-provider/src/schemas.ts @@ -79,6 +79,29 @@ export const isOpenAILikeProvider = (provider?: string | null): boolean => { return openAILikeProviders.has(provider ?? ''); }; +/** + * Providers whose `usage_metadata.input_tokens` ALREADY INCLUDES cached tokens + * (`input_token_details.cache_*` is a subset, not an additional charge): + * Google/Vertex (`promptTokenCount`), OpenAI/Azure (`prompt_tokens`), and the + * OpenAI-compatible family. Anthropic/Bedrock keep cache values separate and + * additive. Single source of truth shared by the backend billing path + * (`packages/api/src/agents/usage.ts`) and the client usage normalization. + */ +export const cacheSubsetProviders = new Set([ + Providers.OPENAI, + Providers.AZURE, + Providers.GOOGLE, + Providers.VERTEXAI, + Providers.XAI, + Providers.DEEPSEEK, + Providers.OPENROUTER, + Providers.MOONSHOT, +]); + +export const inputTokensIncludesCache = (provider?: string | null): boolean => { + return cacheSubsetProviders.has(provider ?? ''); +}; + export const isDocumentSupportedProvider = (provider?: string | null): boolean => { return documentSupportedProviders.has(provider ?? ''); };