diff --git a/api/server/controllers/TokenConfigController.js b/api/server/controllers/TokenConfigController.js index 6826ad304d..8e5db5b3e6 100644 --- a/api/server/controllers/TokenConfigController.js +++ b/api/server/controllers/TokenConfigController.js @@ -1,5 +1,5 @@ const { logger } = require('@librechat/data-schemas'); -const { EModelEndpoint } = require('librechat-data-provider'); +const { EModelEndpoint, normalizeEndpointName } = require('librechat-data-provider'); const { buildTokenConfigMap, getTokenConfigKey, tokenConfigCache } = require('@librechat/api'); const { getModelsConfig } = require('~/server/controllers/ModelController'); const { getValueKey, getMultiplier, getCacheMultiplier } = require('~/models'); @@ -21,7 +21,8 @@ async function tokenConfigController(req, res) { const customEndpoints = appConfig?.endpoints?.[EModelEndpoint.custom] ?? []; const cache = tokenConfigCache(); for (const endpointConfig of customEndpoints) { - const name = endpointConfig?.name; + /** Models config and the token-config cache key by the normalized name */ + const name = normalizeEndpointName(endpointConfig?.name); if (!name) { continue; } diff --git a/api/server/controllers/agents/callbacks.js b/api/server/controllers/agents/callbacks.js index 0ddc8d6939..9373caa1d0 100644 --- a/api/server/controllers/agents/callbacks.js +++ b/api/server/controllers/agents/callbacks.js @@ -114,11 +114,20 @@ class ModelEndHandler { this.collectedUsage.push(taggedUsage); if (this.emitUsage) { + /** Normalize Anthropic/Bedrock-style top-level cache fields into details */ + const cache_creation = + taggedUsage.input_token_details?.cache_creation ?? + taggedUsage.cache_creation_input_tokens; + const cache_read = + taggedUsage.input_token_details?.cache_read ?? taggedUsage.cache_read_input_tokens; await this.emitUsage({ input_tokens: taggedUsage.input_tokens, output_tokens: taggedUsage.output_tokens, total_tokens: taggedUsage.total_tokens, - input_token_details: taggedUsage.input_token_details, + input_token_details: + cache_creation != null || cache_read != null + ? { cache_creation, cache_read } + : undefined, model: taggedUsage.model, provider: taggedUsage.provider, usage_type: taggedUsage.usage_type, diff --git a/api/server/routes/endpoints.js b/api/server/routes/endpoints.js index 03f225f560..8b1fceccc4 100644 --- a/api/server/routes/endpoints.js +++ b/api/server/routes/endpoints.js @@ -1,11 +1,12 @@ const express = require('express'); const requireJwtAuth = require('~/server/middleware/requireJwtAuth'); +const configMiddleware = require('~/server/middleware/config/app'); const endpointController = require('~/server/controllers/EndpointController'); const tokenConfigController = require('~/server/controllers/TokenConfigController'); const router = express.Router(); /** Auth required for role/tenant-scoped endpoint config resolution. */ router.get('/', requireJwtAuth, endpointController); -router.get('/token-config', requireJwtAuth, tokenConfigController); +router.get('/token-config', requireJwtAuth, configMiddleware, tokenConfigController); module.exports = router; diff --git a/client/src/hooks/SSE/useResumableSSE.ts b/client/src/hooks/SSE/useResumableSSE.ts index fd8cb7f2d4..71b3a98789 100644 --- a/client/src/hooks/SSE/useResumableSSE.ts +++ b/client/src/hooks/SSE/useResumableSSE.ts @@ -22,6 +22,7 @@ import type { TSubmission, TConversation, EventSubmission, + TTokenUsageEvent, } from 'librechat-data-provider'; import type { EventHandlerParams } from './useEventHandlers'; import type { ActiveJobsResponse } from '~/data-provider'; @@ -649,14 +650,27 @@ export default function useResumableSSE( submissionRef.current = resumeSubmission; userMessage = resumeSubmission.userMessage; /** - * The run's collected usage is the source of truth at sync time: - * pre-snapshot events are never replayed (only represented via - * aggregated content), so totals are rebuilt from the backfill - * and token-usage replay events are skipped when it's present. + * Totals rebuild from the persisted backfill at sync. Replayed or + * gap usage events already represented in the snapshot are skipped + * via multiset matching; events that raced past the snapshot read + * still fold so multi-call runs don't undercount. */ const backfilledUsage = data.resumeState?.collectedUsage; - const hasUsageBackfill = (backfilledUsage?.length ?? 0) > 0; backfillUsage(backfilledUsage ?? [], resumeSubmission); + const backfillCounts = new Map(); + for (const entry of backfilledUsage ?? []) { + const key = JSON.stringify(entry); + backfillCounts.set(key, (backfillCounts.get(key) ?? 0) + 1); + } + const foldIfNotBackfilled = (usageData: TTokenUsageEvent) => { + const key = JSON.stringify(usageData); + const count = backfillCounts.get(key) ?? 0; + if (count > 0) { + backfillCounts.set(key, count - 1); + return; + } + usageHandler(usageData, resumeSubmission); + }; if (data.resumeState?.contextUsage) { contextHandler(data.resumeState.contextUsage, resumeSubmission); } @@ -750,9 +764,7 @@ export default function useResumableSSE( if (replayEvent.event === UsageEvents.ON_CONTEXT_USAGE) { contextHandler(replayEvent.data, resumeSubmission); } else if (replayEvent.event === UsageEvents.ON_TOKEN_USAGE) { - if (!hasUsageBackfill) { - usageHandler(replayEvent.data, resumeSubmission); - } + foldIfNotBackfilled(replayEvent.data); } else if (replayEvent.event != null) { stepHandler(replayEvent, resumeSubmission); } @@ -767,9 +779,7 @@ export default function useResumableSSE( } else if (pendingEvent.event === UsageEvents.ON_CONTEXT_USAGE) { contextHandler(pendingEvent.data, resumeSubmission); } else if (pendingEvent.event === UsageEvents.ON_TOKEN_USAGE) { - if (!hasUsageBackfill) { - usageHandler(pendingEvent.data, resumeSubmission); - } + foldIfNotBackfilled(pendingEvent.data); } else if (pendingEvent.event != null) { stepHandler(pendingEvent, resumeSubmission); } else if (pendingEvent.type != null) { diff --git a/client/src/hooks/SSE/useUsageHandler.ts b/client/src/hooks/SSE/useUsageHandler.ts index 635c882d0b..6758856efb 100644 --- a/client/src/hooks/SSE/useUsageHandler.ts +++ b/client/src/hooks/SSE/useUsageHandler.ts @@ -105,7 +105,10 @@ export default function useUsageHandler(): UsageHandlers { const tokenConfig = queryClient.getQueryData([QueryKeys.tokenConfig]); const endpoint = submission.conversation?.endpoint ?? ''; const model = data.model ?? submission.conversation?.model ?? ''; - const rates = tokenConfig?.[endpoint]?.[model]; + /** Agent runs report the underlying provider, where rates are keyed */ + const rates = + (data.provider != null ? tokenConfig?.[data.provider]?.[model] : undefined) ?? + tokenConfig?.[endpoint]?.[model]; const totalsAtom = usageTotalsFamily(convoKey); const prev = jotai.get(totalsAtom); diff --git a/packages/data-schemas/src/app/interface.ts b/packages/data-schemas/src/app/interface.ts index 5af5af5423..de74aad9c0 100644 --- a/packages/data-schemas/src/app/interface.ts +++ b/packages/data-schemas/src/app/interface.ts @@ -40,6 +40,8 @@ export async function loadDefaultInterface({ customWelcome: interfaceConfig?.customWelcome ?? defaults.customWelcome, autoSubmitFromUrl: interfaceConfig?.autoSubmitFromUrl ?? defaults.autoSubmitFromUrl, buildInfo: interfaceConfig?.buildInfo ?? defaults.buildInfo, + contextUsage: interfaceConfig?.contextUsage ?? defaults.contextUsage, + contextCost: interfaceConfig?.contextCost ?? defaults.contextCost, // Permissions and related settings - only include if explicitly configured bookmarks: interfaceConfig?.bookmarks,