💵 feat: Show Cost per Record, Step and Response in the Trace Ledger (#16120)
Some checks are pending
Codegraph E2E Votes / vote (full suite) (push) Waiting to run
Docker Dev Branch Images Build / publish (push) Waiting to run
Frontend Unit Tests / Client build recovery regression (push) Waiting to run
Frontend Unit Tests / Codegraph select (push) Waiting to run
Frontend Unit Tests / Build packages (push) Waiting to run
Frontend Unit Tests / TypeScript type checks (client) (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/client (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 1/2) (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 2/2) (push) Blocked by required conditions
Frontend Unit Tests / Vite build verification (push) Blocked by required conditions

* 💵 feat: Show Cost per Record, Step and Response in the Trace Ledger

* 💵 fix: Cost for Whole Responses Only, Hidden Records Included, and Said Aloud

* 💵 fix: Give the Oldest Loaded Response No Cost While Older Records Remain
This commit is contained in:
Danny Avila 2026-09-20 09:39:20 -04:00 • committed by GitHub
parent ceecedaf7a
commit 2551a057f0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 222 additions and 21 deletions

View file

@ -16,9 +16,9 @@ import type { RecordPresentation } from './present';
import StackedToolIcons from '~/components/Chat/Messages/Content/ToolOutput/StackedToolIcons';
import { spanOf, stepSpan, turnSpan, turnKey, boundsOf, isLabelRecord } from './model';
import { useTraceFormat, recordDurationText } from './format';
import { formatCost, formatTokens } from '~/utils/tokens';
import { appearanceOf, STATUS_LABEL } from './kinds';
import { cn, renderAgentAvatar } from '~/utils';
import { formatTokens } from '~/utils/tokens';
import { useRowWindow } from './virtual';
import { useLocalize } from '~/hooks';
@ -32,6 +32,9 @@ const ACTIVE_RING =
'group-focus-visible/tree:ring-2 group-focus-visible/tree:ring-inset group-focus-visible/tree:ring-ring-primary';
const GRID =
'grid grid-cols-[minmax(0,1fr)_4.5rem_minmax(5rem,1fr)] md:grid-cols-[minmax(14rem,2fr)_5rem_4.5rem_minmax(10rem,3fr)]';
/** With a cost column beside the tokens; both are wide-screen columns. */
const GRID_WITH_COST =
'grid grid-cols-[minmax(0,1fr)_4.5rem_minmax(5rem,1fr)] md:grid-cols-[minmax(14rem,2fr)_5rem_4.5rem_5rem_minmax(10rem,3fr)]';
type Domain = { start: number; span: number };
@ -132,6 +135,8 @@ function Ledger({
unrecordedCalls,
stepOffsets,
mcpIconMap,
showCost,
currency,
onSelect,
onToggle,
}: {
@ -152,12 +157,27 @@ function Ledger({
/** Steps of a response that ran before its first loaded one, by response. */
stepOffsets: ReadonlyMap<string, number>;
mcpIconMap: Map<string, string>;
/** Whether the deployment shows cost, as the context usage gauge does. */
showCost: boolean;
currency?: { code: string; rate: number };
onSelect: (id: string) => void;
onToggle: (key: string) => void;
}) {
const localize = useLocalize();
const format = useTraceFormat();
const idPrefix = useId();
const grid = showCost ? GRID_WITH_COST : GRID;
/** The rows name themselves with `aria-label`, so what a column shows has to be said there too. */
const costText = (cost?: number) =>
showCost && cost != null
? `, ${localize('com_ui_trace_summary_cost')} ${formatCost(cost, currency)}`
: '';
const costCell = (cost?: number) =>
showCost && (
<span className="hidden truncate text-right text-xs font-normal tabular-nums text-text-secondary md:block">
{cost != null ? formatCost(cost, currency) : ''}
</span>
);
const scrollRef = treeRef;
const [activeKey, setActiveKey] = useState<string | null>(null);
const { first, last } = useRowWindow(scrollRef, rows.length, ROW_HEIGHT, {
@ -344,11 +364,11 @@ function Ledger({
aria-level={row.level}
aria-expanded={row.expanded}
aria-selected={false}
aria-label={`${label}, ${description}, ${localize('com_ui_trace_turn_records', { 0: String(recordCount) })}`}
aria-label={`${label}, ${description}, ${localize('com_ui_trace_turn_records', { 0: String(recordCount) })}${costText(row.type === 'turn' ? row.turn.cost : row.step.cost)}`}
style={{ top: HEADER_HEIGHT + index * ROW_HEIGHT, height: ROW_HEIGHT }}
onClick={() => activate(row)}
className={cn(
GRID,
grid,
'absolute inset-x-0 cursor-pointer items-center gap-2 px-2 text-xs text-text-primary hover:bg-surface-hover',
row.type === 'turn'
? 'border-t border-border-light bg-surface-primary-alt font-semibold'
@ -395,6 +415,7 @@ function Ledger({
)}
</span>
<span className="hidden md:block" />
{costCell(row.type === 'turn' ? row.turn.cost : row.step.cost)}
<span className="relative h-full overflow-hidden">
<GroupBar span={span} domain={domainFor(messageId)} />
</span>
@ -466,12 +487,12 @@ function Ledger({
aria-label={localize('com_ui_trace_bar_description', {
0: `${name}, ${localize(appearance.label)}`,
1: format.duration(node.start - (turnStart ?? model.start)),
2: statusText,
2: `${statusText}${costText(record.cost)}`,
})}
style={{ top: HEADER_HEIGHT + index * ROW_HEIGHT, height: ROW_HEIGHT }}
onClick={() => activate(row)}
className={cn(
GRID,
grid,
'absolute inset-x-0 cursor-pointer items-center gap-2 px-2 text-sm text-text-primary hover:bg-surface-hover',
selectedId === row.key && 'bg-surface-active-alt',
active && ACTIVE_RING,
@ -553,6 +574,7 @@ function Ledger({
<span className="hidden text-right text-xs tabular-nums text-text-secondary md:block">
{recordTokens(node)}
</span>
{costCell(record.cost)}
<span className="relative h-full overflow-hidden">
<RecordBar node={node} scale={scale} domain={domainFor(record.messageId)} />
</span>
@ -587,13 +609,18 @@ function Ledger({
<div
aria-hidden="true"
className={cn(
GRID,
grid,
'sticky top-0 z-10 h-7 items-center gap-2 border-b border-border-light bg-presentation px-2 text-[11px] font-medium uppercase tracking-wide text-text-secondary',
)}
>
<span>{localize('com_ui_trace_column_name')}</span>
<span className="text-right">{localize('com_ui_trace_column_duration')}</span>
<span className="hidden text-right md:block">{localize('com_ui_trace_column_tokens')}</span>
{showCost && (
<span className="hidden text-right md:block">
{localize('com_ui_trace_summary_cost')}
</span>
)}
{viewDomain ? (
<span className="flex justify-between normal-case tabular-nums tracking-normal">
<span>{position(viewDomain.start)}</span>

View file

@ -159,7 +159,8 @@ export default function Viewer({
pages.every((page) => page.sourceId === langfuseSession.destinationId)
? langfuseSession.url
: undefined;
const model = useMemo(() => buildTraceModel(records, mode), [records, mode]);
const hasOlder = recordsQuery.hasNextPage === true;
const model = useMemo(() => buildTraceModel(records, mode, hasOlder), [records, mode, hasOlder]);
const bounds = useMemo(() => boundsOf(model, scale), [model, scale]);
const minSpan = minimumSpan(bounds, scale);
const previews = useMemo(() => buildPreviews(messages), [messages]);
@ -536,6 +537,8 @@ export default function Viewer({
unrecordedCalls={activity.unrecordedCalls}
stepOffsets={activity.stepOffsets}
mcpIconMap={mcpIconMap}
showCost={showCost}
currency={currency}
onSelect={setSelectedId}
onToggle={toggle}
/>

View file

@ -333,8 +333,30 @@ describe('Trace Viewer', () => {
mockStartupConfig = { interface: { traceViewer: { enabled: true }, contextCost: true } };
renderViewer();
await recordRow('llm');
expect(screen.getByText('com_ui_trace_summary_cost')).toBeInTheDocument();
const modelCall = await recordRow('llm');
const summary = screen.getByLabelText('com_ui_trace_summary');
expect(within(summary).getByText('com_ui_trace_summary_cost').nextSibling).toHaveTextContent(
'$0.02',
);
/** The ledger carries it too: a column, each priced record, and its step and response. */
const ledger = screen.getByTestId('trace-ledger');
expect(within(ledger).getByText('com_ui_trace_summary_cost')).toBeInTheDocument();
expect(within(modelCall).getByText('$0.02')).toBeInTheDocument();
/** Rows name themselves, so a screen reader hears the cost only if the name carries it. */
expect(modelCall).toHaveAccessibleName(/com_ui_trace_summary_cost \$0\.02/);
expect(stepRow(1)).toHaveAccessibleName(/com_ui_trace_summary_cost \$0\.02/);
expect(within(stepRow(1)).getByText('$0.02')).toBeInTheDocument();
expect(
within(screen.getByRole('treeitem', { name: /^com_ui_trace_turn/ })).getByText('$0.02'),
).toBeInTheDocument();
});
it('shows no cost anywhere while the deployment does not show context cost', async () => {
renderViewer();
expect(await recordRow('llm')).not.toHaveAccessibleName(/com_ui_trace_summary_cost/);
expect(screen.queryByText('com_ui_trace_summary_cost')).not.toBeInTheDocument();
expect(screen.queryByText('$0.02')).not.toBeInTheDocument();
});
it('explains a failure by its error code and recovers on retry', async () => {

View file

@ -246,6 +246,103 @@ describe('buildTraceModel', () => {
expect(model.turns[0].errorCount).toBe(1);
});
it('totals cost per step and per response under the same rule as the whole trace', () => {
const model = buildTraceModel([
record({ id: 'llm-1', kind: 'generation', cost: 0.01, startTime: at(0) }),
record({ id: 'tool', kind: 'tool', cost: 0.002, startTime: at(100) }),
record({ id: 'llm-2', kind: 'generation', startTime: at(1000) }),
record({
id: 'other',
messageId: 'response-2',
kind: 'generation',
cost: 0.5,
startTime: at(5000),
}),
record({
id: 'other-title',
messageId: 'response-2',
traceId: 'title',
origin: 'title',
kind: 'generation',
cost: 0.001,
startTime: at(5100),
}),
]);
const [first, second] = model.turns;
const stepCosts = (turn: typeof first) =>
turn.stepKeys.map((key) => model.steps.get(key)?.cost);
expect(stepCosts(first)).toEqual([expect.closeTo(0.012), undefined]);
/** One unpriced model call leaves its step, its response and the trace without a total. */
expect(first.cost).toBeUndefined();
expect(model.summary.cost).toBeUndefined();
/** A response's total counts its title run, which its own steps do not. */
expect(stepCosts(second)).toEqual([expect.closeTo(0.5), expect.closeTo(0.001)]);
expect(second.cost).toBeCloseTo(0.501);
});
it('gives no cost to a response the record limit cut, whose loaded records are not all it spent', () => {
const model = buildTraceModel([
record({ id: 'llm', parentId: 'not-loaded', kind: 'generation', cost: 0.2 }),
record({ id: 'whole', messageId: 'response-2', kind: 'generation', cost: 0.5 }),
]);
const [cut, whole] = model.turns;
expect(cut.split).toBe(true);
expect(cut.cost).toBeUndefined();
expect(model.steps.get(cut.stepKeys[0])?.cost).toBeCloseTo(0.2);
expect(whole.cost).toBeCloseTo(0.5);
});
it('gives the oldest loaded response no cost while older records remain, parents intact or not', () => {
/** A page that ended between a response's traces: its title run loaded, its own run not. */
const records = [
record({
id: 'title',
traceId: 'title-trace',
origin: 'title',
kind: 'generation',
cost: 0.001,
}),
record({
id: 'newer',
messageId: 'response-2',
kind: 'generation',
cost: 0.5,
startTime: at(9000),
}),
];
const paging = buildTraceModel(records, 'simple', true);
const settled = buildTraceModel(records, 'simple', false);
expect(paging.turns[0].split).toBe(false);
expect(paging.turns.map((turn) => turn.cost)).toEqual([undefined, expect.closeTo(0.5)]);
expect(settled.turns.map((turn) => turn.cost)).toEqual([
expect.closeTo(0.001),
expect.closeTo(0.5),
]);
});
it('counts what a step spent in records the simple mode rolls out of sight', () => {
const records = [
record({ id: 'tool', kind: 'tool', cost: 0.01, startTime: at(0) }),
record({ id: 'hidden-span', parentId: 'tool', cost: 0.04, startTime: at(10) }),
record({
id: 'nested-llm',
parentId: 'hidden-span',
kind: 'generation',
cost: 0.1,
startTime: at(20),
}),
];
for (const mode of ['simple', 'full'] as const) {
const model = buildTraceModel(records, mode);
expect(model.nodes.get('hidden-span')?.shown).toBe(mode === 'full');
expect(model.steps.get(model.turns[0].stepKeys[0])?.cost).toBeCloseTo(0.15);
expect(model.turns[0].cost).toBeCloseTo(0.15);
}
});
it('withholds the cost total when any model call has no price, with or without usage', () => {
const priced = record({ id: 'priced', kind: 'generation', usage: { total: 100 }, cost: 0.02 });
const withUsage = buildTraceModel([

View file

@ -8,6 +8,26 @@ export type TraceScale = 'sequence' | 'time';
export type TraceSpan = { start: number; end: number };
/**
* Spend over a set of records. A total that silently skips a model call without a price, with or
* without usage, would under-report, so `costOf` gives one only when every model call has a price.
*/
type Spend = { cost: number; priced: number; unpriced: number };
const noSpend = (): Spend => ({ cost: 0, priced: 0, unpriced: 0 });
function spendOn(spend: Spend, record: TTraceRecord): void {
if (record.cost != null) {
spend.priced++;
spend.cost += record.cost;
} else if (record.kind === 'generation') {
spend.unpriced++;
}
}
const costOf = (spend: Spend): number | undefined =>
spend.priced > 0 && spend.unpriced === 0 ? spend.cost : undefined;
export type TraceWindow = TraceSpan;
export type TraceNode = {
@ -55,6 +75,8 @@ export type TraceStep = {
toolCalls: number;
/** Tool call counts by tool name, in first-call order. */
toolNames: Map<string, number>;
/** What the step's records cost, when every model call among them has a price. */
cost?: number;
sequence: TraceSpan;
};
@ -81,6 +103,9 @@ export type TraceTurn = {
split: boolean;
/** Saved agents that ran in the response, each with the record that stands for it. */
agents: Array<{ agentId: string; recordId: string }>;
/** What the response's records cost, title and label calls included, when every model call has a
* price and the whole response is loaded. */
cost?: number;
sequence: TraceSpan;
};
@ -522,6 +547,7 @@ function groupSteps(
toolNames: new Map(),
sequence: EMPTY_SPAN,
};
const spend = noSpend();
const stack = [...rootIds].reverse();
while (stack.length > 0) {
const node = nodes.get(stack.pop() ?? '');
@ -550,6 +576,21 @@ function groupSteps(
stack.push(node.viewChildIds[i]);
}
}
/** Spend is the whole subtree's, not the listed projection's: the simple mode rolls spans
* and events up out of sight, and any record may carry a cost. */
const below = [...rootIds];
const counted = new Set<string>();
while (below.length > 0) {
const id = below.pop() ?? '';
const node = nodes.get(id);
if (!node || counted.has(id)) {
continue;
}
counted.add(id);
spendOn(spend, node.record);
below.push(...node.childIds);
}
step.cost = costOf(spend);
steps.set(key, step);
turn.stepKeys.push(key);
});
@ -630,6 +671,8 @@ function numberSubtree(nodes: Map<string, TraceNode>, rootIds: string[], next: n
export function buildTraceModel(
records: readonly TTraceRecord[],
mode: TraceMode = 'simple',
/** Older records are still to load, so the oldest loaded response may not be all there. */
hasOlder = false,
): TraceModel {
const nodes = new Map<string, TraceNode>();
const steps = new Map<string, TraceStep>();
@ -654,9 +697,8 @@ export function buildTraceModel(
const summary: TraceSummary = { ...EMPTY_SUMMARY };
let start = Number.POSITIVE_INFINITY;
let end = Number.NEGATIVE_INFINITY;
let cost = 0;
let pricedRecords = 0;
let unpricedRecords = 0;
const spend = noSpend();
const spendByTurn = new Map<string, Spend>();
for (const [id, node] of nodes) {
const { record } = node;
@ -723,14 +765,14 @@ export function buildTraceModel(
summary.inputTokens += input;
summary.outputTokens += output;
summary.totalTokens += total;
if (record.cost == null) {
unpricedRecords++;
}
}
if (record.cost != null) {
pricedRecords++;
cost += record.cost;
spendOn(spend, record);
let turnSpend = spendByTurn.get(record.messageId);
if (turnSpend == null) {
turnSpend = noSpend();
spendByTurn.set(record.messageId, turnSpend);
}
spendOn(turnSpend, record);
}
const compare = byStart(nodes);
@ -804,10 +846,20 @@ export function buildTraceModel(
summary.turns = turns.length;
summary.duration = end - start;
/** A total that silently skips a model call without a price, with or without usage, would under-report spend, so there is none. */
if (pricedRecords > 0 && unpricedRecords === 0) {
summary.cost = cost;
const total = costOf(spend);
if (total != null) {
summary.cost = total;
}
/**
* A response that is not all loaded holds only its newest records, and their sum is not its
* cost. A missing parent proves a cut (`split`), but a page can also end between a response's
* traces, its title run loaded and its own run not, with every parent in place. So while older
* records remain, the oldest loaded response is not known to be whole.
*/
turns.forEach((turn, index) => {
const partial = turn.split || (hasOlder && index === 0);
turn.cost = partial ? undefined : costOf(spendByTurn.get(turn.messageId) ?? noSpend());
});
return { mode, nodes, steps, turns, start, end, count: sequence, summary };
}