mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-17 13:33:45 +00:00
💵 feat: Show Cost per Record, Step and Response in the Trace Ledger (#16120)
Some checks are pending
Codegraph E2E Votes / vote (full suite) (push) Waiting to run
Docker Dev Branch Images Build / publish (push) Waiting to run
Frontend Unit Tests / Client build recovery regression (push) Waiting to run
Frontend Unit Tests / Codegraph select (push) Waiting to run
Frontend Unit Tests / Build packages (push) Waiting to run
Frontend Unit Tests / TypeScript type checks (client) (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/client (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 1/2) (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 2/2) (push) Blocked by required conditions
Frontend Unit Tests / Vite build verification (push) Blocked by required conditions
Some checks are pending
Codegraph E2E Votes / vote (full suite) (push) Waiting to run
Docker Dev Branch Images Build / publish (push) Waiting to run
Frontend Unit Tests / Client build recovery regression (push) Waiting to run
Frontend Unit Tests / Codegraph select (push) Waiting to run
Frontend Unit Tests / Build packages (push) Waiting to run
Frontend Unit Tests / TypeScript type checks (client) (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/client (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 1/2) (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 2/2) (push) Blocked by required conditions
Frontend Unit Tests / Vite build verification (push) Blocked by required conditions
* 💵 feat: Show Cost per Record, Step and Response in the Trace Ledger * 💵 fix: Cost for Whole Responses Only, Hidden Records Included, and Said Aloud * 💵 fix: Give the Oldest Loaded Response No Cost While Older Records Remain
This commit is contained in:
parent
ceecedaf7a
commit
2551a057f0
5 changed files with 222 additions and 21 deletions
|
|
@ -16,9 +16,9 @@ import type { RecordPresentation } from './present';
|
|||
import StackedToolIcons from '~/components/Chat/Messages/Content/ToolOutput/StackedToolIcons';
|
||||
import { spanOf, stepSpan, turnSpan, turnKey, boundsOf, isLabelRecord } from './model';
|
||||
import { useTraceFormat, recordDurationText } from './format';
|
||||
import { formatCost, formatTokens } from '~/utils/tokens';
|
||||
import { appearanceOf, STATUS_LABEL } from './kinds';
|
||||
import { cn, renderAgentAvatar } from '~/utils';
|
||||
import { formatTokens } from '~/utils/tokens';
|
||||
import { useRowWindow } from './virtual';
|
||||
import { useLocalize } from '~/hooks';
|
||||
|
||||
|
|
@ -32,6 +32,9 @@ const ACTIVE_RING =
|
|||
'group-focus-visible/tree:ring-2 group-focus-visible/tree:ring-inset group-focus-visible/tree:ring-ring-primary';
|
||||
const GRID =
|
||||
'grid grid-cols-[minmax(0,1fr)_4.5rem_minmax(5rem,1fr)] md:grid-cols-[minmax(14rem,2fr)_5rem_4.5rem_minmax(10rem,3fr)]';
|
||||
/** With a cost column beside the tokens; both are wide-screen columns. */
|
||||
const GRID_WITH_COST =
|
||||
'grid grid-cols-[minmax(0,1fr)_4.5rem_minmax(5rem,1fr)] md:grid-cols-[minmax(14rem,2fr)_5rem_4.5rem_5rem_minmax(10rem,3fr)]';
|
||||
|
||||
type Domain = { start: number; span: number };
|
||||
|
||||
|
|
@ -132,6 +135,8 @@ function Ledger({
|
|||
unrecordedCalls,
|
||||
stepOffsets,
|
||||
mcpIconMap,
|
||||
showCost,
|
||||
currency,
|
||||
onSelect,
|
||||
onToggle,
|
||||
}: {
|
||||
|
|
@ -152,12 +157,27 @@ function Ledger({
|
|||
/** Steps of a response that ran before its first loaded one, by response. */
|
||||
stepOffsets: ReadonlyMap<string, number>;
|
||||
mcpIconMap: Map<string, string>;
|
||||
/** Whether the deployment shows cost, as the context usage gauge does. */
|
||||
showCost: boolean;
|
||||
currency?: { code: string; rate: number };
|
||||
onSelect: (id: string) => void;
|
||||
onToggle: (key: string) => void;
|
||||
}) {
|
||||
const localize = useLocalize();
|
||||
const format = useTraceFormat();
|
||||
const idPrefix = useId();
|
||||
const grid = showCost ? GRID_WITH_COST : GRID;
|
||||
/** The rows name themselves with `aria-label`, so what a column shows has to be said there too. */
|
||||
const costText = (cost?: number) =>
|
||||
showCost && cost != null
|
||||
? `, ${localize('com_ui_trace_summary_cost')} ${formatCost(cost, currency)}`
|
||||
: '';
|
||||
const costCell = (cost?: number) =>
|
||||
showCost && (
|
||||
<span className="hidden truncate text-right text-xs font-normal tabular-nums text-text-secondary md:block">
|
||||
{cost != null ? formatCost(cost, currency) : ''}
|
||||
</span>
|
||||
);
|
||||
const scrollRef = treeRef;
|
||||
const [activeKey, setActiveKey] = useState<string | null>(null);
|
||||
const { first, last } = useRowWindow(scrollRef, rows.length, ROW_HEIGHT, {
|
||||
|
|
@ -344,11 +364,11 @@ function Ledger({
|
|||
aria-level={row.level}
|
||||
aria-expanded={row.expanded}
|
||||
aria-selected={false}
|
||||
aria-label={`${label}, ${description}, ${localize('com_ui_trace_turn_records', { 0: String(recordCount) })}`}
|
||||
aria-label={`${label}, ${description}, ${localize('com_ui_trace_turn_records', { 0: String(recordCount) })}${costText(row.type === 'turn' ? row.turn.cost : row.step.cost)}`}
|
||||
style={{ top: HEADER_HEIGHT + index * ROW_HEIGHT, height: ROW_HEIGHT }}
|
||||
onClick={() => activate(row)}
|
||||
className={cn(
|
||||
GRID,
|
||||
grid,
|
||||
'absolute inset-x-0 cursor-pointer items-center gap-2 px-2 text-xs text-text-primary hover:bg-surface-hover',
|
||||
row.type === 'turn'
|
||||
? 'border-t border-border-light bg-surface-primary-alt font-semibold'
|
||||
|
|
@ -395,6 +415,7 @@ function Ledger({
|
|||
)}
|
||||
</span>
|
||||
<span className="hidden md:block" />
|
||||
{costCell(row.type === 'turn' ? row.turn.cost : row.step.cost)}
|
||||
<span className="relative h-full overflow-hidden">
|
||||
<GroupBar span={span} domain={domainFor(messageId)} />
|
||||
</span>
|
||||
|
|
@ -466,12 +487,12 @@ function Ledger({
|
|||
aria-label={localize('com_ui_trace_bar_description', {
|
||||
0: `${name}, ${localize(appearance.label)}`,
|
||||
1: format.duration(node.start - (turnStart ?? model.start)),
|
||||
2: statusText,
|
||||
2: `${statusText}${costText(record.cost)}`,
|
||||
})}
|
||||
style={{ top: HEADER_HEIGHT + index * ROW_HEIGHT, height: ROW_HEIGHT }}
|
||||
onClick={() => activate(row)}
|
||||
className={cn(
|
||||
GRID,
|
||||
grid,
|
||||
'absolute inset-x-0 cursor-pointer items-center gap-2 px-2 text-sm text-text-primary hover:bg-surface-hover',
|
||||
selectedId === row.key && 'bg-surface-active-alt',
|
||||
active && ACTIVE_RING,
|
||||
|
|
@ -553,6 +574,7 @@ function Ledger({
|
|||
<span className="hidden text-right text-xs tabular-nums text-text-secondary md:block">
|
||||
{recordTokens(node)}
|
||||
</span>
|
||||
{costCell(record.cost)}
|
||||
<span className="relative h-full overflow-hidden">
|
||||
<RecordBar node={node} scale={scale} domain={domainFor(record.messageId)} />
|
||||
</span>
|
||||
|
|
@ -587,13 +609,18 @@ function Ledger({
|
|||
<div
|
||||
aria-hidden="true"
|
||||
className={cn(
|
||||
GRID,
|
||||
grid,
|
||||
'sticky top-0 z-10 h-7 items-center gap-2 border-b border-border-light bg-presentation px-2 text-[11px] font-medium uppercase tracking-wide text-text-secondary',
|
||||
)}
|
||||
>
|
||||
<span>{localize('com_ui_trace_column_name')}</span>
|
||||
<span className="text-right">{localize('com_ui_trace_column_duration')}</span>
|
||||
<span className="hidden text-right md:block">{localize('com_ui_trace_column_tokens')}</span>
|
||||
{showCost && (
|
||||
<span className="hidden text-right md:block">
|
||||
{localize('com_ui_trace_summary_cost')}
|
||||
</span>
|
||||
)}
|
||||
{viewDomain ? (
|
||||
<span className="flex justify-between normal-case tabular-nums tracking-normal">
|
||||
<span>{position(viewDomain.start)}</span>
|
||||
|
|
|
|||
|
|
@ -159,7 +159,8 @@ export default function Viewer({
|
|||
pages.every((page) => page.sourceId === langfuseSession.destinationId)
|
||||
? langfuseSession.url
|
||||
: undefined;
|
||||
const model = useMemo(() => buildTraceModel(records, mode), [records, mode]);
|
||||
const hasOlder = recordsQuery.hasNextPage === true;
|
||||
const model = useMemo(() => buildTraceModel(records, mode, hasOlder), [records, mode, hasOlder]);
|
||||
const bounds = useMemo(() => boundsOf(model, scale), [model, scale]);
|
||||
const minSpan = minimumSpan(bounds, scale);
|
||||
const previews = useMemo(() => buildPreviews(messages), [messages]);
|
||||
|
|
@ -536,6 +537,8 @@ export default function Viewer({
|
|||
unrecordedCalls={activity.unrecordedCalls}
|
||||
stepOffsets={activity.stepOffsets}
|
||||
mcpIconMap={mcpIconMap}
|
||||
showCost={showCost}
|
||||
currency={currency}
|
||||
onSelect={setSelectedId}
|
||||
onToggle={toggle}
|
||||
/>
|
||||
|
|
|
|||
|
|
@ -333,8 +333,30 @@ describe('Trace Viewer', () => {
|
|||
mockStartupConfig = { interface: { traceViewer: { enabled: true }, contextCost: true } };
|
||||
renderViewer();
|
||||
|
||||
await recordRow('llm');
|
||||
expect(screen.getByText('com_ui_trace_summary_cost')).toBeInTheDocument();
|
||||
const modelCall = await recordRow('llm');
|
||||
const summary = screen.getByLabelText('com_ui_trace_summary');
|
||||
expect(within(summary).getByText('com_ui_trace_summary_cost').nextSibling).toHaveTextContent(
|
||||
'$0.02',
|
||||
);
|
||||
/** The ledger carries it too: a column, each priced record, and its step and response. */
|
||||
const ledger = screen.getByTestId('trace-ledger');
|
||||
expect(within(ledger).getByText('com_ui_trace_summary_cost')).toBeInTheDocument();
|
||||
expect(within(modelCall).getByText('$0.02')).toBeInTheDocument();
|
||||
/** Rows name themselves, so a screen reader hears the cost only if the name carries it. */
|
||||
expect(modelCall).toHaveAccessibleName(/com_ui_trace_summary_cost \$0\.02/);
|
||||
expect(stepRow(1)).toHaveAccessibleName(/com_ui_trace_summary_cost \$0\.02/);
|
||||
expect(within(stepRow(1)).getByText('$0.02')).toBeInTheDocument();
|
||||
expect(
|
||||
within(screen.getByRole('treeitem', { name: /^com_ui_trace_turn/ })).getByText('$0.02'),
|
||||
).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows no cost anywhere while the deployment does not show context cost', async () => {
|
||||
renderViewer();
|
||||
|
||||
expect(await recordRow('llm')).not.toHaveAccessibleName(/com_ui_trace_summary_cost/);
|
||||
expect(screen.queryByText('com_ui_trace_summary_cost')).not.toBeInTheDocument();
|
||||
expect(screen.queryByText('$0.02')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('explains a failure by its error code and recovers on retry', async () => {
|
||||
|
|
|
|||
|
|
@ -246,6 +246,103 @@ describe('buildTraceModel', () => {
|
|||
expect(model.turns[0].errorCount).toBe(1);
|
||||
});
|
||||
|
||||
it('totals cost per step and per response under the same rule as the whole trace', () => {
|
||||
const model = buildTraceModel([
|
||||
record({ id: 'llm-1', kind: 'generation', cost: 0.01, startTime: at(0) }),
|
||||
record({ id: 'tool', kind: 'tool', cost: 0.002, startTime: at(100) }),
|
||||
record({ id: 'llm-2', kind: 'generation', startTime: at(1000) }),
|
||||
record({
|
||||
id: 'other',
|
||||
messageId: 'response-2',
|
||||
kind: 'generation',
|
||||
cost: 0.5,
|
||||
startTime: at(5000),
|
||||
}),
|
||||
record({
|
||||
id: 'other-title',
|
||||
messageId: 'response-2',
|
||||
traceId: 'title',
|
||||
origin: 'title',
|
||||
kind: 'generation',
|
||||
cost: 0.001,
|
||||
startTime: at(5100),
|
||||
}),
|
||||
]);
|
||||
const [first, second] = model.turns;
|
||||
const stepCosts = (turn: typeof first) =>
|
||||
turn.stepKeys.map((key) => model.steps.get(key)?.cost);
|
||||
|
||||
expect(stepCosts(first)).toEqual([expect.closeTo(0.012), undefined]);
|
||||
/** One unpriced model call leaves its step, its response and the trace without a total. */
|
||||
expect(first.cost).toBeUndefined();
|
||||
expect(model.summary.cost).toBeUndefined();
|
||||
/** A response's total counts its title run, which its own steps do not. */
|
||||
expect(stepCosts(second)).toEqual([expect.closeTo(0.5), expect.closeTo(0.001)]);
|
||||
expect(second.cost).toBeCloseTo(0.501);
|
||||
});
|
||||
|
||||
it('gives no cost to a response the record limit cut, whose loaded records are not all it spent', () => {
|
||||
const model = buildTraceModel([
|
||||
record({ id: 'llm', parentId: 'not-loaded', kind: 'generation', cost: 0.2 }),
|
||||
record({ id: 'whole', messageId: 'response-2', kind: 'generation', cost: 0.5 }),
|
||||
]);
|
||||
const [cut, whole] = model.turns;
|
||||
|
||||
expect(cut.split).toBe(true);
|
||||
expect(cut.cost).toBeUndefined();
|
||||
expect(model.steps.get(cut.stepKeys[0])?.cost).toBeCloseTo(0.2);
|
||||
expect(whole.cost).toBeCloseTo(0.5);
|
||||
});
|
||||
|
||||
it('gives the oldest loaded response no cost while older records remain, parents intact or not', () => {
|
||||
/** A page that ended between a response's traces: its title run loaded, its own run not. */
|
||||
const records = [
|
||||
record({
|
||||
id: 'title',
|
||||
traceId: 'title-trace',
|
||||
origin: 'title',
|
||||
kind: 'generation',
|
||||
cost: 0.001,
|
||||
}),
|
||||
record({
|
||||
id: 'newer',
|
||||
messageId: 'response-2',
|
||||
kind: 'generation',
|
||||
cost: 0.5,
|
||||
startTime: at(9000),
|
||||
}),
|
||||
];
|
||||
const paging = buildTraceModel(records, 'simple', true);
|
||||
const settled = buildTraceModel(records, 'simple', false);
|
||||
|
||||
expect(paging.turns[0].split).toBe(false);
|
||||
expect(paging.turns.map((turn) => turn.cost)).toEqual([undefined, expect.closeTo(0.5)]);
|
||||
expect(settled.turns.map((turn) => turn.cost)).toEqual([
|
||||
expect.closeTo(0.001),
|
||||
expect.closeTo(0.5),
|
||||
]);
|
||||
});
|
||||
|
||||
it('counts what a step spent in records the simple mode rolls out of sight', () => {
|
||||
const records = [
|
||||
record({ id: 'tool', kind: 'tool', cost: 0.01, startTime: at(0) }),
|
||||
record({ id: 'hidden-span', parentId: 'tool', cost: 0.04, startTime: at(10) }),
|
||||
record({
|
||||
id: 'nested-llm',
|
||||
parentId: 'hidden-span',
|
||||
kind: 'generation',
|
||||
cost: 0.1,
|
||||
startTime: at(20),
|
||||
}),
|
||||
];
|
||||
for (const mode of ['simple', 'full'] as const) {
|
||||
const model = buildTraceModel(records, mode);
|
||||
expect(model.nodes.get('hidden-span')?.shown).toBe(mode === 'full');
|
||||
expect(model.steps.get(model.turns[0].stepKeys[0])?.cost).toBeCloseTo(0.15);
|
||||
expect(model.turns[0].cost).toBeCloseTo(0.15);
|
||||
}
|
||||
});
|
||||
|
||||
it('withholds the cost total when any model call has no price, with or without usage', () => {
|
||||
const priced = record({ id: 'priced', kind: 'generation', usage: { total: 100 }, cost: 0.02 });
|
||||
const withUsage = buildTraceModel([
|
||||
|
|
|
|||
|
|
@ -8,6 +8,26 @@ export type TraceScale = 'sequence' | 'time';
|
|||
|
||||
export type TraceSpan = { start: number; end: number };
|
||||
|
||||
/**
|
||||
* Spend over a set of records. A total that silently skips a model call without a price, with or
|
||||
* without usage, would under-report, so `costOf` gives one only when every model call has a price.
|
||||
*/
|
||||
type Spend = { cost: number; priced: number; unpriced: number };
|
||||
|
||||
const noSpend = (): Spend => ({ cost: 0, priced: 0, unpriced: 0 });
|
||||
|
||||
function spendOn(spend: Spend, record: TTraceRecord): void {
|
||||
if (record.cost != null) {
|
||||
spend.priced++;
|
||||
spend.cost += record.cost;
|
||||
} else if (record.kind === 'generation') {
|
||||
spend.unpriced++;
|
||||
}
|
||||
}
|
||||
|
||||
const costOf = (spend: Spend): number | undefined =>
|
||||
spend.priced > 0 && spend.unpriced === 0 ? spend.cost : undefined;
|
||||
|
||||
export type TraceWindow = TraceSpan;
|
||||
|
||||
export type TraceNode = {
|
||||
|
|
@ -55,6 +75,8 @@ export type TraceStep = {
|
|||
toolCalls: number;
|
||||
/** Tool call counts by tool name, in first-call order. */
|
||||
toolNames: Map<string, number>;
|
||||
/** What the step's records cost, when every model call among them has a price. */
|
||||
cost?: number;
|
||||
sequence: TraceSpan;
|
||||
};
|
||||
|
||||
|
|
@ -81,6 +103,9 @@ export type TraceTurn = {
|
|||
split: boolean;
|
||||
/** Saved agents that ran in the response, each with the record that stands for it. */
|
||||
agents: Array<{ agentId: string; recordId: string }>;
|
||||
/** What the response's records cost, title and label calls included, when every model call has a
|
||||
* price and the whole response is loaded. */
|
||||
cost?: number;
|
||||
sequence: TraceSpan;
|
||||
};
|
||||
|
||||
|
|
@ -522,6 +547,7 @@ function groupSteps(
|
|||
toolNames: new Map(),
|
||||
sequence: EMPTY_SPAN,
|
||||
};
|
||||
const spend = noSpend();
|
||||
const stack = [...rootIds].reverse();
|
||||
while (stack.length > 0) {
|
||||
const node = nodes.get(stack.pop() ?? '');
|
||||
|
|
@ -550,6 +576,21 @@ function groupSteps(
|
|||
stack.push(node.viewChildIds[i]);
|
||||
}
|
||||
}
|
||||
/** Spend is the whole subtree's, not the listed projection's: the simple mode rolls spans
|
||||
* and events up out of sight, and any record may carry a cost. */
|
||||
const below = [...rootIds];
|
||||
const counted = new Set<string>();
|
||||
while (below.length > 0) {
|
||||
const id = below.pop() ?? '';
|
||||
const node = nodes.get(id);
|
||||
if (!node || counted.has(id)) {
|
||||
continue;
|
||||
}
|
||||
counted.add(id);
|
||||
spendOn(spend, node.record);
|
||||
below.push(...node.childIds);
|
||||
}
|
||||
step.cost = costOf(spend);
|
||||
steps.set(key, step);
|
||||
turn.stepKeys.push(key);
|
||||
});
|
||||
|
|
@ -630,6 +671,8 @@ function numberSubtree(nodes: Map<string, TraceNode>, rootIds: string[], next: n
|
|||
export function buildTraceModel(
|
||||
records: readonly TTraceRecord[],
|
||||
mode: TraceMode = 'simple',
|
||||
/** Older records are still to load, so the oldest loaded response may not be all there. */
|
||||
hasOlder = false,
|
||||
): TraceModel {
|
||||
const nodes = new Map<string, TraceNode>();
|
||||
const steps = new Map<string, TraceStep>();
|
||||
|
|
@ -654,9 +697,8 @@ export function buildTraceModel(
|
|||
const summary: TraceSummary = { ...EMPTY_SUMMARY };
|
||||
let start = Number.POSITIVE_INFINITY;
|
||||
let end = Number.NEGATIVE_INFINITY;
|
||||
let cost = 0;
|
||||
let pricedRecords = 0;
|
||||
let unpricedRecords = 0;
|
||||
const spend = noSpend();
|
||||
const spendByTurn = new Map<string, Spend>();
|
||||
|
||||
for (const [id, node] of nodes) {
|
||||
const { record } = node;
|
||||
|
|
@ -723,14 +765,14 @@ export function buildTraceModel(
|
|||
summary.inputTokens += input;
|
||||
summary.outputTokens += output;
|
||||
summary.totalTokens += total;
|
||||
if (record.cost == null) {
|
||||
unpricedRecords++;
|
||||
}
|
||||
}
|
||||
if (record.cost != null) {
|
||||
pricedRecords++;
|
||||
cost += record.cost;
|
||||
spendOn(spend, record);
|
||||
let turnSpend = spendByTurn.get(record.messageId);
|
||||
if (turnSpend == null) {
|
||||
turnSpend = noSpend();
|
||||
spendByTurn.set(record.messageId, turnSpend);
|
||||
}
|
||||
spendOn(turnSpend, record);
|
||||
}
|
||||
|
||||
const compare = byStart(nodes);
|
||||
|
|
@ -804,10 +846,20 @@ export function buildTraceModel(
|
|||
|
||||
summary.turns = turns.length;
|
||||
summary.duration = end - start;
|
||||
/** A total that silently skips a model call without a price, with or without usage, would under-report spend, so there is none. */
|
||||
if (pricedRecords > 0 && unpricedRecords === 0) {
|
||||
summary.cost = cost;
|
||||
const total = costOf(spend);
|
||||
if (total != null) {
|
||||
summary.cost = total;
|
||||
}
|
||||
/**
|
||||
* A response that is not all loaded holds only its newest records, and their sum is not its
|
||||
* cost. A missing parent proves a cut (`split`), but a page can also end between a response's
|
||||
* traces, its title run loaded and its own run not, with every parent in place. So while older
|
||||
* records remain, the oldest loaded response is not known to be whole.
|
||||
*/
|
||||
turns.forEach((turn, index) => {
|
||||
const partial = turn.split || (hasOlder && index === 0);
|
||||
turn.cost = partial ? undefined : costOf(spendByTurn.get(turn.messageId) ?? noSpend());
|
||||
});
|
||||
return { mode, nodes, steps, turns, start, end, count: sequence, summary };
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue