mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-10-20 17:13:44 +00:00
🟦 chore: Convert Activity-Label Eval Harness to TypeScript (#14530)
This commit is contained in:
Danny Avila
GitHub
parent
f0d3bcb622
commit
ad4ed67070
11 changed files with 528 additions and 133 deletions
|
|
@ -7,11 +7,12 @@ hypotheses that felt most obvious when this was built both turned out wrong
|
|||
under measurement (see **Findings**).
|
||||
|
||||
```bash
|
||||
node scripts/activity-labels/run.js # every variant, 1 sample
|
||||
node scripts/activity-labels/run.js --samples 3 # 3 samples each
|
||||
node scripts/activity-labels/run.js --variants baseline,shipping --cases fib-rapid
|
||||
node scripts/activity-labels/run.js --dry --cases mega-batch # render prompts, no API calls
|
||||
node scripts/activity-labels/rescore.js # re-grade stored results, no re-spend
|
||||
node scripts/activity-labels/run.mts # every variant, 1 sample
|
||||
node scripts/activity-labels/run.mts --samples 3 # 3 samples each
|
||||
node scripts/activity-labels/run.mts --variants baseline,shipping --cases fib-rapid
|
||||
node scripts/activity-labels/run.mts --dry --cases mega-batch # render prompts, no API calls
|
||||
node scripts/activity-labels/rescore.mts # re-grade stored results, no re-spend
|
||||
npx tsc -p scripts/activity-labels/tsconfig.json # type-check the harness
|
||||
```
|
||||
|
||||
Requires `ANTHROPIC_API_KEY` (env or `.env`). A full sweep is roughly $0.03 per
|
||||
|
|
@ -28,14 +29,16 @@ aggregate is the regression guard.
|
|||
|
||||
## Layout
|
||||
|
||||
| File | Role |
|
||||
| ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `captured.json` | 9 real production payloads pulled verbatim from Langfuse, with the headers that shipped. Irreplaceable — traces age out. |
|
||||
| `corpus.js` | 17 cases / 28 steps: the captured run as one sequence, plus synthetic cases for modes it never hit (all-failed, partial, parallel, rapid duplicates, entry overflow, truncated output, error-shaped success). Multi-step cases chain each generated label into the next step's context. |
|
||||
| `prompt.js` | Faithful port of the SDK's `buildActivityLabelPrompt`, so synthetic cases render the bytes production would send. Adds `previousLabelCap` for continuity-window experiments. |
|
||||
| `variants.js` | Single-factor instruction variants. `baseline` is read from the built `packages/api` dist, so drift from the shipped instruction is impossible. |
|
||||
| `checks.js` | Mechanical grading: length, punctuation, markdown, tool-name echo, count echo, and overlap split into `restate` (adds nothing) vs `template` (same frame, new payload). |
|
||||
| `run.js` / `rescore.js` | Live runner (production wire shape, `max_tokens: 256`) and offline re-grader. |
|
||||
| File | Role |
|
||||
| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `captured.json` | 9 real production payloads pulled verbatim from Langfuse, with the headers that shipped. Irreplaceable — traces age out. |
|
||||
| `corpus.mts` | 17 cases / 28 steps: the captured run as one sequence, plus synthetic cases for modes it never hit (all-failed, partial, parallel, rapid duplicates, entry overflow, truncated output, error-shaped success). Multi-step cases chain each generated label into the next step's context. |
|
||||
| `prompt.mts` | Faithful port of the SDK's `buildActivityLabelPrompt`, so synthetic cases render the bytes production would send. Adds `previousLabelCap` for continuity-window experiments. |
|
||||
| `variants.mts` | Single-factor instruction variants. `baseline` is read from the built `packages/api` dist, so drift from the shipped instruction is impossible. |
|
||||
| `checks.mts` | Mechanical grading: length, punctuation, markdown, tool-name echo, count echo, and overlap split into `restate` (adds nothing) vs `template` (same frame, new payload). |
|
||||
| `run.mts` / `rescore.mts` | Live runner (production wire shape, `max_tokens: 256`) and offline re-grader. |
|
||||
| `types.mts` | Shared corpus, variant, result, and report types used by the runner and rescorer. |
|
||||
| `tsconfig.json` | Strict, no-emit type-checking configuration for the harness. |
|
||||
|
||||
## Findings this produced
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,9 @@
|
|||
* cross-batch redundancy via content-word overlap); commit-log READABILITY
|
||||
* still needs the human pass over results/latest.md.
|
||||
*/
|
||||
const STOPWORDS = new Set([
|
||||
import type { ToolEntry } from './types.mts';
|
||||
|
||||
const STOPWORDS = new Set<string>([
|
||||
'the',
|
||||
'a',
|
||||
'an',
|
||||
|
|
@ -37,7 +39,7 @@ const STOPWORDS = new Set([
|
|||
'via',
|
||||
]);
|
||||
|
||||
function words(label) {
|
||||
export function words(label: string): string[] {
|
||||
return label.trim().split(/\s+/).filter(Boolean);
|
||||
}
|
||||
|
||||
|
|
@ -45,7 +47,7 @@ const SUFFIXES = ['ations', 'ation', 'ence', 'ance', 'ings', 'ing', 'ed', 'es',
|
|||
|
||||
/** Crude suffix stemmer so persists/persistence/persisted collide — enough
|
||||
* for overlap detection; linguistic correctness is not the goal. */
|
||||
function stem(word) {
|
||||
export function stem(word: string): string {
|
||||
if (word.length < 5) {
|
||||
return word;
|
||||
}
|
||||
|
|
@ -57,7 +59,7 @@ function stem(word) {
|
|||
return word;
|
||||
}
|
||||
|
||||
function contentWords(label) {
|
||||
export function contentWords(label: string): string[] {
|
||||
return words(label.toLowerCase().replace(/[^a-z0-9/._-]+/g, ' '))
|
||||
.filter((word) => !STOPWORDS.has(word))
|
||||
.map(stem);
|
||||
|
|
@ -65,11 +67,11 @@ function contentWords(label) {
|
|||
|
||||
/** Payload tokens carry the informative delta between template-shaped
|
||||
* labels: numbers, versions, paths, filenames. */
|
||||
function isPayload(word) {
|
||||
function isPayload(word: string): boolean {
|
||||
return /\d/.test(word) || word.includes('/') || word.includes('.');
|
||||
}
|
||||
|
||||
function jaccard(a, b) {
|
||||
export function jaccard(a: readonly string[], b: readonly string[]): number {
|
||||
const setA = new Set(a);
|
||||
const setB = new Set(b);
|
||||
if (setA.size === 0 || setB.size === 0) {
|
||||
|
|
@ -95,8 +97,23 @@ const DUP_THRESHOLD = 0.5;
|
|||
* regardless of whether the variant saw them — redundancy is measured
|
||||
* uniformly so continuity variants can be compared against blind ones.
|
||||
*/
|
||||
function checkLabel(label, { entries = [], previousLabels = [] } = {}) {
|
||||
const flags = [];
|
||||
interface CheckLabelOptions {
|
||||
entries?: readonly ToolEntry[];
|
||||
previousLabels?: readonly string[];
|
||||
}
|
||||
|
||||
interface CheckLabelResult {
|
||||
flags: string[];
|
||||
wordCount: number;
|
||||
firstWord: string;
|
||||
maxOverlap: number;
|
||||
}
|
||||
|
||||
export function checkLabel(
|
||||
label: string,
|
||||
{ entries = [], previousLabels = [] }: CheckLabelOptions = {},
|
||||
): CheckLabelResult {
|
||||
const flags: string[] = [];
|
||||
const wordList = words(label);
|
||||
if (wordList.length < 4 || wordList.length > 9) {
|
||||
flags.push(`len:${wordList.length}`);
|
||||
|
|
@ -115,7 +132,7 @@ function checkLabel(label, { entries = [], previousLabels = [] } = {}) {
|
|||
}
|
||||
const lower = label.toLowerCase();
|
||||
for (const entry of entries) {
|
||||
const name = String(entry.toolName ?? '').toLowerCase();
|
||||
const name = entry.toolName.toLowerCase();
|
||||
if (name.length > 3 && (lower.includes(name) || lower.includes(name.replace(/_/g, ' ')))) {
|
||||
flags.push(`tool-echo:${entry.toolName}`);
|
||||
break;
|
||||
|
|
@ -131,7 +148,7 @@ function checkLabel(label, { entries = [], previousLabels = [] } = {}) {
|
|||
* fib(1)→fib(2). Often fine, arguably better than synonym churn). */
|
||||
const own = contentWords(label);
|
||||
let maxOverlap = 0;
|
||||
let worst = null;
|
||||
let worst: string[] | null = null;
|
||||
for (const previous of previousLabels) {
|
||||
const other = contentWords(previous);
|
||||
const overlap = jaccard(own, other);
|
||||
|
|
@ -152,5 +169,3 @@ function checkLabel(label, { entries = [], previousLabels = [] } = {}) {
|
|||
}
|
||||
return { flags, wordCount: wordList.length, firstWord: wordList[0] ?? '', maxOverlap };
|
||||
}
|
||||
|
||||
module.exports = { checkLabel, contentWords, jaccard, words, stem };
|
||||
|
|
@ -14,12 +14,21 @@
|
|||
* step's generated label into the next step's `previousLabels` for variants
|
||||
* that opt in.
|
||||
*/
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
import { readFileSync } from 'node:fs';
|
||||
|
||||
const captured = JSON.parse(fs.readFileSync(path.join(__dirname, 'captured.json'), 'utf8'));
|
||||
import type { EvalCase, EvalStep, ToolEntry } from './types.mts';
|
||||
|
||||
const capturedRun = {
|
||||
interface CapturedEntry {
|
||||
id: string;
|
||||
prompt: string;
|
||||
productionLabel: string;
|
||||
}
|
||||
|
||||
const captured = JSON.parse(
|
||||
readFileSync(new URL('./captured.json', import.meta.url), 'utf8'),
|
||||
) as CapturedEntry[];
|
||||
|
||||
const capturedRun: EvalCase = {
|
||||
id: 'sandbox-probe-run',
|
||||
notes: 'the real 9-batch production run, verbatim payloads',
|
||||
steps: captured.map((entry) => ({
|
||||
|
|
@ -29,7 +38,7 @@ const capturedRun = {
|
|||
})),
|
||||
};
|
||||
|
||||
const synthetic = [
|
||||
const synthetic: EvalCase[] = [
|
||||
{
|
||||
id: 'all-failed',
|
||||
notes: 'every call fails — failure register, verb-first under failure',
|
||||
|
|
@ -443,13 +452,13 @@ const synthetic = [
|
|||
|
||||
/** Tool names for echo checks; captured steps bake entries into the
|
||||
* verbatim prompt, so they are recovered from the "Tool calls:" lines. */
|
||||
function stepEntries(step) {
|
||||
export function stepEntries(step: EvalStep): ToolEntry[] {
|
||||
if (step.payload?.entries) {
|
||||
return step.payload.entries;
|
||||
}
|
||||
return [...(step.verbatim ?? '').matchAll(/^- ([A-Za-z0-9_]+)\(/gm)].map((match) => ({
|
||||
toolName: match[1],
|
||||
toolName: match[1] ?? '',
|
||||
}));
|
||||
}
|
||||
|
||||
module.exports = { cases: [capturedRun, ...synthetic], stepEntries };
|
||||
export const cases: EvalCase[] = [capturedRun, ...synthetic];
|
||||
|
|
@ -12,12 +12,14 @@
|
|||
* unless a variant opts in. This is the P1 continuity hypothesis — it lets
|
||||
* the harness measure the fix before any SDK field exists.
|
||||
*/
|
||||
import type { EvalStep, SerializableValue, ToolEntry } from './types.mts';
|
||||
|
||||
const INPUT_CONTEXT_LIMIT = 200;
|
||||
const MAX_THINKING_EXCERPTS = 4;
|
||||
const MAX_PROMPT_ENTRIES = 12;
|
||||
const MAX_PREVIOUS_LABELS = 3;
|
||||
|
||||
function truncateForLabel(value, maxLength) {
|
||||
export function truncateForLabel(value: string, maxLength: number): string {
|
||||
if (value.length <= maxLength) {
|
||||
return value;
|
||||
}
|
||||
|
|
@ -26,7 +28,7 @@ function truncateForLabel(value, maxLength) {
|
|||
|
||||
const ABORT_SERIALIZATION = Symbol('abort-label-serialization');
|
||||
|
||||
function serializeForLabel(value, limit) {
|
||||
export function serializeForLabel(value: SerializableValue | undefined, limit: number): string {
|
||||
if (value == null) {
|
||||
return '';
|
||||
}
|
||||
|
|
@ -36,7 +38,7 @@ function serializeForLabel(value, limit) {
|
|||
let budget = limit * 4;
|
||||
try {
|
||||
return (
|
||||
JSON.stringify(value, (_key, nested) => {
|
||||
JSON.stringify(value, (_key: string, nested: SerializableValue) => {
|
||||
if (budget <= 0) {
|
||||
throw ABORT_SERIALIZATION;
|
||||
}
|
||||
|
|
@ -59,7 +61,10 @@ function serializeForLabel(value, limit) {
|
|||
|
||||
/** `cap` of Infinity models the unbounded-history alternative — testing
|
||||
* whether the whole run's story beats a recency window. */
|
||||
function previousHeadersSection(previousLabels, cap = MAX_PREVIOUS_LABELS) {
|
||||
function previousHeadersSection(
|
||||
previousLabels: readonly string[],
|
||||
cap = MAX_PREVIOUS_LABELS,
|
||||
): string | null {
|
||||
const kept = previousLabels.filter(Boolean);
|
||||
const recent = Number.isFinite(cap) ? kept.slice(-cap) : kept;
|
||||
if (recent.length === 0) {
|
||||
|
|
@ -71,14 +76,23 @@ function previousHeadersSection(previousLabels, cap = MAX_PREVIOUS_LABELS) {
|
|||
);
|
||||
}
|
||||
|
||||
function buildActivityLabelPrompt({
|
||||
interface BuildPromptOptions {
|
||||
entries: readonly ToolEntry[];
|
||||
charLimit: number;
|
||||
thinkingExcerpts?: readonly string[];
|
||||
lastAssistantText?: string;
|
||||
previousLabels?: readonly string[] | null;
|
||||
previousLabelCap?: number;
|
||||
}
|
||||
|
||||
export function buildActivityLabelPrompt({
|
||||
entries,
|
||||
charLimit,
|
||||
thinkingExcerpts,
|
||||
lastAssistantText,
|
||||
previousLabels,
|
||||
previousLabelCap,
|
||||
}) {
|
||||
}: BuildPromptOptions): string {
|
||||
const clip = truncateForLabel;
|
||||
const sections = [];
|
||||
if (previousLabels != null) {
|
||||
|
|
@ -128,12 +142,24 @@ function buildActivityLabelPrompt({
|
|||
* prompt (byte-exact from Langfuse); the continuity section, when a variant
|
||||
* opts in, is prepended — the same position the built path gives it.
|
||||
*/
|
||||
function renderStepPrompt(step, { charLimit, previousLabels, previousLabelCap }) {
|
||||
interface RenderPromptOptions {
|
||||
charLimit: number;
|
||||
previousLabels: readonly string[] | null;
|
||||
previousLabelCap?: number;
|
||||
}
|
||||
|
||||
export function renderStepPrompt(
|
||||
step: EvalStep,
|
||||
{ charLimit, previousLabels, previousLabelCap }: RenderPromptOptions,
|
||||
): string {
|
||||
if (step.verbatim != null) {
|
||||
const section =
|
||||
previousLabels != null ? previousHeadersSection(previousLabels, previousLabelCap) : null;
|
||||
return section != null ? `${section}\n\n${step.verbatim}` : step.verbatim;
|
||||
}
|
||||
if (step.payload == null) {
|
||||
throw new Error('corpus step must define either verbatim or payload');
|
||||
}
|
||||
return buildActivityLabelPrompt({
|
||||
...step.payload,
|
||||
charLimit,
|
||||
|
|
@ -142,10 +168,4 @@ function renderStepPrompt(step, { charLimit, previousLabels, previousLabelCap })
|
|||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
buildActivityLabelPrompt,
|
||||
renderStepPrompt,
|
||||
truncateForLabel,
|
||||
serializeForLabel,
|
||||
MAX_PREVIOUS_LABELS,
|
||||
};
|
||||
export { MAX_PREVIOUS_LABELS };
|
||||
|
|
@ -1,28 +1,56 @@
|
|||
/** Aggregation + markdown rendering, shared by the live runner and the
|
||||
* offline rescorer so metric fixes never require re-spending on the API. */
|
||||
const FLAG_TYPES = [
|
||||
'len',
|
||||
'punct',
|
||||
'quote',
|
||||
'md',
|
||||
'opener',
|
||||
'tool-echo',
|
||||
'count-echo',
|
||||
'restate',
|
||||
'template',
|
||||
];
|
||||
import type { Aggregate, ErrorRecord, EvalCase, EvalRecord, FlagType } from './types.mts';
|
||||
import { FLAG_TYPES } from './types.mts';
|
||||
|
||||
const PRICES = { 'claude-haiku-4-5': { input: 1, output: 5 } };
|
||||
|
||||
function flagType(flag) {
|
||||
return flag.split(':')[0];
|
||||
interface Price {
|
||||
input: number;
|
||||
output: number;
|
||||
}
|
||||
|
||||
function aggregate(records, model) {
|
||||
const byVariant = new Map();
|
||||
interface AggregateState {
|
||||
steps: number;
|
||||
errors: number;
|
||||
flagCounts: Partial<Record<FlagType, number>>;
|
||||
firstWords: Record<string, number>;
|
||||
totalWords: number;
|
||||
latencies: number[];
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
}
|
||||
|
||||
interface MarkdownReportOptions {
|
||||
records: readonly EvalRecord[];
|
||||
aggregates: readonly Aggregate[];
|
||||
runCases: readonly EvalCase[];
|
||||
variantNames: readonly string[];
|
||||
model: string;
|
||||
samples: number;
|
||||
}
|
||||
|
||||
const PRICES: Readonly<Record<string, Price>> = {
|
||||
'claude-haiku-4-5': { input: 1, output: 5 },
|
||||
};
|
||||
const FLAG_TYPE_SET = new Set<string>(FLAG_TYPES);
|
||||
|
||||
function flagType(flag: string): FlagType {
|
||||
const type = flag.split(':')[0] ?? '';
|
||||
if (!FLAG_TYPE_SET.has(type)) {
|
||||
throw new Error(`unknown flag type: ${type}`);
|
||||
}
|
||||
return type as FlagType;
|
||||
}
|
||||
|
||||
function isErrorRecord(record: EvalRecord): record is ErrorRecord {
|
||||
return 'error' in record;
|
||||
}
|
||||
|
||||
export function aggregate(records: readonly EvalRecord[], model: string): Aggregate[] {
|
||||
const byVariant = new Map<string, AggregateState>();
|
||||
for (const record of records) {
|
||||
if (!byVariant.has(record.variant)) {
|
||||
byVariant.set(record.variant, {
|
||||
let agg = byVariant.get(record.variant);
|
||||
if (agg == null) {
|
||||
agg = {
|
||||
steps: 0,
|
||||
errors: 0,
|
||||
flagCounts: {},
|
||||
|
|
@ -31,10 +59,10 @@ function aggregate(records, model) {
|
|||
latencies: [],
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
});
|
||||
};
|
||||
byVariant.set(record.variant, agg);
|
||||
}
|
||||
const agg = byVariant.get(record.variant);
|
||||
if (record.error) {
|
||||
if (isErrorRecord(record)) {
|
||||
agg.errors += 1;
|
||||
continue;
|
||||
}
|
||||
|
|
@ -52,7 +80,7 @@ function aggregate(records, model) {
|
|||
const price = PRICES[model];
|
||||
return [...byVariant.entries()].map(([name, agg]) => {
|
||||
const sortedFirst = Object.entries(agg.firstWords).sort((a, b) => b[1] - a[1]);
|
||||
const topOpener = sortedFirst[0] ?? ['—', 0];
|
||||
const topOpener: [string, number] = sortedFirst[0] ?? ['—', 0];
|
||||
return {
|
||||
variant: name,
|
||||
steps: agg.steps,
|
||||
|
|
@ -73,8 +101,15 @@ function aggregate(records, model) {
|
|||
});
|
||||
}
|
||||
|
||||
function markdownReport({ records, aggregates, runCases, variantNames, model, samples }) {
|
||||
const lines = [];
|
||||
export function markdownReport({
|
||||
records,
|
||||
aggregates,
|
||||
runCases,
|
||||
variantNames,
|
||||
model,
|
||||
samples,
|
||||
}: MarkdownReportOptions): string {
|
||||
const lines: string[] = [];
|
||||
lines.push(`# Activity-label eval — ${new Date().toISOString()}`);
|
||||
lines.push('');
|
||||
lines.push(`model: \`${model}\` · samples: ${samples} · cases: ${runCases.length}`);
|
||||
|
|
@ -100,7 +135,9 @@ function markdownReport({ records, aggregates, runCases, variantNames, model, sa
|
|||
lines.push('');
|
||||
lines.push(`*${testCase.notes}*`);
|
||||
lines.push('');
|
||||
const sampleList = [...new Set(records.map((r) => r.sample))].sort();
|
||||
const sampleList = [...new Set(records.map((record) => record.sample))].sort(
|
||||
(first, second) => first - second,
|
||||
);
|
||||
const header = ['step'];
|
||||
if (samples > 1) {
|
||||
header.push('s');
|
||||
|
|
@ -131,7 +168,7 @@ function markdownReport({ records, aggregates, runCases, variantNames, model, sa
|
|||
);
|
||||
if (!record) {
|
||||
row.push('');
|
||||
} else if (record.error) {
|
||||
} else if (isErrorRecord(record)) {
|
||||
row.push(`⛔ ${record.error}`);
|
||||
} else {
|
||||
const flagNote = record.flags.length > 0 ? ` ⚠${record.flags.join(' ⚠')}` : '';
|
||||
|
|
@ -144,5 +181,3 @@ function markdownReport({ records, aggregates, runCases, variantNames, model, sa
|
|||
}
|
||||
return lines.join('\n') + '\n';
|
||||
}
|
||||
|
||||
module.exports = { aggregate, markdownReport, FLAG_TYPES };
|
||||
Binary file not shown.
77
scripts/activity-labels/rescore.mts
Normal file
77
scripts/activity-labels/rescore.mts
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
/**
|
||||
* Offline rescore: recompute checks over a stored results JSON after a
|
||||
* metric change, without re-calling the API. Chains are rebuilt from the
|
||||
* stored labels in push order (steps within a case ran serially).
|
||||
*
|
||||
* Usage: node scripts/activity-labels/rescore.mts [results/<file>.json]
|
||||
*/
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { basename, join, resolve } from 'node:path';
|
||||
import { readdirSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
|
||||
import type { EvalStep, StoredResults } from './types.mts';
|
||||
import { aggregate, markdownReport } from './report.mts';
|
||||
import { cases, stepEntries } from './corpus.mts';
|
||||
import { checkLabel } from './checks.mts';
|
||||
|
||||
const RESULTS_DIR = fileURLToPath(new URL('./results', import.meta.url));
|
||||
|
||||
function newestResults(): string {
|
||||
const files = readdirSync(RESULTS_DIR)
|
||||
.filter((file) => file.endsWith('.json'))
|
||||
.sort();
|
||||
const newest = files.at(-1);
|
||||
if (newest == null) {
|
||||
throw new Error('no stored results to rescore');
|
||||
}
|
||||
return join(RESULTS_DIR, newest);
|
||||
}
|
||||
|
||||
const sourcePath = process.argv[2] ? resolve(process.argv[2]) : newestResults();
|
||||
const { args, records } = JSON.parse(readFileSync(sourcePath, 'utf8')) as StoredResults;
|
||||
|
||||
const stepsByCase = new Map<string, Map<string, EvalStep>>(
|
||||
cases.map((testCase) => [
|
||||
testCase.id,
|
||||
new Map(testCase.steps.map((step) => [step.id ?? testCase.id, step])),
|
||||
]),
|
||||
);
|
||||
|
||||
const chains = new Map<string, string[]>();
|
||||
for (const record of records) {
|
||||
if ('error' in record) {
|
||||
continue;
|
||||
}
|
||||
const key = `${record.variant}\0${record.sample}\0${record.caseId}`;
|
||||
let chain = chains.get(key);
|
||||
if (chain == null) {
|
||||
chain = [];
|
||||
chains.set(key, chain);
|
||||
}
|
||||
const step = stepsByCase.get(record.caseId)?.get(record.stepId);
|
||||
const { flags, wordCount, firstWord } = checkLabel(record.label, {
|
||||
entries: step != null ? stepEntries(step) : [],
|
||||
previousLabels: chain,
|
||||
});
|
||||
record.flags = flags;
|
||||
record.wordCount = wordCount;
|
||||
record.firstWord = firstWord;
|
||||
chain.push(record.label);
|
||||
}
|
||||
|
||||
const variantNames = [...new Set(records.map((record) => record.variant))];
|
||||
const runCases = cases.filter((testCase) =>
|
||||
records.some((record) => record.caseId === testCase.id),
|
||||
);
|
||||
const report = markdownReport({
|
||||
records,
|
||||
aggregates: aggregate(records, args.model),
|
||||
runCases,
|
||||
variantNames,
|
||||
model: args.model,
|
||||
samples: args.samples,
|
||||
});
|
||||
writeFileSync(join(RESULTS_DIR, 'latest.md'), report);
|
||||
console.log(`rescored ${basename(sourcePath)}`);
|
||||
console.log(report.split('## Per-case')[0]);
|
||||
console.log('full per-case tables: scripts/activity-labels/results/latest.md');
|
||||
|
|
@ -11,60 +11,126 @@
|
|||
* continuity variants share one redundancy metric.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/activity-labels/run.js [--variants baseline,continuity]
|
||||
* node scripts/activity-labels/run.mts [--variants baseline,continuity]
|
||||
* [--cases sandbox-probe-run,fib-rapid] [--samples 2] [--model id]
|
||||
* [--concurrency 6] [--dry]
|
||||
*/
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
import { join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
|
||||
const { cases, stepEntries } = require('./corpus');
|
||||
const { variants } = require('./variants');
|
||||
const { checkLabel } = require('./checks');
|
||||
const { renderStepPrompt } = require('./prompt');
|
||||
const { aggregate, markdownReport } = require('./report');
|
||||
import type { DryRunRecord, EvalCase, EvalRecord, RunArgs, RunRecord, Variant } from './types.mts';
|
||||
import { aggregate, markdownReport } from './report.mts';
|
||||
import { cases, stepEntries } from './corpus.mts';
|
||||
import { renderStepPrompt } from './prompt.mts';
|
||||
import { checkLabel } from './checks.mts';
|
||||
import { variants } from './variants.mts';
|
||||
|
||||
const ROOT = path.resolve(__dirname, '..', '..');
|
||||
const RESULTS_DIR = path.join(__dirname, 'results');
|
||||
const ROOT = fileURLToPath(new URL('../../', import.meta.url));
|
||||
const RESULTS_DIR = fileURLToPath(new URL('./results', import.meta.url));
|
||||
const CHAR_LIMIT = 600;
|
||||
const MAX_TOKENS = 256;
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = { samples: 1, concurrency: 6, model: 'claude-haiku-4-5', dry: false };
|
||||
interface AnthropicMessageResponse {
|
||||
content?: Array<{ text?: string }>;
|
||||
usage?: {
|
||||
input_tokens?: number;
|
||||
output_tokens?: number;
|
||||
};
|
||||
}
|
||||
|
||||
interface RequestLabelOptions {
|
||||
apiKey: string;
|
||||
model: string;
|
||||
instruction: string;
|
||||
prompt: string;
|
||||
}
|
||||
|
||||
type RequestLabelResult =
|
||||
| {
|
||||
label: string;
|
||||
latencyMs: number;
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
}
|
||||
| {
|
||||
error: string;
|
||||
latencyMs: number;
|
||||
};
|
||||
|
||||
interface RunCaseOptions {
|
||||
apiKey: string;
|
||||
model: string;
|
||||
variant: Variant;
|
||||
sample: number;
|
||||
testCase: EvalCase;
|
||||
dry: boolean;
|
||||
records: RunRecord[];
|
||||
}
|
||||
|
||||
type Task = () => Promise<void>;
|
||||
|
||||
function nextArgument(argv: readonly string[], index: number, key: string): string {
|
||||
const value = argv[index + 1];
|
||||
if (value == null || value.startsWith('--')) {
|
||||
throw new Error(`${key} requires a value`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function positiveInteger(value: string, key: string): number {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) {
|
||||
throw new Error(`${key} must be a positive integer`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function parseArgs(argv: readonly string[]): RunArgs {
|
||||
const args: RunArgs = {
|
||||
samples: 1,
|
||||
concurrency: 6,
|
||||
model: 'claude-haiku-4-5',
|
||||
dry: false,
|
||||
};
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const key = argv[i];
|
||||
if (key === '--dry') {
|
||||
args.dry = true;
|
||||
} else if (key === '--variants') {
|
||||
args.variants = argv[++i].split(',');
|
||||
args.variants = nextArgument(argv, i, key).split(',');
|
||||
i += 1;
|
||||
} else if (key === '--cases') {
|
||||
args.cases = argv[++i].split(',');
|
||||
args.cases = nextArgument(argv, i, key).split(',');
|
||||
i += 1;
|
||||
} else if (key === '--samples') {
|
||||
args.samples = Number(argv[++i]);
|
||||
args.samples = positiveInteger(nextArgument(argv, i, key), key);
|
||||
i += 1;
|
||||
} else if (key === '--model') {
|
||||
args.model = argv[++i];
|
||||
args.model = nextArgument(argv, i, key);
|
||||
i += 1;
|
||||
} else if (key === '--concurrency') {
|
||||
args.concurrency = Number(argv[++i]);
|
||||
args.concurrency = positiveInteger(nextArgument(argv, i, key), key);
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function loadKey() {
|
||||
function loadKey(): string {
|
||||
if (process.env.ANTHROPIC_API_KEY) {
|
||||
return process.env.ANTHROPIC_API_KEY;
|
||||
}
|
||||
const envPath = path.join(ROOT, '.env');
|
||||
const line = fs.existsSync(envPath)
|
||||
? fs
|
||||
.readFileSync(envPath, 'utf8')
|
||||
const envPath = join(ROOT, '.env');
|
||||
const line = existsSync(envPath)
|
||||
? readFileSync(envPath, 'utf8')
|
||||
.split('\n')
|
||||
.find((entry) => entry.startsWith('ANTHROPIC_API_KEY='))
|
||||
: undefined;
|
||||
if (!line) {
|
||||
throw new Error(
|
||||
`ANTHROPIC_API_KEY not set and not found in ${envPath}.\n` +
|
||||
'Pass it inline: ANTHROPIC_API_KEY=sk-… node scripts/activity-labels/run.js',
|
||||
'Pass it inline: ANTHROPIC_API_KEY=sk-… node scripts/activity-labels/run.mts',
|
||||
);
|
||||
}
|
||||
return line
|
||||
|
|
@ -73,7 +139,12 @@ function loadKey() {
|
|||
.replace(/^["']|["']$/g, '');
|
||||
}
|
||||
|
||||
async function requestLabel({ apiKey, model, instruction, prompt }) {
|
||||
async function requestLabel({
|
||||
apiKey,
|
||||
model,
|
||||
instruction,
|
||||
prompt,
|
||||
}: RequestLabelOptions): Promise<RequestLabelResult> {
|
||||
for (let attempt = 1; attempt <= 3; attempt++) {
|
||||
const started = Date.now();
|
||||
const response = await fetch('https://api.anthropic.com/v1/messages', {
|
||||
|
|
@ -91,7 +162,7 @@ async function requestLabel({ apiKey, model, instruction, prompt }) {
|
|||
}),
|
||||
});
|
||||
if (response.ok) {
|
||||
const json = await response.json();
|
||||
const json = (await response.json()) as AnthropicMessageResponse;
|
||||
const label = (json.content ?? [])
|
||||
.map((block) => block.text ?? '')
|
||||
.join('')
|
||||
|
|
@ -117,11 +188,20 @@ async function requestLabel({ apiKey, model, instruction, prompt }) {
|
|||
latencyMs: Date.now() - started,
|
||||
};
|
||||
}
|
||||
throw new Error('label request exhausted retries without a result');
|
||||
}
|
||||
|
||||
/** One case chain: steps serial, labels feeding forward. */
|
||||
async function runCase({ apiKey, model, variant, sample, testCase, dry, records }) {
|
||||
const chain = [];
|
||||
async function runCase({
|
||||
apiKey,
|
||||
model,
|
||||
variant,
|
||||
sample,
|
||||
testCase,
|
||||
dry,
|
||||
records,
|
||||
}: RunCaseOptions): Promise<void> {
|
||||
const chain: string[] = [];
|
||||
for (const step of testCase.steps) {
|
||||
const prompt = renderStepPrompt(step, {
|
||||
charLimit: CHAR_LIMIT,
|
||||
|
|
@ -134,7 +214,7 @@ async function runCase({ apiKey, model, variant, sample, testCase, dry, records
|
|||
continue;
|
||||
}
|
||||
const result = await requestLabel({ apiKey, model, instruction: variant.instruction, prompt });
|
||||
if (result.error) {
|
||||
if ('error' in result) {
|
||||
records.push({
|
||||
variant: variant.name,
|
||||
sample,
|
||||
|
|
@ -166,28 +246,43 @@ async function runCase({ apiKey, model, variant, sample, testCase, dry, records
|
|||
}
|
||||
}
|
||||
|
||||
async function pool(tasks, size) {
|
||||
async function pool(tasks: readonly Task[], size: number): Promise<void> {
|
||||
const queue = [...tasks];
|
||||
const workers = Array.from({ length: Math.min(size, queue.length) }, async () => {
|
||||
while (queue.length > 0) {
|
||||
await queue.shift()();
|
||||
const task = queue.shift();
|
||||
if (task != null) {
|
||||
await task();
|
||||
}
|
||||
}
|
||||
});
|
||||
await Promise.all(workers);
|
||||
}
|
||||
|
||||
(async () => {
|
||||
function isDryRunRecord(record: RunRecord): record is DryRunRecord {
|
||||
return 'prompt' in record;
|
||||
}
|
||||
|
||||
function isEvalRecord(record: RunRecord): record is EvalRecord {
|
||||
return !isDryRunRecord(record);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const runVariants = args.variants
|
||||
? variants.filter((variant) => args.variants.includes(variant.name))
|
||||
const selectedVariants = args.variants;
|
||||
const selectedCases = args.cases;
|
||||
const runVariants = selectedVariants
|
||||
? variants.filter((variant) => selectedVariants.includes(variant.name))
|
||||
: variants;
|
||||
const runCases = args.cases ? cases.filter((c) => args.cases.includes(c.id)) : cases;
|
||||
const runCases = selectedCases
|
||||
? cases.filter((testCase) => selectedCases.includes(testCase.id))
|
||||
: cases;
|
||||
if (runVariants.length === 0 || runCases.length === 0) {
|
||||
throw new Error('nothing selected — check --variants / --cases names');
|
||||
}
|
||||
const apiKey = args.dry ? '' : loadKey();
|
||||
const records = [];
|
||||
const tasks = [];
|
||||
const records: RunRecord[] = [];
|
||||
const tasks: Task[] = [];
|
||||
for (const variant of runVariants) {
|
||||
for (let sample = 1; sample <= args.samples; sample++) {
|
||||
for (const testCase of runCases) {
|
||||
|
|
@ -206,36 +301,40 @@ async function pool(tasks, size) {
|
|||
console.log(`done in ${((Date.now() - started) / 1000).toFixed(1)}s\n`);
|
||||
|
||||
if (args.dry) {
|
||||
for (const record of records.slice(0, 3)) {
|
||||
const dryRecords = records.filter(isDryRunRecord);
|
||||
for (const record of dryRecords.slice(0, 3)) {
|
||||
console.log(`--- ${record.variant} / ${record.caseId} / ${record.stepId} ---`);
|
||||
console.log(record.prompt);
|
||||
console.log('');
|
||||
}
|
||||
console.log(`rendered ${records.length} prompts (showing 3)`);
|
||||
console.log(`rendered ${dryRecords.length} prompts (showing 3)`);
|
||||
return;
|
||||
}
|
||||
|
||||
const aggregates = aggregate(records, args.model);
|
||||
const evalRecords = records.filter(isEvalRecord);
|
||||
const aggregates = aggregate(evalRecords, args.model);
|
||||
const variantNames = runVariants.map((variant) => variant.name);
|
||||
const report = markdownReport({
|
||||
records,
|
||||
records: evalRecords,
|
||||
aggregates,
|
||||
runCases,
|
||||
variantNames,
|
||||
model: args.model,
|
||||
samples: args.samples,
|
||||
});
|
||||
fs.mkdirSync(RESULTS_DIR, { recursive: true });
|
||||
mkdirSync(RESULTS_DIR, { recursive: true });
|
||||
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
fs.writeFileSync(
|
||||
path.join(RESULTS_DIR, `${stamp}.json`),
|
||||
JSON.stringify({ args, records }, null, 2),
|
||||
writeFileSync(
|
||||
join(RESULTS_DIR, `${stamp}.json`),
|
||||
JSON.stringify({ args, records: evalRecords }, null, 2),
|
||||
);
|
||||
fs.writeFileSync(path.join(RESULTS_DIR, 'latest.md'), report);
|
||||
writeFileSync(join(RESULTS_DIR, 'latest.md'), report);
|
||||
|
||||
console.log(report.split('## Per-case')[0]);
|
||||
console.log(`full per-case tables: scripts/activity-labels/results/latest.md`);
|
||||
})().catch((error) => {
|
||||
}
|
||||
|
||||
main().catch((error: Error) => {
|
||||
console.error('ERR', error.message);
|
||||
process.exit(1);
|
||||
});
|
||||
18
scripts/activity-labels/tsconfig.json
Normal file
18
scripts/activity-labels/tsconfig.json
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"allowImportingTsExtensions": true,
|
||||
"erasableSyntaxOnly": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"lib": ["ES2023", "DOM"],
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"noEmit": true,
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"target": "ES2023",
|
||||
"types": ["node"],
|
||||
"verbatimModuleSyntax": true
|
||||
},
|
||||
"include": ["./*.mts"]
|
||||
}
|
||||
112
scripts/activity-labels/types.mts
Normal file
112
scripts/activity-labels/types.mts
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
export type SerializableValue =
|
||||
| string
|
||||
| number
|
||||
| boolean
|
||||
| null
|
||||
| readonly SerializableValue[]
|
||||
| { readonly [key: string]: SerializableValue | undefined };
|
||||
|
||||
export interface ToolEntry {
|
||||
toolName: string;
|
||||
toolInput?: SerializableValue;
|
||||
toolOutput?: SerializableValue;
|
||||
status?: 'success' | 'error';
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface ActivityLabelPayload {
|
||||
entries: ToolEntry[];
|
||||
thinkingExcerpts?: string[];
|
||||
lastAssistantText?: string;
|
||||
}
|
||||
|
||||
export interface EvalStep {
|
||||
id?: string;
|
||||
verbatim?: string;
|
||||
productionLabel?: string;
|
||||
payload?: ActivityLabelPayload;
|
||||
}
|
||||
|
||||
export interface EvalCase {
|
||||
id: string;
|
||||
notes: string;
|
||||
steps: EvalStep[];
|
||||
}
|
||||
|
||||
export interface Variant {
|
||||
name: string;
|
||||
usePreviousLabels: boolean;
|
||||
instruction: string;
|
||||
previousLabelCap?: number;
|
||||
}
|
||||
|
||||
export interface RunArgs {
|
||||
samples: number;
|
||||
concurrency: number;
|
||||
model: string;
|
||||
dry: boolean;
|
||||
variants?: string[];
|
||||
cases?: string[];
|
||||
}
|
||||
|
||||
interface BaseRecord {
|
||||
variant: string;
|
||||
sample: number;
|
||||
caseId: string;
|
||||
stepId: string;
|
||||
}
|
||||
|
||||
export interface DryRunRecord extends BaseRecord {
|
||||
prompt: string;
|
||||
}
|
||||
|
||||
export interface ErrorRecord extends BaseRecord {
|
||||
error: string;
|
||||
}
|
||||
|
||||
export interface SuccessRecord extends BaseRecord {
|
||||
label: string;
|
||||
production?: string;
|
||||
flags: string[];
|
||||
wordCount: number;
|
||||
firstWord: string;
|
||||
latencyMs: number;
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
}
|
||||
|
||||
export type EvalRecord = ErrorRecord | SuccessRecord;
|
||||
export type RunRecord = DryRunRecord | EvalRecord;
|
||||
|
||||
export interface Aggregate {
|
||||
variant: string;
|
||||
steps: number;
|
||||
errors: number;
|
||||
flagCounts: Partial<Record<FlagType, number>>;
|
||||
distinctOpeners: number;
|
||||
topOpener: string;
|
||||
avgWords: string;
|
||||
meanLatencyMs: number;
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
costUsd: string;
|
||||
}
|
||||
|
||||
export const FLAG_TYPES = [
|
||||
'len',
|
||||
'punct',
|
||||
'quote',
|
||||
'md',
|
||||
'opener',
|
||||
'tool-echo',
|
||||
'count-echo',
|
||||
'restate',
|
||||
'template',
|
||||
] as const;
|
||||
|
||||
export type FlagType = (typeof FLAG_TYPES)[number];
|
||||
|
||||
export interface StoredResults {
|
||||
args: RunArgs;
|
||||
records: EvalRecord[];
|
||||
}
|
||||
|
|
@ -14,9 +14,14 @@
|
|||
* is impossible; the sentence table below is asserted against it so composed
|
||||
* variants can never silently diverge from what production actually sends.
|
||||
*/
|
||||
const path = require('path');
|
||||
import { join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { createRequire } from 'node:module';
|
||||
|
||||
const ROOT = path.resolve(__dirname, '..', '..');
|
||||
import type { Variant } from './types.mts';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const ROOT = fileURLToPath(new URL('../../', import.meta.url));
|
||||
|
||||
/**
|
||||
* The shipped instruction, read from the BUILT package so a variant can never
|
||||
|
|
@ -26,15 +31,19 @@ const ROOT = path.resolve(__dirname, '..', '..');
|
|||
* another checkout's build — useful for grading one branch's instruction from
|
||||
* a worktree that has not been built.
|
||||
*/
|
||||
function loadShippedInstruction() {
|
||||
interface ActivityInstructionModule {
|
||||
ACTIVITY_INSTRUCTION?: string;
|
||||
}
|
||||
|
||||
function loadShippedInstruction(): string {
|
||||
const candidates = [
|
||||
process.env.LABEL_EVAL_DIST,
|
||||
'@librechat/api',
|
||||
path.join(ROOT, 'packages/api/dist/index.cjs'),
|
||||
].filter(Boolean);
|
||||
join(ROOT, 'packages/api/dist/index.cjs'),
|
||||
].filter((candidate): candidate is string => candidate != null);
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
const { ACTIVITY_INSTRUCTION } = require(candidate);
|
||||
const { ACTIVITY_INSTRUCTION } = require(candidate) as ActivityInstructionModule;
|
||||
if (typeof ACTIVITY_INSTRUCTION === 'string' && ACTIVITY_INSTRUCTION.length > 0) {
|
||||
return ACTIVITY_INSTRUCTION;
|
||||
}
|
||||
|
|
@ -47,7 +56,7 @@ function loadShippedInstruction() {
|
|||
'Build it first (from the repo root):\n' +
|
||||
' npm run build:data-provider && npm run build:data-schemas && npm run build:api\n' +
|
||||
'Or point at an existing build:\n' +
|
||||
' LABEL_EVAL_DIST=/path/to/packages/api/dist/index.cjs node scripts/activity-labels/run.js',
|
||||
' LABEL_EVAL_DIST=/path/to/packages/api/dist/index.cjs node scripts/activity-labels/run.mts',
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -106,7 +115,7 @@ if (
|
|||
SHIPPED_ORDER.join(' ') !== ACTIVITY_INSTRUCTION
|
||||
) {
|
||||
console.warn(
|
||||
'WARN: variants.js sentence table has drifted from ACTIVITY_INSTRUCTION — composed variants are stale',
|
||||
'WARN: variants.mts sentence table has drifted from ACTIVITY_INSTRUCTION — composed variants are stale',
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -121,7 +130,7 @@ const CONTINUITY_TIGHT =
|
|||
const FORMAT_HARD =
|
||||
'Write 4 to 9 words, sentence case, no trailing punctuation, no quotes or markdown; when a batch found many things, keep only the most load-bearing one or two.';
|
||||
|
||||
const variants = [
|
||||
export const variants: Variant[] = [
|
||||
{
|
||||
name: 'baseline',
|
||||
usePreviousLabels: SHIPPED_ORDER.join(' ') === ACTIVITY_INSTRUCTION,
|
||||
|
|
@ -235,5 +244,3 @@ const variants = [
|
|||
].join(' '),
|
||||
},
|
||||
];
|
||||
|
||||
module.exports = { variants };
|
||||
Loading…
Add table
Add a link
Reference in a new issue