diff --git a/scripts/activity-labels/README.md b/scripts/activity-labels/README.md index e98d6dfd73..01e5a90984 100644 --- a/scripts/activity-labels/README.md +++ b/scripts/activity-labels/README.md @@ -7,11 +7,12 @@ hypotheses that felt most obvious when this was built both turned out wrong under measurement (see **Findings**). ```bash -node scripts/activity-labels/run.js # every variant, 1 sample -node scripts/activity-labels/run.js --samples 3 # 3 samples each -node scripts/activity-labels/run.js --variants baseline,shipping --cases fib-rapid -node scripts/activity-labels/run.js --dry --cases mega-batch # render prompts, no API calls -node scripts/activity-labels/rescore.js # re-grade stored results, no re-spend +node scripts/activity-labels/run.mts # every variant, 1 sample +node scripts/activity-labels/run.mts --samples 3 # 3 samples each +node scripts/activity-labels/run.mts --variants baseline,shipping --cases fib-rapid +node scripts/activity-labels/run.mts --dry --cases mega-batch # render prompts, no API calls +node scripts/activity-labels/rescore.mts # re-grade stored results, no re-spend +npx tsc -p scripts/activity-labels/tsconfig.json # type-check the harness ``` Requires `ANTHROPIC_API_KEY` (env or `.env`). A full sweep is roughly $0.03 per @@ -28,14 +29,16 @@ aggregate is the regression guard. ## Layout -| File | Role | -| ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `captured.json` | 9 real production payloads pulled verbatim from Langfuse, with the headers that shipped. Irreplaceable — traces age out. | -| `corpus.js` | 17 cases / 28 steps: the captured run as one sequence, plus synthetic cases for modes it never hit (all-failed, partial, parallel, rapid duplicates, entry overflow, truncated output, error-shaped success). Multi-step cases chain each generated label into the next step's context. | -| `prompt.js` | Faithful port of the SDK's `buildActivityLabelPrompt`, so synthetic cases render the bytes production would send. Adds `previousLabelCap` for continuity-window experiments. | -| `variants.js` | Single-factor instruction variants. `baseline` is read from the built `packages/api` dist, so drift from the shipped instruction is impossible. | -| `checks.js` | Mechanical grading: length, punctuation, markdown, tool-name echo, count echo, and overlap split into `restate` (adds nothing) vs `template` (same frame, new payload). | -| `run.js` / `rescore.js` | Live runner (production wire shape, `max_tokens: 256`) and offline re-grader. | +| File | Role | +| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `captured.json` | 9 real production payloads pulled verbatim from Langfuse, with the headers that shipped. Irreplaceable — traces age out. | +| `corpus.mts` | 17 cases / 28 steps: the captured run as one sequence, plus synthetic cases for modes it never hit (all-failed, partial, parallel, rapid duplicates, entry overflow, truncated output, error-shaped success). Multi-step cases chain each generated label into the next step's context. | +| `prompt.mts` | Faithful port of the SDK's `buildActivityLabelPrompt`, so synthetic cases render the bytes production would send. Adds `previousLabelCap` for continuity-window experiments. | +| `variants.mts` | Single-factor instruction variants. `baseline` is read from the built `packages/api` dist, so drift from the shipped instruction is impossible. | +| `checks.mts` | Mechanical grading: length, punctuation, markdown, tool-name echo, count echo, and overlap split into `restate` (adds nothing) vs `template` (same frame, new payload). | +| `run.mts` / `rescore.mts` | Live runner (production wire shape, `max_tokens: 256`) and offline re-grader. | +| `types.mts` | Shared corpus, variant, result, and report types used by the runner and rescorer. | +| `tsconfig.json` | Strict, no-emit type-checking configuration for the harness. | ## Findings this produced diff --git a/scripts/activity-labels/checks.js b/scripts/activity-labels/checks.mts similarity index 83% rename from scripts/activity-labels/checks.js rename to scripts/activity-labels/checks.mts index 8acbea7b88..5260c5f8c5 100644 --- a/scripts/activity-labels/checks.js +++ b/scripts/activity-labels/checks.mts @@ -4,7 +4,9 @@ * cross-batch redundancy via content-word overlap); commit-log READABILITY * still needs the human pass over results/latest.md. */ -const STOPWORDS = new Set([ +import type { ToolEntry } from './types.mts'; + +const STOPWORDS = new Set([ 'the', 'a', 'an', @@ -37,7 +39,7 @@ const STOPWORDS = new Set([ 'via', ]); -function words(label) { +export function words(label: string): string[] { return label.trim().split(/\s+/).filter(Boolean); } @@ -45,7 +47,7 @@ const SUFFIXES = ['ations', 'ation', 'ence', 'ance', 'ings', 'ing', 'ed', 'es', /** Crude suffix stemmer so persists/persistence/persisted collide — enough * for overlap detection; linguistic correctness is not the goal. */ -function stem(word) { +export function stem(word: string): string { if (word.length < 5) { return word; } @@ -57,7 +59,7 @@ function stem(word) { return word; } -function contentWords(label) { +export function contentWords(label: string): string[] { return words(label.toLowerCase().replace(/[^a-z0-9/._-]+/g, ' ')) .filter((word) => !STOPWORDS.has(word)) .map(stem); @@ -65,11 +67,11 @@ function contentWords(label) { /** Payload tokens carry the informative delta between template-shaped * labels: numbers, versions, paths, filenames. */ -function isPayload(word) { +function isPayload(word: string): boolean { return /\d/.test(word) || word.includes('/') || word.includes('.'); } -function jaccard(a, b) { +export function jaccard(a: readonly string[], b: readonly string[]): number { const setA = new Set(a); const setB = new Set(b); if (setA.size === 0 || setB.size === 0) { @@ -95,8 +97,23 @@ const DUP_THRESHOLD = 0.5; * regardless of whether the variant saw them — redundancy is measured * uniformly so continuity variants can be compared against blind ones. */ -function checkLabel(label, { entries = [], previousLabels = [] } = {}) { - const flags = []; +interface CheckLabelOptions { + entries?: readonly ToolEntry[]; + previousLabels?: readonly string[]; +} + +interface CheckLabelResult { + flags: string[]; + wordCount: number; + firstWord: string; + maxOverlap: number; +} + +export function checkLabel( + label: string, + { entries = [], previousLabels = [] }: CheckLabelOptions = {}, +): CheckLabelResult { + const flags: string[] = []; const wordList = words(label); if (wordList.length < 4 || wordList.length > 9) { flags.push(`len:${wordList.length}`); @@ -115,7 +132,7 @@ function checkLabel(label, { entries = [], previousLabels = [] } = {}) { } const lower = label.toLowerCase(); for (const entry of entries) { - const name = String(entry.toolName ?? '').toLowerCase(); + const name = entry.toolName.toLowerCase(); if (name.length > 3 && (lower.includes(name) || lower.includes(name.replace(/_/g, ' ')))) { flags.push(`tool-echo:${entry.toolName}`); break; @@ -131,7 +148,7 @@ function checkLabel(label, { entries = [], previousLabels = [] } = {}) { * fib(1)→fib(2). Often fine, arguably better than synonym churn). */ const own = contentWords(label); let maxOverlap = 0; - let worst = null; + let worst: string[] | null = null; for (const previous of previousLabels) { const other = contentWords(previous); const overlap = jaccard(own, other); @@ -152,5 +169,3 @@ function checkLabel(label, { entries = [], previousLabels = [] } = {}) { } return { flags, wordCount: wordList.length, firstWord: wordList[0] ?? '', maxOverlap }; } - -module.exports = { checkLabel, contentWords, jaccard, words, stem }; diff --git a/scripts/activity-labels/corpus.js b/scripts/activity-labels/corpus.mts similarity index 96% rename from scripts/activity-labels/corpus.js rename to scripts/activity-labels/corpus.mts index bacec6104c..91d1fb1d3d 100644 --- a/scripts/activity-labels/corpus.js +++ b/scripts/activity-labels/corpus.mts @@ -14,12 +14,21 @@ * step's generated label into the next step's `previousLabels` for variants * that opt in. */ -const fs = require('fs'); -const path = require('path'); +import { readFileSync } from 'node:fs'; -const captured = JSON.parse(fs.readFileSync(path.join(__dirname, 'captured.json'), 'utf8')); +import type { EvalCase, EvalStep, ToolEntry } from './types.mts'; -const capturedRun = { +interface CapturedEntry { + id: string; + prompt: string; + productionLabel: string; +} + +const captured = JSON.parse( + readFileSync(new URL('./captured.json', import.meta.url), 'utf8'), +) as CapturedEntry[]; + +const capturedRun: EvalCase = { id: 'sandbox-probe-run', notes: 'the real 9-batch production run, verbatim payloads', steps: captured.map((entry) => ({ @@ -29,7 +38,7 @@ const capturedRun = { })), }; -const synthetic = [ +const synthetic: EvalCase[] = [ { id: 'all-failed', notes: 'every call fails — failure register, verb-first under failure', @@ -443,13 +452,13 @@ const synthetic = [ /** Tool names for echo checks; captured steps bake entries into the * verbatim prompt, so they are recovered from the "Tool calls:" lines. */ -function stepEntries(step) { +export function stepEntries(step: EvalStep): ToolEntry[] { if (step.payload?.entries) { return step.payload.entries; } return [...(step.verbatim ?? '').matchAll(/^- ([A-Za-z0-9_]+)\(/gm)].map((match) => ({ - toolName: match[1], + toolName: match[1] ?? '', })); } -module.exports = { cases: [capturedRun, ...synthetic], stepEntries }; +export const cases: EvalCase[] = [capturedRun, ...synthetic]; diff --git a/scripts/activity-labels/prompt.js b/scripts/activity-labels/prompt.mts similarity index 79% rename from scripts/activity-labels/prompt.js rename to scripts/activity-labels/prompt.mts index f67fcfcae5..1eb653ab74 100644 --- a/scripts/activity-labels/prompt.js +++ b/scripts/activity-labels/prompt.mts @@ -12,12 +12,14 @@ * unless a variant opts in. This is the P1 continuity hypothesis — it lets * the harness measure the fix before any SDK field exists. */ +import type { EvalStep, SerializableValue, ToolEntry } from './types.mts'; + const INPUT_CONTEXT_LIMIT = 200; const MAX_THINKING_EXCERPTS = 4; const MAX_PROMPT_ENTRIES = 12; const MAX_PREVIOUS_LABELS = 3; -function truncateForLabel(value, maxLength) { +export function truncateForLabel(value: string, maxLength: number): string { if (value.length <= maxLength) { return value; } @@ -26,7 +28,7 @@ function truncateForLabel(value, maxLength) { const ABORT_SERIALIZATION = Symbol('abort-label-serialization'); -function serializeForLabel(value, limit) { +export function serializeForLabel(value: SerializableValue | undefined, limit: number): string { if (value == null) { return ''; } @@ -36,7 +38,7 @@ function serializeForLabel(value, limit) { let budget = limit * 4; try { return ( - JSON.stringify(value, (_key, nested) => { + JSON.stringify(value, (_key: string, nested: SerializableValue) => { if (budget <= 0) { throw ABORT_SERIALIZATION; } @@ -59,7 +61,10 @@ function serializeForLabel(value, limit) { /** `cap` of Infinity models the unbounded-history alternative — testing * whether the whole run's story beats a recency window. */ -function previousHeadersSection(previousLabels, cap = MAX_PREVIOUS_LABELS) { +function previousHeadersSection( + previousLabels: readonly string[], + cap = MAX_PREVIOUS_LABELS, +): string | null { const kept = previousLabels.filter(Boolean); const recent = Number.isFinite(cap) ? kept.slice(-cap) : kept; if (recent.length === 0) { @@ -71,14 +76,23 @@ function previousHeadersSection(previousLabels, cap = MAX_PREVIOUS_LABELS) { ); } -function buildActivityLabelPrompt({ +interface BuildPromptOptions { + entries: readonly ToolEntry[]; + charLimit: number; + thinkingExcerpts?: readonly string[]; + lastAssistantText?: string; + previousLabels?: readonly string[] | null; + previousLabelCap?: number; +} + +export function buildActivityLabelPrompt({ entries, charLimit, thinkingExcerpts, lastAssistantText, previousLabels, previousLabelCap, -}) { +}: BuildPromptOptions): string { const clip = truncateForLabel; const sections = []; if (previousLabels != null) { @@ -128,12 +142,24 @@ function buildActivityLabelPrompt({ * prompt (byte-exact from Langfuse); the continuity section, when a variant * opts in, is prepended — the same position the built path gives it. */ -function renderStepPrompt(step, { charLimit, previousLabels, previousLabelCap }) { +interface RenderPromptOptions { + charLimit: number; + previousLabels: readonly string[] | null; + previousLabelCap?: number; +} + +export function renderStepPrompt( + step: EvalStep, + { charLimit, previousLabels, previousLabelCap }: RenderPromptOptions, +): string { if (step.verbatim != null) { const section = previousLabels != null ? previousHeadersSection(previousLabels, previousLabelCap) : null; return section != null ? `${section}\n\n${step.verbatim}` : step.verbatim; } + if (step.payload == null) { + throw new Error('corpus step must define either verbatim or payload'); + } return buildActivityLabelPrompt({ ...step.payload, charLimit, @@ -142,10 +168,4 @@ function renderStepPrompt(step, { charLimit, previousLabels, previousLabelCap }) }); } -module.exports = { - buildActivityLabelPrompt, - renderStepPrompt, - truncateForLabel, - serializeForLabel, - MAX_PREVIOUS_LABELS, -}; +export { MAX_PREVIOUS_LABELS }; diff --git a/scripts/activity-labels/report.js b/scripts/activity-labels/report.mts similarity index 69% rename from scripts/activity-labels/report.js rename to scripts/activity-labels/report.mts index e06eb14413..6e7972ddfc 100644 --- a/scripts/activity-labels/report.js +++ b/scripts/activity-labels/report.mts @@ -1,28 +1,56 @@ /** Aggregation + markdown rendering, shared by the live runner and the * offline rescorer so metric fixes never require re-spending on the API. */ -const FLAG_TYPES = [ - 'len', - 'punct', - 'quote', - 'md', - 'opener', - 'tool-echo', - 'count-echo', - 'restate', - 'template', -]; +import type { Aggregate, ErrorRecord, EvalCase, EvalRecord, FlagType } from './types.mts'; +import { FLAG_TYPES } from './types.mts'; -const PRICES = { 'claude-haiku-4-5': { input: 1, output: 5 } }; - -function flagType(flag) { - return flag.split(':')[0]; +interface Price { + input: number; + output: number; } -function aggregate(records, model) { - const byVariant = new Map(); +interface AggregateState { + steps: number; + errors: number; + flagCounts: Partial>; + firstWords: Record; + totalWords: number; + latencies: number[]; + inputTokens: number; + outputTokens: number; +} + +interface MarkdownReportOptions { + records: readonly EvalRecord[]; + aggregates: readonly Aggregate[]; + runCases: readonly EvalCase[]; + variantNames: readonly string[]; + model: string; + samples: number; +} + +const PRICES: Readonly> = { + 'claude-haiku-4-5': { input: 1, output: 5 }, +}; +const FLAG_TYPE_SET = new Set(FLAG_TYPES); + +function flagType(flag: string): FlagType { + const type = flag.split(':')[0] ?? ''; + if (!FLAG_TYPE_SET.has(type)) { + throw new Error(`unknown flag type: ${type}`); + } + return type as FlagType; +} + +function isErrorRecord(record: EvalRecord): record is ErrorRecord { + return 'error' in record; +} + +export function aggregate(records: readonly EvalRecord[], model: string): Aggregate[] { + const byVariant = new Map(); for (const record of records) { - if (!byVariant.has(record.variant)) { - byVariant.set(record.variant, { + let agg = byVariant.get(record.variant); + if (agg == null) { + agg = { steps: 0, errors: 0, flagCounts: {}, @@ -31,10 +59,10 @@ function aggregate(records, model) { latencies: [], inputTokens: 0, outputTokens: 0, - }); + }; + byVariant.set(record.variant, agg); } - const agg = byVariant.get(record.variant); - if (record.error) { + if (isErrorRecord(record)) { agg.errors += 1; continue; } @@ -52,7 +80,7 @@ function aggregate(records, model) { const price = PRICES[model]; return [...byVariant.entries()].map(([name, agg]) => { const sortedFirst = Object.entries(agg.firstWords).sort((a, b) => b[1] - a[1]); - const topOpener = sortedFirst[0] ?? ['—', 0]; + const topOpener: [string, number] = sortedFirst[0] ?? ['—', 0]; return { variant: name, steps: agg.steps, @@ -73,8 +101,15 @@ function aggregate(records, model) { }); } -function markdownReport({ records, aggregates, runCases, variantNames, model, samples }) { - const lines = []; +export function markdownReport({ + records, + aggregates, + runCases, + variantNames, + model, + samples, +}: MarkdownReportOptions): string { + const lines: string[] = []; lines.push(`# Activity-label eval — ${new Date().toISOString()}`); lines.push(''); lines.push(`model: \`${model}\` · samples: ${samples} · cases: ${runCases.length}`); @@ -100,7 +135,9 @@ function markdownReport({ records, aggregates, runCases, variantNames, model, sa lines.push(''); lines.push(`*${testCase.notes}*`); lines.push(''); - const sampleList = [...new Set(records.map((r) => r.sample))].sort(); + const sampleList = [...new Set(records.map((record) => record.sample))].sort( + (first, second) => first - second, + ); const header = ['step']; if (samples > 1) { header.push('s'); @@ -131,7 +168,7 @@ function markdownReport({ records, aggregates, runCases, variantNames, model, sa ); if (!record) { row.push(''); - } else if (record.error) { + } else if (isErrorRecord(record)) { row.push(`⛔ ${record.error}`); } else { const flagNote = record.flags.length > 0 ? ` ⚠${record.flags.join(' ⚠')}` : ''; @@ -144,5 +181,3 @@ function markdownReport({ records, aggregates, runCases, variantNames, model, sa } return lines.join('\n') + '\n'; } - -module.exports = { aggregate, markdownReport, FLAG_TYPES }; diff --git a/scripts/activity-labels/rescore.js b/scripts/activity-labels/rescore.js deleted file mode 100644 index 30922970d8..0000000000 Binary files a/scripts/activity-labels/rescore.js and /dev/null differ diff --git a/scripts/activity-labels/rescore.mts b/scripts/activity-labels/rescore.mts new file mode 100644 index 0000000000..fa2853523b --- /dev/null +++ b/scripts/activity-labels/rescore.mts @@ -0,0 +1,77 @@ +/** + * Offline rescore: recompute checks over a stored results JSON after a + * metric change, without re-calling the API. Chains are rebuilt from the + * stored labels in push order (steps within a case ran serially). + * + * Usage: node scripts/activity-labels/rescore.mts [results/.json] + */ +import { fileURLToPath } from 'node:url'; +import { basename, join, resolve } from 'node:path'; +import { readdirSync, readFileSync, writeFileSync } from 'node:fs'; + +import type { EvalStep, StoredResults } from './types.mts'; +import { aggregate, markdownReport } from './report.mts'; +import { cases, stepEntries } from './corpus.mts'; +import { checkLabel } from './checks.mts'; + +const RESULTS_DIR = fileURLToPath(new URL('./results', import.meta.url)); + +function newestResults(): string { + const files = readdirSync(RESULTS_DIR) + .filter((file) => file.endsWith('.json')) + .sort(); + const newest = files.at(-1); + if (newest == null) { + throw new Error('no stored results to rescore'); + } + return join(RESULTS_DIR, newest); +} + +const sourcePath = process.argv[2] ? resolve(process.argv[2]) : newestResults(); +const { args, records } = JSON.parse(readFileSync(sourcePath, 'utf8')) as StoredResults; + +const stepsByCase = new Map>( + cases.map((testCase) => [ + testCase.id, + new Map(testCase.steps.map((step) => [step.id ?? testCase.id, step])), + ]), +); + +const chains = new Map(); +for (const record of records) { + if ('error' in record) { + continue; + } + const key = `${record.variant}\0${record.sample}\0${record.caseId}`; + let chain = chains.get(key); + if (chain == null) { + chain = []; + chains.set(key, chain); + } + const step = stepsByCase.get(record.caseId)?.get(record.stepId); + const { flags, wordCount, firstWord } = checkLabel(record.label, { + entries: step != null ? stepEntries(step) : [], + previousLabels: chain, + }); + record.flags = flags; + record.wordCount = wordCount; + record.firstWord = firstWord; + chain.push(record.label); +} + +const variantNames = [...new Set(records.map((record) => record.variant))]; +const runCases = cases.filter((testCase) => + records.some((record) => record.caseId === testCase.id), +); +const report = markdownReport({ + records, + aggregates: aggregate(records, args.model), + runCases, + variantNames, + model: args.model, + samples: args.samples, +}); +writeFileSync(join(RESULTS_DIR, 'latest.md'), report); +console.log(`rescored ${basename(sourcePath)}`); +console.log(report.split('## Per-case')[0]); +console.log('full per-case tables: scripts/activity-labels/results/latest.md'); diff --git a/scripts/activity-labels/run.js b/scripts/activity-labels/run.mts similarity index 58% rename from scripts/activity-labels/run.js rename to scripts/activity-labels/run.mts index 7035a1362d..980e295fb1 100644 --- a/scripts/activity-labels/run.js +++ b/scripts/activity-labels/run.mts @@ -11,60 +11,126 @@ * continuity variants share one redundancy metric. * * Usage: - * node scripts/activity-labels/run.js [--variants baseline,continuity] + * node scripts/activity-labels/run.mts [--variants baseline,continuity] * [--cases sandbox-probe-run,fib-rapid] [--samples 2] [--model id] * [--concurrency 6] [--dry] */ -const fs = require('fs'); -const path = require('path'); +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; -const { cases, stepEntries } = require('./corpus'); -const { variants } = require('./variants'); -const { checkLabel } = require('./checks'); -const { renderStepPrompt } = require('./prompt'); -const { aggregate, markdownReport } = require('./report'); +import type { DryRunRecord, EvalCase, EvalRecord, RunArgs, RunRecord, Variant } from './types.mts'; +import { aggregate, markdownReport } from './report.mts'; +import { cases, stepEntries } from './corpus.mts'; +import { renderStepPrompt } from './prompt.mts'; +import { checkLabel } from './checks.mts'; +import { variants } from './variants.mts'; -const ROOT = path.resolve(__dirname, '..', '..'); -const RESULTS_DIR = path.join(__dirname, 'results'); +const ROOT = fileURLToPath(new URL('../../', import.meta.url)); +const RESULTS_DIR = fileURLToPath(new URL('./results', import.meta.url)); const CHAR_LIMIT = 600; const MAX_TOKENS = 256; -function parseArgs(argv) { - const args = { samples: 1, concurrency: 6, model: 'claude-haiku-4-5', dry: false }; +interface AnthropicMessageResponse { + content?: Array<{ text?: string }>; + usage?: { + input_tokens?: number; + output_tokens?: number; + }; +} + +interface RequestLabelOptions { + apiKey: string; + model: string; + instruction: string; + prompt: string; +} + +type RequestLabelResult = + | { + label: string; + latencyMs: number; + inputTokens: number; + outputTokens: number; + } + | { + error: string; + latencyMs: number; + }; + +interface RunCaseOptions { + apiKey: string; + model: string; + variant: Variant; + sample: number; + testCase: EvalCase; + dry: boolean; + records: RunRecord[]; +} + +type Task = () => Promise; + +function nextArgument(argv: readonly string[], index: number, key: string): string { + const value = argv[index + 1]; + if (value == null || value.startsWith('--')) { + throw new Error(`${key} requires a value`); + } + return value; +} + +function positiveInteger(value: string, key: string): number { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`${key} must be a positive integer`); + } + return parsed; +} + +function parseArgs(argv: readonly string[]): RunArgs { + const args: RunArgs = { + samples: 1, + concurrency: 6, + model: 'claude-haiku-4-5', + dry: false, + }; for (let i = 0; i < argv.length; i++) { const key = argv[i]; if (key === '--dry') { args.dry = true; } else if (key === '--variants') { - args.variants = argv[++i].split(','); + args.variants = nextArgument(argv, i, key).split(','); + i += 1; } else if (key === '--cases') { - args.cases = argv[++i].split(','); + args.cases = nextArgument(argv, i, key).split(','); + i += 1; } else if (key === '--samples') { - args.samples = Number(argv[++i]); + args.samples = positiveInteger(nextArgument(argv, i, key), key); + i += 1; } else if (key === '--model') { - args.model = argv[++i]; + args.model = nextArgument(argv, i, key); + i += 1; } else if (key === '--concurrency') { - args.concurrency = Number(argv[++i]); + args.concurrency = positiveInteger(nextArgument(argv, i, key), key); + i += 1; } } return args; } -function loadKey() { +function loadKey(): string { if (process.env.ANTHROPIC_API_KEY) { return process.env.ANTHROPIC_API_KEY; } - const envPath = path.join(ROOT, '.env'); - const line = fs.existsSync(envPath) - ? fs - .readFileSync(envPath, 'utf8') + const envPath = join(ROOT, '.env'); + const line = existsSync(envPath) + ? readFileSync(envPath, 'utf8') .split('\n') .find((entry) => entry.startsWith('ANTHROPIC_API_KEY=')) : undefined; if (!line) { throw new Error( `ANTHROPIC_API_KEY not set and not found in ${envPath}.\n` + - 'Pass it inline: ANTHROPIC_API_KEY=sk-… node scripts/activity-labels/run.js', + 'Pass it inline: ANTHROPIC_API_KEY=sk-… node scripts/activity-labels/run.mts', ); } return line @@ -73,7 +139,12 @@ function loadKey() { .replace(/^["']|["']$/g, ''); } -async function requestLabel({ apiKey, model, instruction, prompt }) { +async function requestLabel({ + apiKey, + model, + instruction, + prompt, +}: RequestLabelOptions): Promise { for (let attempt = 1; attempt <= 3; attempt++) { const started = Date.now(); const response = await fetch('https://api.anthropic.com/v1/messages', { @@ -91,7 +162,7 @@ async function requestLabel({ apiKey, model, instruction, prompt }) { }), }); if (response.ok) { - const json = await response.json(); + const json = (await response.json()) as AnthropicMessageResponse; const label = (json.content ?? []) .map((block) => block.text ?? '') .join('') @@ -117,11 +188,20 @@ async function requestLabel({ apiKey, model, instruction, prompt }) { latencyMs: Date.now() - started, }; } + throw new Error('label request exhausted retries without a result'); } /** One case chain: steps serial, labels feeding forward. */ -async function runCase({ apiKey, model, variant, sample, testCase, dry, records }) { - const chain = []; +async function runCase({ + apiKey, + model, + variant, + sample, + testCase, + dry, + records, +}: RunCaseOptions): Promise { + const chain: string[] = []; for (const step of testCase.steps) { const prompt = renderStepPrompt(step, { charLimit: CHAR_LIMIT, @@ -134,7 +214,7 @@ async function runCase({ apiKey, model, variant, sample, testCase, dry, records continue; } const result = await requestLabel({ apiKey, model, instruction: variant.instruction, prompt }); - if (result.error) { + if ('error' in result) { records.push({ variant: variant.name, sample, @@ -166,28 +246,43 @@ async function runCase({ apiKey, model, variant, sample, testCase, dry, records } } -async function pool(tasks, size) { +async function pool(tasks: readonly Task[], size: number): Promise { const queue = [...tasks]; const workers = Array.from({ length: Math.min(size, queue.length) }, async () => { while (queue.length > 0) { - await queue.shift()(); + const task = queue.shift(); + if (task != null) { + await task(); + } } }); await Promise.all(workers); } -(async () => { +function isDryRunRecord(record: RunRecord): record is DryRunRecord { + return 'prompt' in record; +} + +function isEvalRecord(record: RunRecord): record is EvalRecord { + return !isDryRunRecord(record); +} + +async function main(): Promise { const args = parseArgs(process.argv.slice(2)); - const runVariants = args.variants - ? variants.filter((variant) => args.variants.includes(variant.name)) + const selectedVariants = args.variants; + const selectedCases = args.cases; + const runVariants = selectedVariants + ? variants.filter((variant) => selectedVariants.includes(variant.name)) : variants; - const runCases = args.cases ? cases.filter((c) => args.cases.includes(c.id)) : cases; + const runCases = selectedCases + ? cases.filter((testCase) => selectedCases.includes(testCase.id)) + : cases; if (runVariants.length === 0 || runCases.length === 0) { throw new Error('nothing selected — check --variants / --cases names'); } const apiKey = args.dry ? '' : loadKey(); - const records = []; - const tasks = []; + const records: RunRecord[] = []; + const tasks: Task[] = []; for (const variant of runVariants) { for (let sample = 1; sample <= args.samples; sample++) { for (const testCase of runCases) { @@ -206,36 +301,40 @@ async function pool(tasks, size) { console.log(`done in ${((Date.now() - started) / 1000).toFixed(1)}s\n`); if (args.dry) { - for (const record of records.slice(0, 3)) { + const dryRecords = records.filter(isDryRunRecord); + for (const record of dryRecords.slice(0, 3)) { console.log(`--- ${record.variant} / ${record.caseId} / ${record.stepId} ---`); console.log(record.prompt); console.log(''); } - console.log(`rendered ${records.length} prompts (showing 3)`); + console.log(`rendered ${dryRecords.length} prompts (showing 3)`); return; } - const aggregates = aggregate(records, args.model); + const evalRecords = records.filter(isEvalRecord); + const aggregates = aggregate(evalRecords, args.model); const variantNames = runVariants.map((variant) => variant.name); const report = markdownReport({ - records, + records: evalRecords, aggregates, runCases, variantNames, model: args.model, samples: args.samples, }); - fs.mkdirSync(RESULTS_DIR, { recursive: true }); + mkdirSync(RESULTS_DIR, { recursive: true }); const stamp = new Date().toISOString().replace(/[:.]/g, '-'); - fs.writeFileSync( - path.join(RESULTS_DIR, `${stamp}.json`), - JSON.stringify({ args, records }, null, 2), + writeFileSync( + join(RESULTS_DIR, `${stamp}.json`), + JSON.stringify({ args, records: evalRecords }, null, 2), ); - fs.writeFileSync(path.join(RESULTS_DIR, 'latest.md'), report); + writeFileSync(join(RESULTS_DIR, 'latest.md'), report); console.log(report.split('## Per-case')[0]); console.log(`full per-case tables: scripts/activity-labels/results/latest.md`); -})().catch((error) => { +} + +main().catch((error: Error) => { console.error('ERR', error.message); process.exit(1); }); diff --git a/scripts/activity-labels/tsconfig.json b/scripts/activity-labels/tsconfig.json new file mode 100644 index 0000000000..15002c43d4 --- /dev/null +++ b/scripts/activity-labels/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "allowImportingTsExtensions": true, + "erasableSyntaxOnly": true, + "forceConsistentCasingInFileNames": true, + "lib": ["ES2023", "DOM"], + "module": "NodeNext", + "moduleResolution": "NodeNext", + "noEmit": true, + "noUncheckedIndexedAccess": true, + "skipLibCheck": true, + "strict": true, + "target": "ES2023", + "types": ["node"], + "verbatimModuleSyntax": true + }, + "include": ["./*.mts"] +} diff --git a/scripts/activity-labels/types.mts b/scripts/activity-labels/types.mts new file mode 100644 index 0000000000..495ae00822 --- /dev/null +++ b/scripts/activity-labels/types.mts @@ -0,0 +1,112 @@ +export type SerializableValue = + | string + | number + | boolean + | null + | readonly SerializableValue[] + | { readonly [key: string]: SerializableValue | undefined }; + +export interface ToolEntry { + toolName: string; + toolInput?: SerializableValue; + toolOutput?: SerializableValue; + status?: 'success' | 'error'; + error?: string; +} + +export interface ActivityLabelPayload { + entries: ToolEntry[]; + thinkingExcerpts?: string[]; + lastAssistantText?: string; +} + +export interface EvalStep { + id?: string; + verbatim?: string; + productionLabel?: string; + payload?: ActivityLabelPayload; +} + +export interface EvalCase { + id: string; + notes: string; + steps: EvalStep[]; +} + +export interface Variant { + name: string; + usePreviousLabels: boolean; + instruction: string; + previousLabelCap?: number; +} + +export interface RunArgs { + samples: number; + concurrency: number; + model: string; + dry: boolean; + variants?: string[]; + cases?: string[]; +} + +interface BaseRecord { + variant: string; + sample: number; + caseId: string; + stepId: string; +} + +export interface DryRunRecord extends BaseRecord { + prompt: string; +} + +export interface ErrorRecord extends BaseRecord { + error: string; +} + +export interface SuccessRecord extends BaseRecord { + label: string; + production?: string; + flags: string[]; + wordCount: number; + firstWord: string; + latencyMs: number; + inputTokens: number; + outputTokens: number; +} + +export type EvalRecord = ErrorRecord | SuccessRecord; +export type RunRecord = DryRunRecord | EvalRecord; + +export interface Aggregate { + variant: string; + steps: number; + errors: number; + flagCounts: Partial>; + distinctOpeners: number; + topOpener: string; + avgWords: string; + meanLatencyMs: number; + inputTokens: number; + outputTokens: number; + costUsd: string; +} + +export const FLAG_TYPES = [ + 'len', + 'punct', + 'quote', + 'md', + 'opener', + 'tool-echo', + 'count-echo', + 'restate', + 'template', +] as const; + +export type FlagType = (typeof FLAG_TYPES)[number]; + +export interface StoredResults { + args: RunArgs; + records: EvalRecord[]; +} diff --git a/scripts/activity-labels/variants.js b/scripts/activity-labels/variants.mts similarity index 90% rename from scripts/activity-labels/variants.js rename to scripts/activity-labels/variants.mts index f9ba3ca549..9c9096fa55 100644 --- a/scripts/activity-labels/variants.js +++ b/scripts/activity-labels/variants.mts @@ -14,9 +14,14 @@ * is impossible; the sentence table below is asserted against it so composed * variants can never silently diverge from what production actually sends. */ -const path = require('path'); +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createRequire } from 'node:module'; -const ROOT = path.resolve(__dirname, '..', '..'); +import type { Variant } from './types.mts'; + +const require = createRequire(import.meta.url); +const ROOT = fileURLToPath(new URL('../../', import.meta.url)); /** * The shipped instruction, read from the BUILT package so a variant can never @@ -26,15 +31,19 @@ const ROOT = path.resolve(__dirname, '..', '..'); * another checkout's build — useful for grading one branch's instruction from * a worktree that has not been built. */ -function loadShippedInstruction() { +interface ActivityInstructionModule { + ACTIVITY_INSTRUCTION?: string; +} + +function loadShippedInstruction(): string { const candidates = [ process.env.LABEL_EVAL_DIST, '@librechat/api', - path.join(ROOT, 'packages/api/dist/index.cjs'), - ].filter(Boolean); + join(ROOT, 'packages/api/dist/index.cjs'), + ].filter((candidate): candidate is string => candidate != null); for (const candidate of candidates) { try { - const { ACTIVITY_INSTRUCTION } = require(candidate); + const { ACTIVITY_INSTRUCTION } = require(candidate) as ActivityInstructionModule; if (typeof ACTIVITY_INSTRUCTION === 'string' && ACTIVITY_INSTRUCTION.length > 0) { return ACTIVITY_INSTRUCTION; } @@ -47,7 +56,7 @@ function loadShippedInstruction() { 'Build it first (from the repo root):\n' + ' npm run build:data-provider && npm run build:data-schemas && npm run build:api\n' + 'Or point at an existing build:\n' + - ' LABEL_EVAL_DIST=/path/to/packages/api/dist/index.cjs node scripts/activity-labels/run.js', + ' LABEL_EVAL_DIST=/path/to/packages/api/dist/index.cjs node scripts/activity-labels/run.mts', ); } @@ -106,7 +115,7 @@ if ( SHIPPED_ORDER.join(' ') !== ACTIVITY_INSTRUCTION ) { console.warn( - 'WARN: variants.js sentence table has drifted from ACTIVITY_INSTRUCTION — composed variants are stale', + 'WARN: variants.mts sentence table has drifted from ACTIVITY_INSTRUCTION — composed variants are stale', ); } @@ -121,7 +130,7 @@ const CONTINUITY_TIGHT = const FORMAT_HARD = 'Write 4 to 9 words, sentence case, no trailing punctuation, no quotes or markdown; when a batch found many things, keep only the most load-bearing one or two.'; -const variants = [ +export const variants: Variant[] = [ { name: 'baseline', usePreviousLabels: SHIPPED_ORDER.join(' ') === ACTIVITY_INSTRUCTION, @@ -235,5 +244,3 @@ const variants = [ ].join(' '), }, ]; - -module.exports = { variants };