diff --git a/.gitignore b/.gitignore index 89d6c2e99a..836745d271 100644 --- a/.gitignore +++ b/.gitignore @@ -183,3 +183,4 @@ hive-mind-prompt-*.txt CLAUDE.md .gsd codedb.snapshot +scripts/activity-labels/results/ diff --git a/scripts/activity-labels/README.md b/scripts/activity-labels/README.md new file mode 100644 index 0000000000..e98d6dfd73 --- /dev/null +++ b/scripts/activity-labels/README.md @@ -0,0 +1,57 @@ +# Activity-label eval harness + +Measures the prose quality of fast-model activity-label headers (the collapsed +group headers produced by `activityLabel`). Instruction changes are graded +against a fixed corpus instead of eyeballed on one conversation — the two +hypotheses that felt most obvious when this was built both turned out wrong +under measurement (see **Findings**). + +```bash +node scripts/activity-labels/run.js # every variant, 1 sample +node scripts/activity-labels/run.js --samples 3 # 3 samples each +node scripts/activity-labels/run.js --variants baseline,shipping --cases fib-rapid +node scripts/activity-labels/run.js --dry --cases mega-batch # render prompts, no API calls +node scripts/activity-labels/rescore.js # re-grade stored results, no re-spend +``` + +Requires `ANTHROPIC_API_KEY` (env or `.env`). A full sweep is roughly $0.03 per +variant and ~45s. Results land in `results/` (gitignored): a timestamped JSON +of every record plus `latest.md` with per-case tables. + +## Why the tables matter more than the aggregate + +The mechanical checks catch format violations and lexical repetition, but the +failure this feature actually had in production — headers that were +_informationally_ redundant while lexically varied — scores below the overlap +threshold. `results/latest.md` read by eye is the recall instrument; the +aggregate is the regression guard. + +## Layout + +| File | Role | +| ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `captured.json` | 9 real production payloads pulled verbatim from Langfuse, with the headers that shipped. Irreplaceable — traces age out. | +| `corpus.js` | 17 cases / 28 steps: the captured run as one sequence, plus synthetic cases for modes it never hit (all-failed, partial, parallel, rapid duplicates, entry overflow, truncated output, error-shaped success). Multi-step cases chain each generated label into the next step's context. | +| `prompt.js` | Faithful port of the SDK's `buildActivityLabelPrompt`, so synthetic cases render the bytes production would send. Adds `previousLabelCap` for continuity-window experiments. | +| `variants.js` | Single-factor instruction variants. `baseline` is read from the built `packages/api` dist, so drift from the shipped instruction is impossible. | +| `checks.js` | Mechanical grading: length, punctuation, markdown, tool-name echo, count echo, and overlap split into `restate` (adds nothing) vs `template` (same frame, new payload). | +| `run.js` / `rescore.js` | Live runner (production wire shape, `max_tokens: 256`) and offline re-grader. | + +## Findings this produced + +- **Sentence order is load-bearing.** Moving format constraints after content + rules cut length violations and register collapse measurably. This is why + `ACTIVITY_INSTRUCTION` is ordered the way it is — a "tidying" reshuffle + regresses real output. +- **Enumerating verbs backfires.** An instruction listing acceptable opening + verbs _anchored_ the model: `Confirmed` went from 18 to 23 occurrences and + opener diversity halved. +- **Diverse examples alone changed nothing.** Register collapse is task-shaped, + not example-seeded. +- **Continuity context fixed the real defect.** Feeding committed headers back + eliminated restatement and, unexpectedly, stopped setup batches from being + labeled with conclusions their tools had not yet established. +- **A 3-label window is enough.** Unbounded history scored no better — + restatement is inherently a recency problem — while prompt growth is linear + (+82 input tokens by batch 9, extrapolating to ~+250 at the `activityMaxPerRun` + default of 20). diff --git a/scripts/activity-labels/captured.json b/scripts/activity-labels/captured.json new file mode 100644 index 0000000000..30e306b741 --- /dev/null +++ b/scripts/activity-labels/captured.json @@ -0,0 +1,56 @@ +[ + { + "id": "sandbox-python-version", + "traceId": "eeab6f47000fe2a6aac76cde6bbc67fd", + "prompt": "Reasoning excerpts:\n- I'll work through each check methodically, running them one at a time and documenting what I find.\n\nTool calls:\n- run_tools_with_bash({\"code\":\"python3 --version\"}) → stdout:\nPython 3.14.4\n\nLabel:", + "productionLabel": "Confirmed Python 3.14.4 is installed" + }, + { + "id": "mnt-data-write-test", + "traceId": "8ee41468850d42e5477af07759519e84", + "prompt": "Intent (assistant's last message): **Python version: 3.14.4**\n\nNow let's check if files in /mnt/data survive between calls.\n\nTool calls:\n- bash_tool({\"command\":\"echo \\\"hello from probe 1, written at $(date)\\\" > /mnt/data/persist_test.txt; cat /mnt/data/persist_test.txt\"}) → stdout:\nhello from probe 1, written at Wed Jul 29 11:30:09 AM UTC 2026\n\nLabel:", + "productionLabel": "Wrote test file to /mnt/data, confirmed persistence" + }, + { + "id": "mnt-data-persistence", + "traceId": "64b318bd7251b66baccd75009d947682", + "prompt": "Intent (assistant's last message): **Python version: 3.14.4**\n\nNow let's check if files in /mnt/data survive between calls.\n\nReasoning excerpts:\n- I should make a second request to verify the behavior persists across multiple calls.\n\nTool calls:\n- bash_tool({\"command\":\"ls -la /mnt/data/persist_test.txt && cat /mnt/data/persist_test.txt\"}) → stdout:\n-rw------- 1 65534 65534 63 Jul 29 11:30 /mnt/data/persist_test.txt\nhello from probe 1, written at Wed Jul 29 11:30:09 AM UTC 2026\n\nLabel:", + "productionLabel": "Confirmed /mnt/data persists between tool calls" + }, + { + "id": "mnt-data-tmpfs", + "traceId": "bff7f9766cc2916230403a04c47c92af", + "prompt": "Intent (assistant's last message): **Files in /mnt/data DO survive between calls** — the file I wrote in the previous, separate sandbox invocation is still there with its original content intact.\n\nNow let's check RAM and disk.\n\nTool calls:\n- bash_tool({\"command\":\"echo \\\"=== MEMORY ===\\\"; free -h; echo; echo \\\"=== DISK ===\\\"; df -h /mnt/data / /tmp 2>/dev/null\"}) → stdout:\n=== MEMORY ===\n\n=== DISK ===\nFilesystem Size Used Avail Use% Mounted on\ntmpfs 1.0G 32K 1.0G 1% /mnt/data\nnone 16M 0 16M 0% /\nnone 20M 0 20M 0% /tmp\n\nstderr:\nfree: Unable to create meminfo structure\nNote: /tmp files are same-call scratch only and were not persisted; use /mnt/data for files needed later.\n\nLabel:", + "productionLabel": "Confirmed /mnt/data is tmpfs with 1GB capacity" + }, + { + "id": "cpu-meminfo-disk", + "traceId": "6faf26136e125c29fe0bae7ddd87c59e", + "prompt": "Intent (assistant's last message): `free` failed, let me try /proc/meminfo directly — and note disk is quite constrained (tmpfs-based, tiny root/tmp).\n\nTool calls:\n- bash_tool({\"command\":\"echo \\\"=== /proc/meminfo ===\\\"; cat /proc/meminfo 2>/dev/null | head -5\\necho\\necho \\\"=== nproc ===\\\"; nproc\\necho\\necho \\\"=== /proc/cpuinfo count ===\\\"; grep -c ^processor /proc/cpuinfo 2>/dev/null\\necho\\necho \\\"=== disk full df ===\\\"; df -h 2>/dev/null\"}) → stdout:\n=== /proc/meminfo ===\n\n=== nproc ===\n1\n\n=== /proc/cpuinfo count ===\n0\n\n=== disk full df ===\nFilesystem Size Used Avail Use% Mounted on\nnone 16M 0 16M 0% /\n/dev/vdc 16G 4.4G 11G 30% /usr\nnone 16M 4.0K 16M 1% /etc/hosts\ntmpfs 64M 0 64M 0% /dev/null\nnone 20M 0 20M 0% /tmp\ntmpfs 1.0G 32K 1.0G 1% /mnt/data\n\nLabel:", + "productionLabel": "Confirmed single CPU, /proc/meminfo empty, disk constrained with 16M root and 20M tmp" + }, + { + "id": "rlimit-cgroup", + "traceId": "2797232991aa7e841081a6b6175bd62b", + "prompt": "Intent (assistant's last message): `/proc/meminfo` is empty/restricted, but I can check cgroup limits for a real memory figure.\n\nTool calls:\n- bash_tool({\"command\":\"echo \\\"=== cgroup memory limit ===\\\"\\nfor f in /sys/fs/cgroup/memory.max /sys/fs/cgroup/memory/memory.limit_in_bytes; do\\n [ -f \\\"$f\\\" ] && echo \\\"$f: $(cat $f)\\\"\\ndone\\necho\\necho \\\"=== cgroup cpu ===\\\"\\nfor f in /sys/fs/cgroup/cpu.max /sys/fs/cgroup/cpu/cpu.cfs_quota_us; do\\n [ -f \\\"$f\\\" ] && echo \\\"$f: $(cat $f)\\\"\\ndone\\necho\\npython3 -c \\\"import resource; print('soft/hard RLIMIT_AS:', resource.getrlimit(resource.RLIMIT_AS))\\\" 2>/dev/null\"}) → stdout:\n=== cgroup memory limit ===\n\n=== cgroup cpu ===\n\nsoft/hard RLIMIT_AS: (17179869184, 17179869184)\n\nLabel:", + "productionLabel": "Found RLIMIT_AS ceiling at 16GB, cgroup limits unavailable" + }, + { + "id": "network-tools", + "traceId": "e813a6ee932da05d573ced56424265b8", + "prompt": "Intent (assistant's last message): No cgroup files readable, but RLIMIT_AS shows a **16 GiB address-space limit** — likely the effective memory ceiling. Now let's check network access.\n\nTool calls:\n- bash_tool({\"command\":\"echo \\\"=== ping ===\\\"; timeout 3 ping -c1 8.8.8.8 2>&1\\necho\\necho \\\"=== curl ===\\\"; timeout 5 curl -sS -o /dev/null -w \\\"HTTP_CODE:%{http_code}\\\\n\\\" https://www.google.com 2>&1\\necho\\necho \\\"=== curl example.com ===\\\"; timeout 5 curl -sS -o /dev/null -w \\\"HTTP_CODE:%{http_code}\\\\n\\\" http://example.com 2>&1\\necho\\necho \\\"=== DNS resolution ===\\\"; timeout 3 getent hosts google.com 2>&1\"}) → stdout:\n=== ping ===\ntimeout: failed to run command ‘ping’: No such file or directory\n\n=== curl ===\ntimeout: failed to run command ‘curl’: No such file or directory\n\n=== curl example.com ===\ntimeout: failed to run command ‘curl’: No such file or directory\n\n=== DNS resolution ===\n\nLabel:", + "productionLabel": "Network tools unavailable, DNS resolution failed" + }, + { + "id": "dns-tcp", + "traceId": "d96b49502c4a0ecb4c9325c2e74b43bf", + "prompt": "Intent (assistant's last message): No curl/ping binaries — let me test connectivity directly via Python sockets/urllib instead.\n\nTool calls:\n- bash_tool({\"command\":\"python3 - <<'EOF'\\nimport socket, urllib.request, time\\n\\n# DNS test\\ntry:\\n ip = socket.gethostbyname(\\\"example.com\\\")\\n print(\\\"DNS resolution OK:\\\", ip)\\nexcept Exception as e:\\n print(\\\"DNS resolution FAILED:\\\", repr(e))\\n\\n# raw TCP connect test\\ntry:\\n s = socket.create_connection((\\\"8.8.8.8\\\", 53), timeout=4)\\n print(\\\"TCP connect to 8.8.8.8:53 OK\\\")\\n s.close()\\nexcept Exception as e:\\n print(\\\"TCP connect FAILED:\\\", repr(e))\\n\\n# HTTP fetch test\\ntry:\\n t0 = time.time()\\n r = urllib.request.urlopen(\\\"http://example.com\\\", timeout=5)\\n print(\\…) → stdout:\nDNS resolution FAILED: gaierror(-2, 'Name or service not known')\nTCP connect FAILED: PermissionError(1, 'Operation not permitted')\nHTTP fetch FAILED: URLError(gaierror(-3, 'Temporary failure in name resolution'))\n\nLabel:", + "productionLabel": "DNS and TCP connectivity both blocked or unavailable" + }, + { + "id": "python-packages", + "traceId": "a6fcd0fd40b490285ced22c236023e2c", + "prompt": "Intent (assistant's last message): **No network access** — DNS resolution fails and raw TCP connections are blocked with a permission error. This sandbox is fully network-isolated.\n\nNow let's check for numpy/pandas/scipy/torch.\n\nTool calls:\n- bash_tool({\"command\":\"python3 - <<'EOF'\\nmods = [\\\"numpy\\\", \\\"pandas\\\", \\\"scipy\\\", \\\"torch\\\"]\\nfor m in mods:\\n try:\\n mod = __import__(m)\\n ver = getattr(mod, \\\"__version__\\\", \\\"unknown\\\")\\n print(f\\\"{m}: INSTALLED, version {ver}\\\")\\n except ImportError as e:\\n print(f\\\"{m}: NOT INSTALLED ({e})\\\")\\nEOF\"}) → stdout:\nnumpy: INSTALLED, version 2.4.6\npandas: INSTALLED, version 2.3.3\nscipy: INSTALLED, version 1.18.0\ntorch: NOT INSTALLED (No module named 'torch')\n\nLabel:", + "productionLabel": "Confirmed numpy, pandas, scipy installed; torch unavailable" + } +] diff --git a/scripts/activity-labels/checks.js b/scripts/activity-labels/checks.js new file mode 100644 index 0000000000..8acbea7b88 --- /dev/null +++ b/scripts/activity-labels/checks.js @@ -0,0 +1,156 @@ +/** + * Mechanical label checks. These catch format violations and the two + * measurable prose failures (register collapse via first-word tallies, + * cross-batch redundancy via content-word overlap); commit-log READABILITY + * still needs the human pass over results/latest.md. + */ +const STOPWORDS = new Set([ + 'the', + 'a', + 'an', + 'to', + 'of', + 'and', + 'or', + 'with', + 'for', + 'in', + 'on', + 'at', + 'is', + 'are', + 'was', + 'were', + 'be', + 'been', + 'it', + 'its', + 'as', + 'by', + 'from', + 'that', + 'this', + 'both', + 'all', + 'no', + 'not', + 'via', +]); + +function words(label) { + return label.trim().split(/\s+/).filter(Boolean); +} + +const SUFFIXES = ['ations', 'ation', 'ence', 'ance', 'ings', 'ing', 'ed', 'es', 's']; + +/** Crude suffix stemmer so persists/persistence/persisted collide — enough + * for overlap detection; linguistic correctness is not the goal. */ +function stem(word) { + if (word.length < 5) { + return word; + } + for (const suffix of SUFFIXES) { + if (word.endsWith(suffix) && word.length - suffix.length >= 4) { + return word.slice(0, word.length - suffix.length); + } + } + return word; +} + +function contentWords(label) { + return words(label.toLowerCase().replace(/[^a-z0-9/._-]+/g, ' ')) + .filter((word) => !STOPWORDS.has(word)) + .map(stem); +} + +/** Payload tokens carry the informative delta between template-shaped + * labels: numbers, versions, paths, filenames. */ +function isPayload(word) { + return /\d/.test(word) || word.includes('/') || word.includes('.'); +} + +function jaccard(a, b) { + const setA = new Set(a); + const setB = new Set(b); + if (setA.size === 0 || setB.size === 0) { + return 0; + } + let intersection = 0; + for (const word of setA) { + if (setB.has(word)) { + intersection += 1; + } + } + return intersection / (setA.size + setB.size - intersection); +} + +const GENERIC_OPENER = /^(ran|used|executed|called|invoked|performed)\b/i; +const COUNT_ECHO = /\b\d+\s+(tools?|commands?|calls?)\b/i; +const DUP_THRESHOLD = 0.5; + +/** + * @param label generated label text + * @param entries the batch's tool entries (for tool-name echo detection) + * @param previousLabels labels generated EARLIER in the same case chain, + * regardless of whether the variant saw them — redundancy is measured + * uniformly so continuity variants can be compared against blind ones. + */ +function checkLabel(label, { entries = [], previousLabels = [] } = {}) { + const flags = []; + const wordList = words(label); + if (wordList.length < 4 || wordList.length > 9) { + flags.push(`len:${wordList.length}`); + } + if (/[.!?,;:]$/.test(label.trim())) { + flags.push('punct'); + } + if (/^["'`]|["'`]$/.test(label.trim())) { + flags.push('quote'); + } + if (/[*`]|^#|\[.*\]\(/.test(label)) { + flags.push('md'); + } + if (GENERIC_OPENER.test(label.trim())) { + flags.push('opener'); + } + const lower = label.toLowerCase(); + for (const entry of entries) { + const name = String(entry.toolName ?? '').toLowerCase(); + if (name.length > 3 && (lower.includes(name) || lower.includes(name.replace(/_/g, ' ')))) { + flags.push(`tool-echo:${entry.toolName}`); + break; + } + } + if (COUNT_ECHO.test(label)) { + flags.push('count-echo'); + } + /** Overlap splits into two flags: `restate` (high overlap, no payload + * delta — the line adds nothing over a previous header; the production + * 2/3 and 7/8 failure) and `template` (high overlap but the differing + * tokens are numbers/paths — same sentence frame, new information, e.g. + * fib(1)→fib(2). Often fine, arguably better than synonym churn). */ + const own = contentWords(label); + let maxOverlap = 0; + let worst = null; + for (const previous of previousLabels) { + const other = contentWords(previous); + const overlap = jaccard(own, other); + if (overlap > maxOverlap) { + maxOverlap = overlap; + worst = other; + } + } + if (maxOverlap > DUP_THRESHOLD && worst != null) { + const otherSet = new Set(worst); + const ownSet = new Set(own); + const differing = [ + ...own.filter((word) => !otherSet.has(word)), + ...worst.filter((word) => !ownSet.has(word)), + ]; + const informativeDelta = differing.some(isPayload); + flags.push(`${informativeDelta ? 'template' : 'restate'}:${maxOverlap.toFixed(2)}`); + } + return { flags, wordCount: wordList.length, firstWord: wordList[0] ?? '', maxOverlap }; +} + +module.exports = { checkLabel, contentWords, jaccard, words, stem }; diff --git a/scripts/activity-labels/corpus.js b/scripts/activity-labels/corpus.js new file mode 100644 index 0000000000..bacec6104c --- /dev/null +++ b/scripts/activity-labels/corpus.js @@ -0,0 +1,455 @@ +/** + * Eval corpus for activity-label prose. Two halves: + * + * - captured.json: the 9 real payloads from the 2026-07-29 sandbox-probe run, + * verbatim from Langfuse, replayed as ONE sequence so continuity variants + * see the same run shape production did. `productionLabel` is what shipped. + * - synthetic: cases built for the failure modes the captured run surfaced + * (redundant consecutive batches, register collapse, length overflow) plus + * the modes it never exercised (all-failed, partial, parallel columns, + * truncation, entry overflow, error-shaped success). + * + * A case is a sequence of steps; a step is one label request. Multi-step + * cases exist to measure cross-batch redundancy: the runner chains each + * step's generated label into the next step's `previousLabels` for variants + * that opt in. + */ +const fs = require('fs'); +const path = require('path'); + +const captured = JSON.parse(fs.readFileSync(path.join(__dirname, 'captured.json'), 'utf8')); + +const capturedRun = { + id: 'sandbox-probe-run', + notes: 'the real 9-batch production run, verbatim payloads', + steps: captured.map((entry) => ({ + id: entry.id, + verbatim: entry.prompt, + productionLabel: entry.productionLabel, + })), +}; + +const synthetic = [ + { + id: 'all-failed', + notes: 'every call fails — failure register, verb-first under failure', + steps: [ + { + payload: { + lastAssistantText: "I'll run each of these and report exactly what happens.", + entries: [ + { + toolName: 'run_tools_with_bash', + toolInput: { code: 'cat /etc/shadow' }, + status: 'error', + error: 'cat: /etc/shadow: Permission denied', + }, + { + toolName: 'run_tools_with_bash', + toolInput: { code: 'ls /nonexistent-dir' }, + status: 'error', + error: "ls: cannot access '/nonexistent-dir': No such file or directory", + }, + { + toolName: 'run_tools_with_bash', + toolInput: { code: 'curl -sS https://nope.invalid' }, + status: 'error', + error: 'curl: (6) Could not resolve host: nope.invalid', + }, + ], + }, + }, + ], + }, + { + id: 'partial-failure', + notes: 'mixed batch — must not read as all-success or all-failure', + steps: [ + { + payload: { + thinkingExcerpts: [ + 'Three probes: create the marker dir, read the shadow file, resolve an invalid host. The first should work, the other two should fail for different reasons.', + ], + entries: [ + { + toolName: 'run_tools_with_bash', + toolInput: { code: 'mkdir -p /tmp/probe && echo ok' }, + toolOutput: 'stdout:\nok', + status: 'success', + }, + { + toolName: 'run_tools_with_bash', + toolInput: { code: 'cat /etc/shadow' }, + status: 'error', + error: 'cat: /etc/shadow: Permission denied', + }, + { + toolName: 'run_tools_with_bash', + toolInput: { code: 'getent hosts nope.invalid' }, + status: 'error', + error: 'exit code 2', + }, + ], + }, + }, + ], + }, + { + id: 'parallel-versions', + notes: 'one batch of parallel lookups — the groupId/parallel-columns shape', + steps: [ + { + payload: { + lastAssistantText: 'Let me look up all three at once.', + entries: [ + { + toolName: 'web_search', + toolInput: { query: 'Node.js latest stable version 2026' }, + toolOutput: + 'Node.js 24.5.0 (Current) released 2026-07-22; v24 enters LTS October 2026. nodejs.org/en/blog/release/v24.5.0', + status: 'success', + }, + { + toolName: 'web_search', + toolInput: { query: 'Deno latest release version' }, + toolOutput: + 'Deno 2.4.2 released 2026-07-16 with improved node:sqlite compat. deno.com/blog/v2.4', + status: 'success', + }, + { + toolName: 'web_search', + toolInput: { query: 'Bun latest release version' }, + toolOutput: + 'Bun 1.2.19 released 2026-07-25, adds --compile cross-target for linux-arm64. bun.sh/blog/bun-v1.2.19', + status: 'success', + }, + ], + }, + }, + ], + }, + { + id: 'fib-rapid', + notes: 'three near-identical consecutive batches — redundancy stress', + steps: [1, 2, 3].map((n) => ({ + id: `fib-${n}`, + payload: { + entries: [ + { + toolName: 'execute_code', + toolInput: { code: `print(fib(${n}))` }, + toolOutput: `stdout:\n${[1, 1, 2][n - 1]}`, + status: 'success', + }, + ], + }, + })), + }, + { + id: 'mega-batch', + notes: 'six heterogeneous probes in one batch — length-cap stress (mirrors cpu-meminfo-disk)', + steps: [ + { + payload: { + thinkingExcerpts: [ + "I'll gather the full system picture in one pass: CPU count, memory, disk, limits, user, kernel.", + ], + entries: [ + { + toolName: 'run_tools_with_bash', + toolInput: { code: 'nproc' }, + toolOutput: 'stdout:\n1', + status: 'success', + }, + { + toolName: 'run_tools_with_bash', + toolInput: { code: 'cat /proc/meminfo | head -3' }, + toolOutput: 'stdout:\n', + status: 'success', + }, + { + toolName: 'run_tools_with_bash', + toolInput: { code: 'df -h / /tmp' }, + toolOutput: + 'stdout:\nFilesystem Size Used Avail Use% Mounted on\noverlay 16M 12M 4.0M 75% /\ntmpfs 20M 0 20M 0% /tmp', + status: 'success', + }, + { + toolName: 'run_tools_with_bash', + toolInput: { code: 'ulimit -v' }, + toolOutput: 'stdout:\n16777216', + status: 'success', + }, + { + toolName: 'run_tools_with_bash', + toolInput: { code: 'whoami' }, + toolOutput: 'stdout:\nsandbox', + status: 'success', + }, + { + toolName: 'run_tools_with_bash', + toolInput: { code: 'uname -r' }, + toolOutput: 'stdout:\n6.1.102', + status: 'success', + }, + ], + }, + }, + ], + }, + { + id: 'single-trivial', + notes: 'one boring call — header must still say something the card cannot', + steps: [ + { + payload: { + entries: [ + { + toolName: 'run_tools_with_bash', + toolInput: { code: 'ls /mnt/data' }, + toolOutput: 'stdout:\nnotes.md\nresults.csv\nprobe.txt', + status: 'success', + }, + ], + }, + }, + ], + }, + { + id: 'answer-found', + notes: 'the answer IS the line — a question resolved by one call', + steps: [ + { + payload: { + lastAssistantText: 'Let me find where that 30-second timeout is actually set.', + entries: [ + { + toolName: 'grep', + toolInput: { pattern: 'timeout', path: 'api/server/utils/streams.js' }, + toolOutput: + 'streams.js:41: const STREAM_TIMEOUT_MS = 30_000; // hard cap per SSE flush\nstreams.js:88: setTimeout(() => controller.abort(), STREAM_TIMEOUT_MS);', + status: 'success', + }, + ], + }, + }, + ], + }, + { + id: 'bare-batch', + notes: 'no intent, no reasoning — minimum context', + steps: [ + { + payload: { + entries: [ + { + toolName: 'read_file', + toolInput: { path: 'package.json' }, + toolOutput: '{\n "name": "librechat",\n "version": "0.8.1",\n ...', + status: 'success', + }, + ], + }, + }, + ], + }, + { + id: 'misleading-intent', + notes: 'intent asks one question, output answers it the other way', + steps: [ + { + payload: { + lastAssistantText: 'Now checking whether response caching is enabled in this deployment.', + entries: [ + { + toolName: 'run_tools_with_bash', + toolInput: { code: 'grep -A2 "cache:" config/deploy.yaml' }, + toolOutput: 'stdout:\ncache:\n enabled: false\n ttl: 3600', + status: 'success', + }, + ], + }, + }, + ], + }, + { + id: 'truncated-output', + notes: 'output clipped mid-JSON by the 600-char limit', + steps: [ + { + payload: { + entries: [ + { + toolName: 'run_tools_with_bash', + toolInput: { code: 'pip list --format=json' }, + toolOutput: + 'stdout:\n' + + JSON.stringify( + Array.from({ length: 60 }, (_, i) => ({ + name: `package-${i}`, + version: `1.${i}.0`, + })), + ), + status: 'success', + }, + ], + }, + }, + ], + }, + { + id: 'silent-success', + notes: 'empty output — nothing came back to summarize', + steps: [ + { + payload: { + lastAssistantText: "I'll write the results file now.", + entries: [ + { + toolName: 'write_file', + toolInput: { path: '/mnt/data/results.csv', content: 'run,ms\n1,412\n2,398\n' }, + toolOutput: '', + status: 'success', + }, + ], + }, + }, + ], + }, + { + id: 'edit-verify', + notes: 'edit plus read-back in one batch — one activity, two calls', + steps: [ + { + payload: { + thinkingExcerpts: [ + 'The retry cap is what causes the duplicate sends; dropping it from 5 to 1 and verifying the file took the change.', + ], + entries: [ + { + toolName: 'edit_file', + toolInput: { + path: 'api/server/utils/queue.js', + old: 'const MAX_RETRIES = 5;', + new: 'const MAX_RETRIES = 1;', + }, + toolOutput: 'OK', + status: 'success', + }, + { + toolName: 'read_file', + toolInput: { path: 'api/server/utils/queue.js', range: [10, 14] }, + toolOutput: 'const MAX_RETRIES = 1;\nconst BACKOFF_MS = 250;', + status: 'success', + }, + ], + }, + }, + ], + }, + { + id: 'error-shaped-success', + notes: 'tool returns an error payload with success status — must not read as success', + steps: [ + { + payload: { + lastAssistantText: 'Searching for the changelog now.', + entries: [ + { + toolName: 'web_search', + toolInput: { query: 'librechat 0.8.1 changelog' }, + toolOutput: + '{"error":{"code":"rate_limited","message":"Search quota exceeded, retry after 3600s"}}', + status: 'success', + }, + ], + }, + }, + ], + }, + { + id: 'mcp-long-name', + notes: 'namespaced MCP tool name — echo temptation', + steps: [ + { + payload: { + entries: [ + { + toolName: 'mcp__github__search_repositories', + toolInput: { query: 'org:danny-avila librechat-agents' }, + toolOutput: + '{"total_count":2,"items":[{"full_name":"danny-avila/LibreChat","stars":31200},{"full_name":"danny-avila/agents","stars":410}]}', + status: 'success', + }, + ], + }, + }, + ], + }, + { + id: 'dup-activity-seq', + notes: 'controlled mirror of captured steps 2/3 — same activity twice in a row', + steps: [ + { + id: 'dup-write', + payload: { + thinkingExcerpts: [ + 'First write a marker file, then a separate call will check it survives.', + ], + entries: [ + { + toolName: 'run_tools_with_bash', + toolInput: { + code: 'echo "marker-$(date +%s)" > /tmp/persist-probe.txt && cat /tmp/persist-probe.txt', + }, + toolOutput: 'stdout:\nmarker-1785932011', + status: 'success', + }, + ], + }, + }, + { + id: 'dup-confirm', + payload: { + entries: [ + { + toolName: 'run_tools_with_bash', + toolInput: { code: 'cat /tmp/persist-probe.txt' }, + toolOutput: 'stdout:\nmarker-1785932011', + status: 'success', + }, + ], + }, + }, + ], + }, + { + id: 'overflow-entries', + notes: '14 calls — exercises the 12-entry cap and the "…and 2 more" suffix', + steps: [ + { + payload: { + entries: Array.from({ length: 14 }, (_, i) => ({ + toolName: 'run_tools_with_bash', + toolInput: { code: `convert page-${i + 1}.svg page-${i + 1}.png` }, + toolOutput: '', + status: 'success', + })), + }, + }, + ], + }, +]; + +/** Tool names for echo checks; captured steps bake entries into the + * verbatim prompt, so they are recovered from the "Tool calls:" lines. */ +function stepEntries(step) { + if (step.payload?.entries) { + return step.payload.entries; + } + return [...(step.verbatim ?? '').matchAll(/^- ([A-Za-z0-9_]+)\(/gm)].map((match) => ({ + toolName: match[1], + })); +} + +module.exports = { cases: [capturedRun, ...synthetic], stepEntries }; diff --git a/scripts/activity-labels/prompt.js b/scripts/activity-labels/prompt.js new file mode 100644 index 0000000000..f67fcfcae5 --- /dev/null +++ b/scripts/activity-labels/prompt.js @@ -0,0 +1,151 @@ +/** + * Faithful port of the SDK's `buildActivityLabelPrompt` + * (agentus src/prompts/activityLabel.ts) — the PREFERRED path that serves + * every production label. The older live-check script mirrors a simplified + * e2e capture; this port keeps section order (Intent → Reasoning excerpts → + * Tool calls → Label:), the 12-entry cap with the "…and N more" suffix, and + * the exact truncation semantics, so synthetic corpus cases render the same + * bytes production would send. Redaction is intentionally not ported — the + * corpus models unredacted single-agent runs. + * + * One addition beyond the SDK: an optional "Previous headers" section, OFF + * unless a variant opts in. This is the P1 continuity hypothesis — it lets + * the harness measure the fix before any SDK field exists. + */ +const INPUT_CONTEXT_LIMIT = 200; +const MAX_THINKING_EXCERPTS = 4; +const MAX_PROMPT_ENTRIES = 12; +const MAX_PREVIOUS_LABELS = 3; + +function truncateForLabel(value, maxLength) { + if (value.length <= maxLength) { + return value; + } + return value.slice(0, Math.max(0, maxLength - 1)) + '…'; +} + +const ABORT_SERIALIZATION = Symbol('abort-label-serialization'); + +function serializeForLabel(value, limit) { + if (value == null) { + return ''; + } + if (typeof value === 'string') { + return value.length > limit ? value.slice(0, limit + 1) : value; + } + let budget = limit * 4; + try { + return ( + JSON.stringify(value, (_key, nested) => { + if (budget <= 0) { + throw ABORT_SERIALIZATION; + } + if (typeof nested === 'string') { + const clipped = nested.length > limit ? nested.slice(0, limit) : nested; + budget -= clipped.length; + return clipped; + } + budget -= 8; + return nested; + }) ?? '' + ); + } catch (error) { + if (error === ABORT_SERIALIZATION) { + return Array.isArray(value) ? `[Array(${value.length})]` : '[Object]'; + } + return String(value); + } +} + +/** `cap` of Infinity models the unbounded-history alternative — testing + * whether the whole run's story beats a recency window. */ +function previousHeadersSection(previousLabels, cap = MAX_PREVIOUS_LABELS) { + const kept = previousLabels.filter(Boolean); + const recent = Number.isFinite(cap) ? kept.slice(-cap) : kept; + if (recent.length === 0) { + return null; + } + return ( + 'Previous headers in this run (most recent last):\n' + + recent.map((label) => `- ${label}`).join('\n') + ); +} + +function buildActivityLabelPrompt({ + entries, + charLimit, + thinkingExcerpts, + lastAssistantText, + previousLabels, + previousLabelCap, +}) { + const clip = truncateForLabel; + const sections = []; + if (previousLabels != null) { + const section = previousHeadersSection(previousLabels, previousLabelCap); + if (section != null) { + sections.push(section); + } + } + if (lastAssistantText != null && lastAssistantText.length > 0) { + sections.push( + `Intent (assistant's last message): ${clip(lastAssistantText, INPUT_CONTEXT_LIMIT)}`, + ); + } + if (thinkingExcerpts != null && thinkingExcerpts.length > 0) { + sections.push( + 'Reasoning excerpts:\n' + + thinkingExcerpts + .slice(0, MAX_THINKING_EXCERPTS) + .map((excerpt) => `- ${clip(excerpt, charLimit)}`) + .join('\n'), + ); + } + if (entries.length > 0) { + const shown = entries.slice(0, MAX_PROMPT_ENTRIES); + const omitted = entries.length - shown.length; + sections.push( + 'Tool calls:\n' + + shown + .map((entry) => { + const input = clip(serializeForLabel(entry.toolInput, charLimit), charLimit); + const outcome = + entry.status === 'error' + ? `ERROR: ${clip(entry.error ?? 'unknown error', charLimit)}` + : clip(serializeForLabel(entry.toolOutput, charLimit), charLimit); + return `- ${entry.toolName}(${input}) → ${outcome}`; + }) + .join('\n') + + (omitted > 0 ? `\n- …and ${omitted} more tool ${omitted === 1 ? 'call' : 'calls'}` : ''), + ); + } + sections.push('Label:'); + return sections.join('\n\n'); +} + +/** + * Renders a corpus step. Captured steps carry the verbatim production + * prompt (byte-exact from Langfuse); the continuity section, when a variant + * opts in, is prepended — the same position the built path gives it. + */ +function renderStepPrompt(step, { charLimit, previousLabels, previousLabelCap }) { + if (step.verbatim != null) { + const section = + previousLabels != null ? previousHeadersSection(previousLabels, previousLabelCap) : null; + return section != null ? `${section}\n\n${step.verbatim}` : step.verbatim; + } + return buildActivityLabelPrompt({ + ...step.payload, + charLimit, + previousLabels, + previousLabelCap, + }); +} + +module.exports = { + buildActivityLabelPrompt, + renderStepPrompt, + truncateForLabel, + serializeForLabel, + MAX_PREVIOUS_LABELS, +}; diff --git a/scripts/activity-labels/report.js b/scripts/activity-labels/report.js new file mode 100644 index 0000000000..e06eb14413 --- /dev/null +++ b/scripts/activity-labels/report.js @@ -0,0 +1,148 @@ +/** Aggregation + markdown rendering, shared by the live runner and the + * offline rescorer so metric fixes never require re-spending on the API. */ +const FLAG_TYPES = [ + 'len', + 'punct', + 'quote', + 'md', + 'opener', + 'tool-echo', + 'count-echo', + 'restate', + 'template', +]; + +const PRICES = { 'claude-haiku-4-5': { input: 1, output: 5 } }; + +function flagType(flag) { + return flag.split(':')[0]; +} + +function aggregate(records, model) { + const byVariant = new Map(); + for (const record of records) { + if (!byVariant.has(record.variant)) { + byVariant.set(record.variant, { + steps: 0, + errors: 0, + flagCounts: {}, + firstWords: {}, + totalWords: 0, + latencies: [], + inputTokens: 0, + outputTokens: 0, + }); + } + const agg = byVariant.get(record.variant); + if (record.error) { + agg.errors += 1; + continue; + } + agg.steps += 1; + agg.totalWords += record.wordCount; + agg.latencies.push(record.latencyMs); + agg.inputTokens += record.inputTokens; + agg.outputTokens += record.outputTokens; + agg.firstWords[record.firstWord] = (agg.firstWords[record.firstWord] ?? 0) + 1; + for (const flag of record.flags) { + const type = flagType(flag); + agg.flagCounts[type] = (agg.flagCounts[type] ?? 0) + 1; + } + } + const price = PRICES[model]; + return [...byVariant.entries()].map(([name, agg]) => { + const sortedFirst = Object.entries(agg.firstWords).sort((a, b) => b[1] - a[1]); + const topOpener = sortedFirst[0] ?? ['—', 0]; + return { + variant: name, + steps: agg.steps, + errors: agg.errors, + flagCounts: agg.flagCounts, + distinctOpeners: sortedFirst.length, + topOpener: `${topOpener[0]} ×${topOpener[1]}`, + avgWords: agg.steps > 0 ? (agg.totalWords / agg.steps).toFixed(1) : '—', + meanLatencyMs: agg.latencies.length + ? Math.round(agg.latencies.reduce((a, b) => a + b, 0) / agg.latencies.length) + : 0, + inputTokens: agg.inputTokens, + outputTokens: agg.outputTokens, + costUsd: price + ? ((agg.inputTokens * price.input + agg.outputTokens * price.output) / 1e6).toFixed(4) + : 'n/a', + }; + }); +} + +function markdownReport({ records, aggregates, runCases, variantNames, model, samples }) { + const lines = []; + lines.push(`# Activity-label eval — ${new Date().toISOString()}`); + lines.push(''); + lines.push(`model: \`${model}\` · samples: ${samples} · cases: ${runCases.length}`); + lines.push(''); + lines.push('## Aggregate'); + lines.push(''); + lines.push( + `| variant | steps | ${FLAG_TYPES.join(' | ')} | distinct openers | top opener | avg words | mean ms | cost |`, + ); + lines.push(`|---|---:|${FLAG_TYPES.map(() => '---:').join('|')}|---:|---|---:|---:|---:|`); + for (const agg of aggregates) { + lines.push( + `| ${agg.variant} | ${agg.steps}${agg.errors ? ` (+${agg.errors} err)` : ''} | ` + + FLAG_TYPES.map((type) => agg.flagCounts[type] ?? 0).join(' | ') + + ` | ${agg.distinctOpeners} | ${agg.topOpener} | ${agg.avgWords} | ${agg.meanLatencyMs} | $${agg.costUsd} |`, + ); + } + lines.push(''); + lines.push('## Per-case'); + for (const testCase of runCases) { + lines.push(''); + lines.push(`### ${testCase.id}`); + lines.push(''); + lines.push(`*${testCase.notes}*`); + lines.push(''); + const sampleList = [...new Set(records.map((r) => r.sample))].sort(); + const header = ['step']; + if (samples > 1) { + header.push('s'); + } + if (testCase.steps.some((step) => step.productionLabel)) { + header.push('production'); + } + header.push(...variantNames); + lines.push(`| ${header.join(' | ')} |`); + lines.push(`|${header.map(() => '---').join('|')}|`); + for (const step of testCase.steps) { + const stepId = step.id ?? testCase.id; + for (const sample of sampleList) { + const row = [stepId]; + if (samples > 1) { + row.push(String(sample)); + } + if (header.includes('production')) { + row.push(step.productionLabel ?? ''); + } + for (const variantName of variantNames) { + const record = records.find( + (r) => + r.variant === variantName && + r.sample === sample && + r.caseId === testCase.id && + r.stepId === stepId, + ); + if (!record) { + row.push(''); + } else if (record.error) { + row.push(`⛔ ${record.error}`); + } else { + const flagNote = record.flags.length > 0 ? ` ⚠${record.flags.join(' ⚠')}` : ''; + row.push(`${record.label}${flagNote}`); + } + } + lines.push(`| ${row.map((cell) => cell.replace(/\|/g, '\\|')).join(' | ')} |`); + } + } + } + return lines.join('\n') + '\n'; +} + +module.exports = { aggregate, markdownReport, FLAG_TYPES }; diff --git a/scripts/activity-labels/rescore.js b/scripts/activity-labels/rescore.js new file mode 100644 index 0000000000..30922970d8 Binary files /dev/null and b/scripts/activity-labels/rescore.js differ diff --git a/scripts/activity-labels/run.js b/scripts/activity-labels/run.js new file mode 100644 index 0000000000..7035a1362d --- /dev/null +++ b/scripts/activity-labels/run.js @@ -0,0 +1,241 @@ +/** + * Activity-label eval runner: replays the corpus against the production wire + * shape (system = instruction variant, user = SDK-built prompt, max_tokens + * 256 — matching the traced production requests) and reports per-variant + * quality: format violations, first-word register distribution, cross-batch + * redundancy, latency, and cost. + * + * Sequences run their steps serially; each generated label chains into the + * next step's `previousLabels`. Variants with `usePreviousLabels` see that + * context in the prompt; every variant is MEASURED against it, so blind and + * continuity variants share one redundancy metric. + * + * Usage: + * node scripts/activity-labels/run.js [--variants baseline,continuity] + * [--cases sandbox-probe-run,fib-rapid] [--samples 2] [--model id] + * [--concurrency 6] [--dry] + */ +const fs = require('fs'); +const path = require('path'); + +const { cases, stepEntries } = require('./corpus'); +const { variants } = require('./variants'); +const { checkLabel } = require('./checks'); +const { renderStepPrompt } = require('./prompt'); +const { aggregate, markdownReport } = require('./report'); + +const ROOT = path.resolve(__dirname, '..', '..'); +const RESULTS_DIR = path.join(__dirname, 'results'); +const CHAR_LIMIT = 600; +const MAX_TOKENS = 256; + +function parseArgs(argv) { + const args = { samples: 1, concurrency: 6, model: 'claude-haiku-4-5', dry: false }; + for (let i = 0; i < argv.length; i++) { + const key = argv[i]; + if (key === '--dry') { + args.dry = true; + } else if (key === '--variants') { + args.variants = argv[++i].split(','); + } else if (key === '--cases') { + args.cases = argv[++i].split(','); + } else if (key === '--samples') { + args.samples = Number(argv[++i]); + } else if (key === '--model') { + args.model = argv[++i]; + } else if (key === '--concurrency') { + args.concurrency = Number(argv[++i]); + } + } + return args; +} + +function loadKey() { + if (process.env.ANTHROPIC_API_KEY) { + return process.env.ANTHROPIC_API_KEY; + } + const envPath = path.join(ROOT, '.env'); + const line = fs.existsSync(envPath) + ? fs + .readFileSync(envPath, 'utf8') + .split('\n') + .find((entry) => entry.startsWith('ANTHROPIC_API_KEY=')) + : undefined; + if (!line) { + throw new Error( + `ANTHROPIC_API_KEY not set and not found in ${envPath}.\n` + + 'Pass it inline: ANTHROPIC_API_KEY=sk-… node scripts/activity-labels/run.js', + ); + } + return line + .slice('ANTHROPIC_API_KEY='.length) + .trim() + .replace(/^["']|["']$/g, ''); +} + +async function requestLabel({ apiKey, model, instruction, prompt }) { + for (let attempt = 1; attempt <= 3; attempt++) { + const started = Date.now(); + const response = await fetch('https://api.anthropic.com/v1/messages', { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'x-api-key': apiKey, + 'anthropic-version': '2023-06-01', + }, + body: JSON.stringify({ + model, + max_tokens: MAX_TOKENS, + system: instruction, + messages: [{ role: 'user', content: prompt }], + }), + }); + if (response.ok) { + const json = await response.json(); + const label = (json.content ?? []) + .map((block) => block.text ?? '') + .join('') + .trim() + .replace(/^["']|["']$/g, ''); + return { + label, + latencyMs: Date.now() - started, + inputTokens: json.usage?.input_tokens ?? 0, + outputTokens: json.usage?.output_tokens ?? 0, + }; + } + const body = await response.text(); + if (attempt < 3 && [429, 500, 529].includes(response.status)) { + const retryAfter = Number(response.headers.get('retry-after')); + const waitMs = + Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter * 1000 : attempt * 2000; + await new Promise((resolve) => setTimeout(resolve, Math.min(waitMs, 15000))); + continue; + } + return { + error: `HTTP ${response.status}: ${body.slice(0, 160)}`, + latencyMs: Date.now() - started, + }; + } +} + +/** One case chain: steps serial, labels feeding forward. */ +async function runCase({ apiKey, model, variant, sample, testCase, dry, records }) { + const chain = []; + for (const step of testCase.steps) { + const prompt = renderStepPrompt(step, { + charLimit: CHAR_LIMIT, + previousLabels: variant.usePreviousLabels ? chain : null, + previousLabelCap: variant.previousLabelCap, + }); + const stepId = step.id ?? testCase.id; + if (dry) { + records.push({ variant: variant.name, sample, caseId: testCase.id, stepId, prompt }); + continue; + } + const result = await requestLabel({ apiKey, model, instruction: variant.instruction, prompt }); + if (result.error) { + records.push({ + variant: variant.name, + sample, + caseId: testCase.id, + stepId, + error: result.error, + }); + continue; + } + const { flags, wordCount, firstWord } = checkLabel(result.label, { + entries: stepEntries(step), + previousLabels: chain, + }); + chain.push(result.label); + records.push({ + variant: variant.name, + sample, + caseId: testCase.id, + stepId, + label: result.label, + production: step.productionLabel, + flags, + wordCount, + firstWord, + latencyMs: result.latencyMs, + inputTokens: result.inputTokens, + outputTokens: result.outputTokens, + }); + } +} + +async function pool(tasks, size) { + const queue = [...tasks]; + const workers = Array.from({ length: Math.min(size, queue.length) }, async () => { + while (queue.length > 0) { + await queue.shift()(); + } + }); + await Promise.all(workers); +} + +(async () => { + const args = parseArgs(process.argv.slice(2)); + const runVariants = args.variants + ? variants.filter((variant) => args.variants.includes(variant.name)) + : variants; + const runCases = args.cases ? cases.filter((c) => args.cases.includes(c.id)) : cases; + if (runVariants.length === 0 || runCases.length === 0) { + throw new Error('nothing selected — check --variants / --cases names'); + } + const apiKey = args.dry ? '' : loadKey(); + const records = []; + const tasks = []; + for (const variant of runVariants) { + for (let sample = 1; sample <= args.samples; sample++) { + for (const testCase of runCases) { + tasks.push(() => + runCase({ apiKey, model: args.model, variant, sample, testCase, dry: args.dry, records }), + ); + } + } + } + const totalSteps = runCases.reduce((sum, c) => sum + c.steps.length, 0); + console.log( + `${args.dry ? 'DRY RUN — rendering only' : `model ${args.model}`} · ${runVariants.length} variants × ${args.samples} samples × ${runCases.length} cases (${totalSteps} steps each pass)`, + ); + const started = Date.now(); + await pool(tasks, args.concurrency); + console.log(`done in ${((Date.now() - started) / 1000).toFixed(1)}s\n`); + + if (args.dry) { + for (const record of records.slice(0, 3)) { + console.log(`--- ${record.variant} / ${record.caseId} / ${record.stepId} ---`); + console.log(record.prompt); + console.log(''); + } + console.log(`rendered ${records.length} prompts (showing 3)`); + return; + } + + const aggregates = aggregate(records, args.model); + const variantNames = runVariants.map((variant) => variant.name); + const report = markdownReport({ + records, + aggregates, + runCases, + variantNames, + model: args.model, + samples: args.samples, + }); + fs.mkdirSync(RESULTS_DIR, { recursive: true }); + const stamp = new Date().toISOString().replace(/[:.]/g, '-'); + fs.writeFileSync( + path.join(RESULTS_DIR, `${stamp}.json`), + JSON.stringify({ args, records }, null, 2), + ); + fs.writeFileSync(path.join(RESULTS_DIR, 'latest.md'), report); + + console.log(report.split('## Per-case')[0]); + console.log(`full per-case tables: scripts/activity-labels/results/latest.md`); +})().catch((error) => { + console.error('ERR', error.message); + process.exit(1); +}); diff --git a/scripts/activity-labels/variants.js b/scripts/activity-labels/variants.js new file mode 100644 index 0000000000..f9ba3ca549 --- /dev/null +++ b/scripts/activity-labels/variants.js @@ -0,0 +1,239 @@ +/** + * Instruction variants under test. Each is a SINGLE-factor change against the + * production instruction so a result implicates one hypothesis: + * + * - baseline — ACTIVITY_INSTRUCTION exactly as the branch ships it + * - verbs — H: the Good-example verb distribution seeds register + * collapse (6/9 production labels opened "Confirmed") + * - ordered — H: the 4–9 word cap gets crowded out mid-paragraph; moving + * format constraints last improves adherence + * - continuity — H: showing the run's previous headers kills cross-batch + * redundancy (production pairs 2/3 and 7/8) + * + * The baseline is required from packages/api/dist so drift against the branch + * is impossible; the sentence table below is asserted against it so composed + * variants can never silently diverge from what production actually sends. + */ +const path = require('path'); + +const ROOT = path.resolve(__dirname, '..', '..'); + +/** + * The shipped instruction, read from the BUILT package so a variant can never + * be graded against a stale copy of it. Tries the workspace resolution first + * (an installed checkout), then the dist path directly, so the harness works + * whether or not `node_modules` is populated. `LABEL_EVAL_DIST` points it at + * another checkout's build — useful for grading one branch's instruction from + * a worktree that has not been built. + */ +function loadShippedInstruction() { + const candidates = [ + process.env.LABEL_EVAL_DIST, + '@librechat/api', + path.join(ROOT, 'packages/api/dist/index.cjs'), + ].filter(Boolean); + for (const candidate of candidates) { + try { + const { ACTIVITY_INSTRUCTION } = require(candidate); + if (typeof ACTIVITY_INSTRUCTION === 'string' && ACTIVITY_INSTRUCTION.length > 0) { + return ACTIVITY_INSTRUCTION; + } + } catch { + /* try the next candidate */ + } + } + throw new Error( + 'Could not load ACTIVITY_INSTRUCTION from a built @librechat/api.\n' + + 'Build it first (from the repo root):\n' + + ' npm run build:data-provider && npm run build:data-schemas && npm run build:api\n' + + 'Or point at an existing build:\n' + + ' LABEL_EVAL_DIST=/path/to/packages/api/dist/index.cjs node scripts/activity-labels/run.js', + ); +} + +const ACTIVITY_INSTRUCTION = loadShippedInstruction(); + +const S = { + role: 'You write the one-line header above a group of tool calls an AI agent just made.', + register: + 'Write it like a git commit subject: past tense, verb first, leading with the most distinctive file, name, or finding.', + outcome: + 'Say what the calls established or produced — the outcome, not the attempt. If they answered a question, the answer is the line.', + prohibitions: + 'Never name the tools, never count them, never echo the arguments: the cards below the header already show all three.', + format: 'Write 4 to 9 words, sentence case, no trailing punctuation, no quotes or markdown.', + good: 'Good: "Confirmed /mnt/data resets between calls". "Traced the leak to formatAgentMessages". "Found 3 failing auth tests".', + bad: 'Bad: "Ran 1 command". "Used bash_tool twice". "Executed ls /mnt/data". "Searched the codebase".', + failure: 'If every call failed, say what failed and why, plainly.', + output: 'Output only the line.', +}; + +const CONTINUITY_SENTENCE = + 'A "Previous headers" list may precede the batch: never restate one — if this batch continues that activity, say only what is new.'; + +/** Pre-P1 instruction — kept as the `legacy` variant for regression sweeps. */ +const LEGACY_ORDER = [ + S.role, + S.register, + S.outcome, + S.prohibitions, + S.format, + S.good, + S.bad, + S.failure, + S.output, +]; + +/** The shipped instruction (P1): ordered structure + continuity clause. */ +const SHIPPED_ORDER = [ + S.role, + S.outcome, + S.register, + S.good, + S.bad, + S.failure, + CONTINUITY_SENTENCE, + S.prohibitions, + S.format, + S.output, +]; + +/** `baseline` is whatever the BUILT dist ships. Before the P1 rebuild that is + * the legacy order, after it the shipped order; anything else means the + * sentence table here has drifted and composed variants are stale. */ +if ( + LEGACY_ORDER.join(' ') !== ACTIVITY_INSTRUCTION && + SHIPPED_ORDER.join(' ') !== ACTIVITY_INSTRUCTION +) { + console.warn( + 'WARN: variants.js sentence table has drifted from ACTIVITY_INSTRUCTION — composed variants are stale', + ); +} + +const VERB_CHOICE = + 'Open with whichever past-tense verb the outcome dictates — confirmed, found, traced, measured, wrote, ruled out, failed — not the same verb every time.'; +const DIVERSE_GOOD = + 'Good: "Traced the leak to formatAgentMessages". "Ruled out DNS as the failure cause". "Measured cold start at 412ms". "Found 3 failing auth tests".'; +const CONTINUITY = + 'A "Previous headers" list may precede the batch: those lines already stand above earlier groups, so never write a line that merely restates one. If this batch continues that same activity, lead with what is new or different in THIS batch.'; +const CONTINUITY_TIGHT = + 'A "Previous headers" list may precede the batch: never restate one — if this batch continues that activity, say only what is new.'; +const FORMAT_HARD = + 'Write 4 to 9 words, sentence case, no trailing punctuation, no quotes or markdown; when a batch found many things, keep only the most load-bearing one or two.'; + +const variants = [ + { + name: 'baseline', + usePreviousLabels: SHIPPED_ORDER.join(' ') === ACTIVITY_INSTRUCTION, + instruction: ACTIVITY_INSTRUCTION, + }, + { + name: 'legacy', + usePreviousLabels: false, + instruction: LEGACY_ORDER.join(' '), + }, + { + name: 'verbs', + usePreviousLabels: false, + instruction: [ + S.role, + S.register, + VERB_CHOICE, + S.outcome, + S.prohibitions, + S.format, + DIVERSE_GOOD, + S.bad, + S.failure, + S.output, + ].join(' '), + }, + { + name: 'ordered', + usePreviousLabels: false, + instruction: [ + S.role, + S.outcome, + S.register, + S.good, + S.bad, + S.failure, + S.prohibitions, + S.format, + S.output, + ].join(' '), + }, + { + name: 'continuity', + usePreviousLabels: true, + instruction: [ + S.role, + S.register, + S.outcome, + S.prohibitions, + S.format, + S.good, + S.bad, + S.failure, + CONTINUITY, + S.output, + ].join(' '), + }, + { + name: 'examples', + usePreviousLabels: false, + instruction: [ + S.role, + S.register, + S.outcome, + S.prohibitions, + S.format, + DIVERSE_GOOD, + S.bad, + S.failure, + S.output, + ].join(' '), + }, + { + name: 'composed', + usePreviousLabels: true, + instruction: [ + S.role, + S.outcome, + S.register, + DIVERSE_GOOD, + S.bad, + S.failure, + CONTINUITY_TIGHT, + S.prohibitions, + FORMAT_HARD, + S.output, + ].join(' '), + }, + { + name: 'shipping-full', + usePreviousLabels: true, + /** Whole-run history instead of the 3-label recency window: does more + * story beat recency, or does it dilute the batch content? */ + previousLabelCap: Infinity, + instruction: SHIPPED_ORDER.join(' '), + }, + { + name: 'shipping', + usePreviousLabels: true, + instruction: [ + S.role, + S.outcome, + S.register, + S.good, + S.bad, + S.failure, + CONTINUITY_TIGHT, + S.prohibitions, + S.format, + S.output, + ].join(' '), + }, +]; + +module.exports = { variants };