mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-08-04 14:57:42 +00:00
a07c0e4ae8
Grades fast-model activity-label headers against a fixed corpus so instruction changes are measured rather than eyeballed on one conversation. This existed untracked while the continuity work was developed; committing it because it is the only reproducible record of WHY `ACTIVITY_INSTRUCTION` is ordered and capped the way it is. - captured.json: 9 real production payloads pulled verbatim from Langfuse with the headers that shipped. Irreplaceable — traces age out. - corpus.js: 17 cases / 28 steps. The captured run replays as one sequence, plus synthetic cases for the modes it never exercised (all-failed, partial, parallel batches, rapid near-duplicates, entry overflow, truncated output, error-shaped success). Multi-step cases chain each generated label into the next step's context, which is what makes cross-batch redundancy measurable at all. - prompt.js: faithful port of the SDK's buildActivityLabelPrompt so synthetic cases render the bytes production sends, plus a previousLabelCap knob for continuity-window experiments. - variants.js: single-factor instruction variants. The baseline is read from the BUILT package (workspace resolution, then dist, then LABEL_EVAL_DIST) so a variant can never be graded against a stale copy of the shipped instruction. - checks.js: length/punctuation/markdown/tool-echo/count-echo, plus overlap split into `restate` (adds nothing over an earlier header) vs `template` (same frame, new payload — often fine). - run.js / rescore.js: live runner on the production wire shape (max_tokens 256) and an offline re-grader, so metric fixes never require re-spending on the API. Results are gitignored — regenerable, and 292K of the 364K. A full sweep is ~$0.03 per variant and ~45s. Findings are recorded in the README, two of them counter-intuitive: enumerating acceptable opening verbs ANCHORED the model rather than diversifying it (Confirmed 18→23, opener diversity halved), and diverse examples alone changed nothing. Sentence order is load-bearing, so a tidying reshuffle of ACTIVITY_INSTRUCTION regresses real output.
72 lines
2.3 KiB
JavaScript
72 lines
2.3 KiB
JavaScript
/**
|
||
* Offline rescore: recompute checks over a stored results JSON after a
|
||
* metric change, without re-calling the API. Chains are rebuilt from the
|
||
* stored labels in push order (steps within a case ran serially).
|
||
*
|
||
* Usage: node scripts/activity-labels/rescore.js [results/<file>.json]
|
||
*/
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
|
||
const { cases, stepEntries } = require('./corpus');
|
||
const { checkLabel } = require('./checks');
|
||
const { aggregate, markdownReport } = require('./report');
|
||
|
||
const RESULTS_DIR = path.join(__dirname, 'results');
|
||
|
||
function newestResults() {
|
||
const files = fs
|
||
.readdirSync(RESULTS_DIR)
|
||
.filter((file) => file.endsWith('.json'))
|
||
.sort();
|
||
if (files.length === 0) {
|
||
throw new Error('no stored results to rescore');
|
||
}
|
||
return path.join(RESULTS_DIR, files[files.length - 1]);
|
||
}
|
||
|
||
const sourcePath = process.argv[2] ? path.resolve(process.argv[2]) : newestResults();
|
||
const { args, records } = JSON.parse(fs.readFileSync(sourcePath, 'utf8'));
|
||
|
||
const stepsByCase = new Map(
|
||
cases.map((testCase) => [
|
||
testCase.id,
|
||
new Map(testCase.steps.map((step) => [step.id ?? testCase.id, step])),
|
||
]),
|
||
);
|
||
|
||
const chains = new Map();
|
||
for (const record of records) {
|
||
if (record.error || record.label == null) {
|
||
continue;
|
||
}
|
||
const key = `${record.variant} ${record.sample} ${record.caseId}`;
|
||
if (!chains.has(key)) {
|
||
chains.set(key, []);
|
||
}
|
||
const chain = chains.get(key);
|
||
const step = stepsByCase.get(record.caseId)?.get(record.stepId);
|
||
const { flags, wordCount, firstWord } = checkLabel(record.label, {
|
||
entries: step != null ? stepEntries(step) : [],
|
||
previousLabels: chain,
|
||
});
|
||
record.flags = flags;
|
||
record.wordCount = wordCount;
|
||
record.firstWord = firstWord;
|
||
chain.push(record.label);
|
||
}
|
||
|
||
const variantNames = [...new Set(records.map((record) => record.variant))];
|
||
const runCases = cases.filter((testCase) => records.some((r) => r.caseId === testCase.id));
|
||
const report = markdownReport({
|
||
records,
|
||
aggregates: aggregate(records, args.model),
|
||
runCases,
|
||
variantNames,
|
||
model: args.model,
|
||
samples: args.samples,
|
||
});
|
||
fs.writeFileSync(path.join(RESULTS_DIR, 'latest.md'), report);
|
||
console.log(`rescored ${path.basename(sourcePath)}`);
|
||
console.log(report.split('## Per-case')[0]);
|
||
console.log('full per-case tables: scripts/activity-labels/results/latest.md');
|