🦠 test: Add Portable Favorites Contracts and Mutation Testing (#17690)

Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
lia-by-librechat[bot] 2026-10-31 16:16:39 -04:00 • committed by GitHub
parent e3e7dc6d7c
commit e70cec8fd5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 2670 additions and 238 deletions

296
scripts/crap.mts Normal file
View file

@ -0,0 +1,296 @@
import { Linter } from 'eslint';
import { parseArgs } from 'node:util';
import { fileURLToPath } from 'node:url';
import parser from '@typescript-eslint/parser';
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import { basename, dirname, join, relative, resolve, sep } from 'node:path';
import type { Rule } from 'eslint';
const ROOT = fileURLToPath(new URL('../', import.meta.url));
const FAVORITES = join(ROOT, 'packages/data-schemas/src/methods/favorite.ts');
const FUNCTION_TYPES = new Set([
'FunctionDeclaration',
'FunctionExpression',
'ArrowFunctionExpression',
]);
export interface FunctionReport {
file: string;
function: string;
line: number;
column: number;
complexity: number;
statements: number;
coveredStatements: number;
/** Fraction in [0, 1], excluding nested functions; null means no instrumentation. */
statementCoverage: number | null;
crap: number | null;
}
type FunctionNode = Extract<
Rule.Node,
{ type: 'FunctionDeclaration' | 'FunctionExpression' | 'ArrowFunctionExpression' }
>;
interface MeasuredFunction {
head: number;
start: number;
end: number;
report: FunctionReport;
}
const isRecord = (value: unknown): value is Record<string, unknown> =>
value !== null && typeof value === 'object' && !Array.isArray(value);
export function crapScore(complexity: number, coverage: number | null): number | null {
if (!Number.isInteger(complexity) || complexity < 1) {
throw new Error('Complexity must be a positive integer');
}
if (coverage === null) return null;
if (!Number.isFinite(coverage) || coverage < 0 || coverage > 1) {
throw new Error('Statement coverage must be a fraction in [0, 1]');
}
return complexity ** 2 * (1 - coverage) ** 3 + complexity;
}
function functionName(node: FunctionNode, context: Rule.RuleContext): string {
if ('id' in node && node.id?.type === 'Identifier') return node.id.name;
const parent = node.parent;
if (parent.type === 'VariableDeclarator' && parent.id.type === 'Identifier')
return parent.id.name;
if (
parent.type === 'Property' ||
parent.type === 'MethodDefinition' ||
parent.type === 'PropertyDefinition'
) {
return context.sourceCode.getText(parent.key);
}
return '(anonymous)';
}
/** Uses ESLint's real classic rule, not an approximation of its branch counting. */
export function measureFunctions(
file: string,
source: string,
coverage?: unknown,
): FunctionReport[] {
const absoluteFile = resolve(file);
const functions: MeasuredFunction[] = [];
const lines = source.split(/\r\n|[\n\r\u2028\u2029]/u);
const offsets = [0];
for (const match of source.matchAll(/\r\n|[\n\r\u2028\u2029]/gu)) {
offsets.push(match.index + match[0].length);
}
const offset = (position: unknown): number => {
if (!isRecord(position)) throw new Error(`${file}: invalid coverage location`);
const { line, column } = position;
if (
typeof line !== 'number' ||
typeof column !== 'number' ||
!Number.isInteger(line) ||
!Number.isInteger(column) ||
line < 1 ||
line > lines.length ||
column < 0 ||
column > lines[line - 1].length
) {
throw new Error(`${file}: coverage location is outside the source`);
}
return offsets[line - 1] + column;
};
const displayFile = relative(ROOT, absoluteFile);
const name = displayFile.startsWith(`..${sep}`) ? absoluteFile : displayFile.split(sep).join('/');
const linter = new Linter();
const messages = linter.verify(
source,
{
files: ['**/*.{js,jsx,ts,tsx,mjs,mts,cjs,cts}'],
languageOptions: { parser, ecmaVersion: 'latest', sourceType: 'module' },
linterOptions: { noInlineConfig: true },
plugins: {
metrics: {
rules: {
functions: {
create(context: Rule.RuleContext) {
return {
'FunctionDeclaration, FunctionExpression, ArrowFunctionExpression'(
node: Rule.Node,
) {
if (
node.type !== 'FunctionDeclaration' &&
node.type !== 'FunctionExpression' &&
node.type !== 'ArrowFunctionExpression'
)
return;
const parent = node.parent;
// ESLint reports methods at their property key and arrows at the => token.
const isMethod =
parent.type === 'Property' ||
parent.type === 'MethodDefinition' ||
parent.type === 'PropertyDefinition';
const declaration = isMethod ? parent : node;
const head =
!isMethod && node.type === 'ArrowFunctionExpression'
? context.sourceCode.getTokenBefore(
node.body,
(token) => token.value === '=>',
)
: declaration;
if (!node.range || !node.loc || !head?.range)
throw new Error('Parser did not provide ranges');
functions.push({
head: head.range[0],
start: node.range[0],
end: node.range[1],
report: {
file: name,
function: functionName(node, context),
line: node.loc.start.line,
column: node.loc.start.column + 1,
complexity: 0,
statements: 0,
coveredStatements: 0,
statementCoverage: null,
crap: null,
},
});
},
};
},
},
},
},
},
rules: {
'metrics/functions': 'error',
complexity: ['error', { max: 0, variant: 'classic' }],
},
},
{ filename: basename(absoluteFile) },
);
for (const message of messages) {
if (message.fatal || (message.severity === 2 && message.ruleId !== 'complexity')) {
throw new Error(`${file}:${message.line}: ${message.message}`);
}
// ESLint also measures implicit class initializers, which are not source functions.
if (
message.ruleId !== 'complexity' ||
!FUNCTION_TYPES.has(message.nodeType ?? '') ||
/^Class (?:field initializer|static block)/u.test(message.message)
) {
continue;
}
const point = offset({ line: message.line, column: message.column - 1 });
const owner = functions.find((fn) => fn.head === point);
const complexity = /has a complexity of (\d+)\./u.exec(message.message)?.[1];
if (!owner || !complexity || owner.report.complexity !== 0) {
throw new Error(`${file}: unrecognized ESLint complexity diagnostic: ${message.message}`);
}
owner.report.complexity = Number(complexity);
}
if (functions.some(({ report }) => report.complexity === 0)) {
throw new Error(`${file}: ESLint did not measure every function`);
}
if (coverage !== undefined) {
if (
!isRecord(coverage) ||
typeof coverage.path !== 'string' ||
resolve(coverage.path) !== absoluteFile ||
!isRecord(coverage.statementMap) ||
!isRecord(coverage.s)
) {
throw new Error(`${file}: invalid coverage entry or source path mismatch`);
}
const statements = Object.entries(coverage.statementMap);
if (statements.length !== Object.keys(coverage.s).length) {
throw new Error(`${file}: statement map/counter mismatch`);
}
for (const [id, location] of statements) {
const hits = coverage.s[id];
if (
!isRecord(location) ||
typeof hits !== 'number' ||
!Number.isSafeInteger(hits) ||
hits < 0
) {
throw new Error(`${file}: invalid statement or counter ${id}`);
}
const start = offset(location.start);
const end = offset(location.end);
if (start >= end) throw new Error(`${file}: invalid statement range ${id}`);
// The declaration of a nested function belongs to its parent; its body does not.
const owner = functions.findLast((fn) => fn.start <= start && end <= fn.end);
if (!owner) continue;
owner.report.statements++;
if (hits > 0) owner.report.coveredStatements++;
}
}
return functions.map(({ report }) => {
report.statementCoverage = report.statements
? report.coveredStatements / report.statements
: null;
report.crap = crapScore(report.complexity, report.statementCoverage);
return report;
});
}
export function createReport(coverageFile: string, sources: string[] = [FAVORITES]) {
const coverage: unknown = JSON.parse(readFileSync(resolve(coverageFile), 'utf8'));
if (!isRecord(coverage)) throw new Error('Expected an Istanbul coverage object');
const entries = new Map<string, unknown>();
for (const [file, entry] of Object.entries(coverage)) {
const key = resolve(file);
if (entries.has(key)) throw new Error(`Duplicate coverage path: ${file}`);
entries.set(key, entry);
}
return {
complexityMetric: 'eslint/classic',
coverageMetric: 'per-function statement fraction (nested functions excluded)',
functions: [...new Set(sources.map((file) => resolve(file)))].flatMap((file) =>
measureFunctions(file, readFileSync(file, 'utf8'), entries.get(file)),
),
};
}
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
try {
const { values, positionals } = parseArgs({
allowPositionals: true,
options: { output: { type: 'string', short: 'o' }, help: { type: 'boolean', short: 'h' } },
});
if (values.help) {
console.log(
'Usage: node scripts/crap.mts [--output path] <coverage-final.json> [source.ts ...]',
);
} else {
const [coverageFile, ...sources] = positionals;
if (!coverageFile) throw new Error('Expected coverage-final.json; use --help for usage');
const files = sources.length ? sources : [FAVORITES];
const output = resolve(values.output ?? join(dirname(resolve(coverageFile)), 'crap.json'));
if ([coverageFile, ...files].some((file) => resolve(file) === output)) {
throw new Error('Output must not overwrite coverage or source files');
}
const report = createReport(coverageFile, files);
mkdirSync(dirname(output), { recursive: true });
writeFileSync(output, `${JSON.stringify(report, null, 2)}\n`);
console.log(
'Advisory CRAP baseline | CC: ESLint classic | Coverage: statements, excluding nested functions',
);
console.log('File\tFunction\tLine:Column\tCC\tStatement coverage\tCRAP');
for (const row of report.functions) {
const coverage =
row.statementCoverage === null
? 'N/A'
: `${(row.statementCoverage * 100).toFixed(1)}% (${row.coveredStatements}/${row.statements})`;
console.log(
`${row.file}\t${row.function}\t${row.line}:${row.column}\t${row.complexity}\t${coverage}\t${row.crap?.toFixed(2) ?? 'N/A'}`,
);
}
console.log(`JSON: ${output}`);
}
} catch (error) {
console.error(`crap: ${error instanceof Error ? error.message : String(error)}`);
process.exitCode = 1;
}
}

236
scripts/crap.test.mts Normal file
View file

@ -0,0 +1,236 @@
import { test } from 'node:test';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { strict as assert } from 'node:assert';
import { spawnSync } from 'node:child_process';
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { crapScore, createReport, measureFunctions } from './crap.mts';
const FILE = resolve('sample.ts');
const SCRIPT = fileURLToPath(new URL('./crap.mts', import.meta.url));
function coverageFor(source: string, statements: [string, number][], file = FILE) {
const position = (index: number) => {
const lines = source.slice(0, index).split('\n');
return { line: lines.length, column: lines.at(-1)!.length };
};
return {
path: file,
statementMap: Object.fromEntries(
statements.map(([text], id) => {
const start = source.indexOf(text);
assert.notEqual(start, -1, `Missing fixture statement: ${text}`);
return [id, { start: position(start), end: position(start + text.length) }];
}),
),
s: Object.fromEntries(statements.map(([, hits], id) => [id, hits])),
};
}
test('CRAP uses statement coverage fractions and preserves unknown coverage', () => {
assert.equal(crapScore(4, 1), 4);
assert.equal(crapScore(4, 0), 20);
assert.equal(crapScore(3, 0.5), 4.125);
assert.equal(crapScore(4, null), null);
for (const coverage of [-1, 1.1, NaN, Infinity]) {
assert.throws(() => crapScore(4, coverage), /fraction/);
}
for (const complexity of [0, -1, 1.5, Infinity]) {
assert.throws(() => crapScore(complexity, 1), /positive integer/);
}
});
test('real ESLint counts classic switch cases, logical and conditional branches in TypeScript', () => {
const source = `/* eslint complexity: off */
function choose(value: number): number {
switch (value) {
case 0: return 0;
case 1: return 1;
default: return value > 2 && value < 5 ? 2 : -1;
}
}`;
const [row] = measureFunctions(FILE, source);
assert.equal(row.function, 'choose');
assert.equal(row.line, 2);
assert.equal(row.complexity, 5);
assert.equal(row.statementCoverage, null);
assert.equal(row.crap, null);
});
test('nested functions own only their statements, including when sharing a line', () => {
for (const separator of [' ', '\n']) {
const source = `const outer = () => {${separator}const inner = () => 1;${separator}return inner(); };`;
const coverage = coverageFor(source, [
['const inner = () => 1;', 0],
['1', 1],
['return inner();', 0],
]);
const [outer, inner] = measureFunctions(FILE, source, coverage);
assert.equal(outer.function, 'outer');
assert.equal(outer.complexity, 1);
assert.equal(outer.statements, 2);
assert.equal(outer.coveredStatements, 0);
assert.equal(outer.statementCoverage, 0);
assert.equal(outer.crap, 2);
assert.equal(inner.function, 'inner');
assert.equal(inner.statements, 1);
assert.equal(inner.statementCoverage, 1);
assert.equal(inner.crap, inner.complexity);
}
});
test('nested branches and implicit class initializers do not inflate enclosing complexity', () => {
const source = `function outer() {
class Box {
value = true ? 1 : 2;
handler = () => false ? 1 : 2;
}
return Box;
}`;
const rows = measureFunctions(FILE, source);
assert.deepEqual(
rows.map((row) => [row.function, row.complexity]),
[
['outer', 1],
['handler', 2],
],
);
});
test('attributes method diagnostics reported before the function node itself', () => {
const source = `const box = {
read(value: number) { if (value) return 1; return 0; },
write: (value: number) => value ? 1 : 0,
};`;
const rows = measureFunctions(FILE, source);
assert.deepEqual(
rows.map((row) => [row.function, row.complexity]),
[
['read', 2],
['write', 2],
],
);
});
test('missing or empty instrumentation is N/A, not 100%; explicit zero hits stays zero', () => {
const source = 'const value = 1; function read() { return value; }';
const missing = [
undefined,
coverageFor(source, []),
coverageFor(source, [['const value = 1;', 1]]),
];
for (const coverage of missing) {
const [row] = measureFunctions(FILE, source, coverage);
assert.equal(row.statements, 0);
assert.equal(row.statementCoverage, null);
assert.equal(row.crap, null);
}
const [zero] = measureFunctions(FILE, source, coverageFor(source, [['return value;', 0]]));
assert.equal(zero.statementCoverage, 0);
assert.equal(zero.crap, 2);
});
test('rejects malformed coverage, counter mismatches and stale out-of-source ranges', () => {
const source = 'function read() { return 1; }';
const valid = coverageFor(source, [['return 1;', 1]]);
const bad = [
null,
[],
{},
{ ...valid, path: resolve('other.ts') },
{ ...valid, s: {} },
{ ...valid, s: { 1: 1 } },
{ ...valid, s: { 0: -1 } },
{ ...valid, s: { 0: 0.5 } },
{ ...valid, s: { 0: '1' } },
...[
{ line: 0, column: 0 },
{ line: 2, column: 0 },
{ line: 1, column: source.length + 1 },
{ line: 1, column: -1 },
{ line: 1, column: 1.5 },
{ line: 1, column: source.length },
].map((start) => ({ ...valid, statementMap: { 0: { ...valid.statementMap[0], start } } })),
];
for (const coverage of bad) {
assert.throws(() => measureFunctions(FILE, source, coverage), /invalid|mismatch|outside/i);
}
assert.throws(() => measureFunctions(FILE, 'function broken('), /Parsing error/);
});
test('validates CRLF source locations', () => {
const source = 'function read() {\r\n return 1;\r\n}';
const coverage = {
path: FILE,
statementMap: { 0: { start: { line: 2, column: 2 }, end: { line: 2, column: 11 } } },
s: { 0: 1 },
};
assert.equal(measureFunctions(FILE, source, coverage)[0].statementCoverage, 1);
});
test('CLI resolves source/output paths from package cwd and writes advisory JSON', (t) => {
const directory = mkdtempSync(join(tmpdir(), 'crap-'));
t.after(() => rmSync(directory, { recursive: true, force: true }));
const cwd = join(directory, 'package');
const file = join(cwd, 'src', 'favorite.ts');
const coverageFile = join(cwd, 'coverage', 'favorites', 'coverage-final.json');
mkdirSync(join(cwd, 'src'), { recursive: true });
mkdirSync(join(cwd, 'coverage', 'favorites'), { recursive: true });
const source = 'export function read(): number { return 1; }';
writeFileSync(file, source);
writeFileSync(
coverageFile,
JSON.stringify({ [file]: coverageFor(source, [['return 1;', 0]], file) }),
);
const run = (...args: string[]) =>
spawnSync(process.execPath, [SCRIPT, ...args], { cwd, encoding: 'utf8' });
const result = run('coverage/favorites/coverage-final.json', 'src/favorite.ts');
assert.equal(result.status, 0, result.stderr);
assert.match(result.stdout, /Advisory CRAP/);
assert.match(result.stdout, /0\.0% \(0\/1\)\s+2\.00/);
const report = JSON.parse(readFileSync(join(cwd, 'coverage', 'favorites', 'crap.json'), 'utf8'));
assert.deepEqual(report, createReport(coverageFile, [file]));
assert.equal(report.functions[0].file, file);
assert.equal(run('--output', 'custom/crap.json', coverageFile, file).status, 0);
assert.deepEqual(JSON.parse(readFileSync(join(cwd, 'custom', 'crap.json'), 'utf8')), report);
assert.equal(run('--output', file, coverageFile, file).status, 1);
assert.equal(readFileSync(file, 'utf8'), source);
assert.throws(() => createReport(join(directory, 'missing.json'), [file]), /ENOENT/);
assert.throws(() => createReport(coverageFile, [join(directory, 'missing.ts')]), /ENOENT/);
writeFileSync(coverageFile, '{}');
const unknown = run(coverageFile, file);
assert.equal(unknown.status, 0, unknown.stderr);
assert.match(unknown.stdout, /N\/A\s+N\/A/);
const defaults = run(coverageFile);
assert.equal(defaults.status, 0, defaults.stderr);
const defaultReport = JSON.parse(
readFileSync(join(cwd, 'coverage', 'favorites', 'crap.json'), 'utf8'),
);
assert.ok(defaultReport.functions.length > 0);
assert.ok(
defaultReport.functions.every(
(row: { file: string }) => row.file === 'packages/data-schemas/src/methods/favorite.ts',
),
);
writeFileSync(coverageFile, '{invalid');
assert.equal(run(coverageFile, file).status, 1);
writeFileSync(coverageFile, '[]');
assert.throws(() => createReport(coverageFile, [file]), /Istanbul coverage object/);
});
test('importing the reporter does not execute its CLI', () => {
const result = spawnSync(
process.execPath,
[
'--input-type=module',
'-e',
`import ${JSON.stringify(new URL('./crap.mts', import.meta.url).href)}`,
],
{ encoding: 'utf8' },
);
assert.equal(result.status, 0, result.stderr);
assert.equal(result.stdout, '');
assert.equal(result.stderr, '');
});