LibreChat/.github/scripts/guard-specs.test.mjs
Danny Avila 97f7fcdbac
🥅 ci: Run Playwright After Merges to Dev and Guard Specs Under Test Selection (#18049)
* 🥅 ci: Run Playwright After Every Merge to Dev

The mock e2e suite ran on pull requests, nightly on main and on demand, so
when a spec broke on dev no run on dev could go red: the vote workflow that
runs the suite after merges cannot fail by design.

Add a push trigger for dev with the pull request path filter. A push runs
the pull request profile with no lane narrowed (10 memory shards, 2 Redis
transport shards, both MCP lanes and MCP Apps), and the gate job reports it.

Pushes take their own concurrency group that never cancels: the run in
progress finishes, and GitHub keeps one pending run per group, so merges
that land meanwhile collapse into one run of the newest commit. Pull
request, nightly and manual runs keep their per-ref cancelling group.

The matrix test now evaluates the workflow's own expressions per event
rather than matching their text.

* 🔭 ci: Always Run Source-Scanning Guard Specs Under Test Selection

Codegraph test selection narrows a pull request run to the test files an
import graph reaches. A spec that reads source with fs has no import edge
to what it reads: toolCapabilityGates.spec.js scans four source trees, was
not selected when #17864 changed one of the files it counts, and failed on
dev afterwards.

List every such spec, with the paths it reads, in .github/guard-specs.json.
Both selectors now add a workspace's guards to a FILES decision and turn a
NONE into a guards-only run when the change touches a path one of them
reads, renames a file, or edits the list. A list that cannot be read runs
the workspace FULL. FULL decisions and every earlier fail-open path are
unchanged.

The rule keys off the decision, not the event. A selection with no file
left in the checkout gets no guards, so the test job still runs FULL on a
stale selection instead of running only the guards.

guard-specs.test.mjs fails when a spec in a gated workspace reads the
repository through fs without being listed, when a listed path is missing,
and when a workflow would not run for a path one of its guards reads. That
last check adds the five backend and one frontend paths the triggers
lacked. The test also runs both real select steps against canned codegraph
responses.
2026-11-11 10:30:09 -05:00

540 lines
20 KiB
JavaScript

import yaml from 'js-yaml';
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';
import { spawnSync } from 'node:child_process';
import {
copyFileSync,
existsSync,
mkdirSync,
mkdtempSync,
readdirSync,
readFileSync,
rmSync,
statSync,
writeFileSync,
} from 'node:fs';
import { condition } from './expressions.mjs';
import {
loadManifest,
MANIFEST_PATH,
parseManifest,
readsRepository,
SPEC_FILE,
withGuards,
} from './guard-specs.mjs';
const scriptsDir = fileURLToPath(new URL('.', import.meta.url));
const repoRoot = fileURLToPath(new URL('../..', import.meta.url));
const manifest = loadManifest();
const WORKFLOWS = ['backend-review.yml', 'frontend-review.yml'];
const workflows = new Map(
WORKFLOWS.map((file) => {
const workflow = yaml.load(readFileSync(join(repoRoot, '.github/workflows', file), 'utf8'));
const selector = workflow.jobs.codegraph_select.steps.find((step) => step.id === 'sel');
return [file, { workflow, selector }];
}),
);
/** `emit <key> <workspace> [ignore]` is how a selector names each workspace it gates. */
const gated = new Map(
WORKFLOWS.flatMap((file) =>
Array.from(
workflows.get(file).selector.run.matchAll(/^\s*emit \w+ (\S+)(?: (?:'[^']*'|""))?$/gm),
([, workspace]) => [workspace, { file }],
),
),
);
const SKIPPED_DIRECTORIES = new Set(['node_modules', 'dist', 'build', 'coverage', '.turbo']);
function specsUnder(directory) {
return readdirSync(join(repoRoot, directory), { withFileTypes: true }).flatMap((entry) => {
const path = `${directory}/${entry.name}`;
if (entry.isDirectory()) return SKIPPED_DIRECTORIES.has(entry.name) ? [] : specsUnder(path);
return SPEC_FILE.test(entry.name) ? [path] : [];
});
}
const workspaceOf = (spec) =>
[...gated.keys()].find((workspace) => spec.startsWith(`${workspace}/`));
/** GitHub's path filter, for the shapes these triggers use: exact paths and `dir/**`. */
const triggers = (pattern, path) =>
pattern.endsWith('/**') ? path.startsWith(pattern.slice(0, -2)) : pattern === path;
test('both selectors gate the workspaces this test walks', () => {
assert.deepEqual(
[...gated].map(([workspace, { file }]) => [workspace, file]),
[
['api', 'backend-review.yml'],
['packages/api', 'backend-review.yml'],
['packages/data-provider', 'backend-review.yml'],
['packages/data-schemas', 'backend-review.yml'],
['client', 'frontend-review.yml'],
['packages/client', 'frontend-review.yml'],
],
);
});
test('only a pull request synchronize is narrowed; every other event runs each workspace FULL', () => {
const event = (event_name, action) => ({ github: { event_name, event: { action } }, vars: {} });
const unselected = { build: { result: 'success' }, codegraph_select: { outputs: {} } };
for (const [file, { workflow }] of workflows) {
const selects = (context) => condition(workflow.jobs.codegraph_select.if, context);
assert.equal(selects(event('pull_request', 'synchronize')), true, file);
for (const context of [
event('pull_request', 'opened'),
event('pull_request', 'reopened'),
event('push'),
event('workflow_dispatch'),
event('merge_group', 'checks_requested'),
]) {
assert.equal(selects(context), false, `${file} ${context.github.event_name}`);
}
const gatedJobs = Object.values(workflow.jobs).filter((job) =>
[job.needs].flat().includes('codegraph_select'),
);
assert.ok(gatedJobs.length > 0, file);
for (const job of gatedJobs) {
assert.equal(
condition(job.if, { needs: unselected }, { cancelled: () => false }),
true,
`${file} ${job.name}`,
);
}
}
});
test('every spec that reads the repository through fs is listed as a guard or exempt', () => {
const listed = new Set([...Object.keys(manifest.guards), ...Object.keys(manifest.exempt)]);
const unlisted = [...gated.keys()]
.flatMap(specsUnder)
.filter((spec) => !listed.has(spec))
.filter((spec) => readsRepository(readFileSync(join(repoRoot, spec), 'utf8')));
assert.deepEqual(
unlisted,
[],
`These specs read files with fs from a path anchored in the repository, so codegraph test ` +
`selection cannot tell when a change affects them. Add each to "guards" in ` +
`${MANIFEST_PATH} with the repository paths it reads, or to "exempt" with the reason it ` +
`reads no tracked file.`,
);
});
test('every listed spec exists, sits in a gated workspace and reads paths that exist', () => {
for (const spec of [...Object.keys(manifest.guards), ...Object.keys(manifest.exempt)]) {
assert.ok(existsSync(join(repoRoot, spec)), `${spec} is listed but does not exist`);
assert.ok(workspaceOf(spec), `${spec} is not in a workspace that test selection gates`);
}
for (const [spec, scans] of Object.entries(manifest.guards)) {
for (const scan of scans) {
const target = join(repoRoot, scan);
assert.ok(existsSync(target), `${spec} lists ${scan}, which does not exist`);
assert.equal(
statSync(target).isDirectory(),
scan.endsWith('/'),
`${spec} lists ${scan}: a directory ends with a slash and a file does not`,
);
}
}
});
test('a workflow runs whenever a path one of its guard specs reads changes', () => {
for (const [spec, scans] of Object.entries(manifest.guards)) {
const { file } = gated.get(workspaceOf(spec));
const { workflow } = workflows.get(file);
for (const event of ['pull_request', 'push']) {
const patterns = workflow.on[event].paths.filter((pattern) => !pattern.startsWith('!'));
for (const scan of scans) {
assert.ok(
patterns.some((pattern) => triggers(pattern, scan)),
`${spec} reads ${scan}, but ${file} does not run on ${event} when it changes; ` +
`add it to that trigger's paths`,
);
}
}
}
});
test('the detector flags repository reads and ignores temp-only and import-only specs', () => {
assert.ok(readsRepository("fs.readFileSync(path.join(__dirname, '../index.js'), 'utf8')"));
assert.ok(readsRepository('for (const e of readdirSync(join(process.cwd(), "src"))) {}'));
assert.ok(readsRepository('await readFile(new URL("./a.css", import.meta.url))'));
assert.ok(readsRepository("globSync('src/**/*.ts', { cwd: __dirname })"));
assert.ok(!readsRepository("fs.readFileSync(path.join(os.tmpdir(), 'out.txt'))"));
assert.ok(!readsRepository("const { run } = require(path.join(__dirname, 'run.js'))"));
assert.ok(!readsRepository("jest.mock('fs'); expect(reader.readFile).toBeDefined()"));
});
test('a manifest is refused when a path could split, glob or leave the repository', () => {
const guards = (entries) => JSON.stringify({ guards: entries });
assert.throws(() => parseManifest('{}'));
assert.throws(() => parseManifest(guards({ 'api/a b.spec.js': ['api/'] })));
assert.throws(() => parseManifest(guards({ 'api/*.spec.js': ['api/'] })));
assert.throws(() => parseManifest(guards({ 'api/$(id).spec.js': ['api/'] })));
assert.throws(() => parseManifest(guards({ 'api/../a.spec.js': ['api/'] })));
assert.throws(() => parseManifest(guards({ '/api/a.spec.js': ['api/'] })));
assert.throws(() => parseManifest(guards({ 'api/a.js': ['api/'] })));
assert.throws(() => parseManifest(guards({ 'api/a.spec.js': [] })));
assert.throws(() => parseManifest(guards({ 'api/a.spec.js': ['api/ server/'] })));
assert.throws(() => parseManifest(guards({ 'api/a.spec.js': 'api/' })));
assert.throws(() =>
parseManifest(
JSON.stringify({ guards: { 'api/a.spec.js': ['api/'] }, exempt: { 'api/a.spec.js': 'x' } }),
),
);
assert.throws(() =>
parseManifest(JSON.stringify({ guards: {}, exempt: { 'api/a.spec.js': '' } })),
);
assert.deepEqual(parseManifest(guards({ 'api/a.spec.js': ['api/', 'b.json'] })).guards, {
'api/a.spec.js': ['api/', 'b.json'],
});
});
const FIXTURE = {
guards: {
'api/server/guard.spec.js': ['api/server/', 'packages/api/src/'],
'packages/api/src/guard.spec.ts': ['packages/api/src/'],
'packages/api/src/scan.integration.spec.ts': ['packages/api/src/'],
'packages/data-schemas/src/guard.spec.ts': ['api/'],
'client/src/guard.spec.ts': ['client/index.html', 'packages/chat/src/'],
'packages/client/src/guard.spec.tsx': ['packages/client/src/theme/'],
},
};
test('a rename, or a change to the list or its reader, counts for every guard', () => {
const run = (changed) =>
withGuards({
manifest: FIXTURE,
mode: 'NONE',
workspace: 'packages/client',
changed,
exists: () => true,
});
assert.deepEqual(run([{ path: 'docs/a.txt', status: 'modified' }]), []);
assert.deepEqual(run([{ path: 'docs/b.txt', status: 'renamed' }]), ['src/guard.spec.tsx']);
assert.deepEqual(run([{ path: MANIFEST_PATH, status: 'modified' }]), ['src/guard.spec.tsx']);
assert.deepEqual(run([{ path: '.github/scripts/guard-specs.mjs' }]), ['src/guard.spec.tsx']);
assert.deepEqual(run([{ path: 'packages/client/src/guard.spec.tsx' }]), ['src/guard.spec.tsx']);
assert.deepEqual(run([{ path: 'packages/client/src/theme/a.css' }]), ['src/guard.spec.tsx']);
assert.deepEqual(run([{ path: 'packages/client/src/themes.ts' }]), []);
assert.throws(() => run([{ filename: 'a' }]));
assert.throws(() => run({}));
});
test('a selection keeps its guards only while one of its files is in the checkout', () => {
const selected = ['src/gone.spec.tsx', 'src/kept.spec.tsx'];
const run = (exists) =>
withGuards({
manifest: FIXTURE,
mode: 'FILES',
workspace: 'packages/client',
changed: [],
selected,
exists,
});
assert.deepEqual(
run((file) => file === 'src/kept.spec.tsx'),
[...selected, 'src/guard.spec.tsx'],
);
assert.deepEqual(
run(() => false),
selected,
);
assert.throws(() => run(undefined));
});
const mode = (value, files) => ({ mode: value, ...(files ? { files } : {}) });
const NONE = mode('NONE');
const FULL = mode('FULL');
/**
* Runs a workflow's real select step against a canned codegraph response. `gh` and `curl` are
* stubs on PATH; the guard spec reader is the real script beside a fixture list. Every selected
* file exists in the scratch checkout unless `stale` says the selection outlived its files.
*/
function select(t, file, { selected, changed, list = FIXTURE, env = {}, stale = false }) {
const { selector } = workflows.get(file);
const scratch = mkdtempSync(join(scriptsDir, '.guard-specs-test-'));
t.after(() => rmSync(scratch, { recursive: true, force: true }));
const write = (name, content, options) => {
mkdirSync(dirname(join(scratch, name)), { recursive: true });
writeFileSync(join(scratch, name), content, options);
};
write('response.json', JSON.stringify({ selected }));
write('changed.ndjson', changed.map((entry) => JSON.stringify(entry)).join('\n') + '\n');
write('output', '');
write('selector.sh', selector.run);
write('gh', '#!/usr/bin/env bash\ncat changed.ndjson\nexit "${GH_EXIT:-0}"\n', { mode: 0o700 });
write('curl', '#!/usr/bin/env bash\ncat response.json\nexit "${CURL_EXIT:-0}"\n', {
mode: 0o700,
});
if (list !== null) {
write(MANIFEST_PATH, typeof list === 'string' ? list : JSON.stringify(list));
}
if (!stale) {
Object.values(selected)
.flatMap((decision) => decision.files ?? [])
.forEach((path) => write(path, ''));
}
mkdirSync(join(scratch, '.github/scripts'), { recursive: true });
copyFileSync(
join(scriptsDir, 'guard-specs.mjs'),
join(scratch, '.github/scripts/guard-specs.mjs'),
);
const result = spawnSync('bash', ['selector.sh'], {
cwd: scratch,
encoding: 'utf8',
timeout: 20000,
env: {
PATH: `${scratch}:${process.env.PATH}`,
URL: 'https://selector.invalid',
TOKEN: 'test-fixture',
REPO: 'fixture/repo',
PR: '1',
BASE_SHA: 'base',
HEAD_SHA: 'head',
CHANGED: String(changed.length),
GITHUB_OUTPUT: join(scratch, 'output'),
GITHUB_STEP_SUMMARY: join(scratch, 'summary'),
...env,
},
});
assert.equal(result.status, 0, result.stdout + result.stderr);
return Object.fromEntries(
readFileSync(join(scratch, 'output'), 'utf8')
.split('\n')
.filter(Boolean)
.map((line) => {
const separator = line.indexOf('=');
return [line.slice(0, separator), line.slice(separator + 1)];
}),
);
}
const modified = (...paths) => paths.map((path) => ({ path, status: 'modified' }));
test('backend: a narrowed workspace runs its guards with the selection (the #17864 shape)', (t) => {
const outputs = select(t, 'backend-review.yml', {
changed: modified('packages/api/src/agents/tools.ts'),
selected: {
api: mode('FILES', ['api/server/services/ToolService.spec.js']),
'packages/api': mode('FILES', [
'packages/api/src/agents/tools.spec.ts',
'packages/api/src/guard.spec.ts',
]),
'packages/data-provider': NONE,
'packages/data-schemas': NONE,
},
});
assert.deepEqual(outputs, {
api_run: 'true',
api_files: 'server/services/ToolService.spec.js server/guard.spec.js',
pkgapi_run: 'true',
pkgapi_files: 'src/agents/tools.spec.ts src/guard.spec.ts',
dataprovider_run: 'false',
dataschemas_run: 'false',
decided: 'true',
});
});
test('backend: a skipped workspace runs its guards alone when the change touches what they read', (t) => {
const outputs = select(t, 'backend-review.yml', {
changed: modified('api/server/routes/new.js'),
selected: {
api: NONE,
'packages/api': NONE,
'packages/data-provider': NONE,
'packages/data-schemas': NONE,
},
});
assert.deepEqual(outputs, {
api_run: 'true',
api_files: 'server/guard.spec.js',
pkgapi_run: 'false',
dataprovider_run: 'false',
dataschemas_run: 'true',
dataschemas_files: 'src/guard.spec.ts',
decided: 'true',
});
});
test('backend: a skip stands when nothing a guard reads changed, and FULL is left alone', (t) => {
const skipped = select(t, 'backend-review.yml', {
changed: modified('packages/client/src/components/Button.tsx'),
selected: {
api: NONE,
'packages/api': NONE,
'packages/data-provider': NONE,
'packages/data-schemas': NONE,
},
});
assert.deepEqual(skipped, {
api_run: 'false',
pkgapi_run: 'false',
dataprovider_run: 'false',
dataschemas_run: 'false',
decided: 'true',
});
const full = select(t, 'backend-review.yml', {
changed: modified('package-lock.json'),
selected: {
api: FULL,
'packages/api': FULL,
'packages/data-provider': FULL,
'packages/data-schemas': FULL,
},
});
assert.deepEqual(full, {
api_run: 'true',
pkgapi_run: 'true',
dataprovider_run: 'true',
dataschemas_run: 'true',
decided: 'true',
});
});
test('backend: guards obey the exclusions of the workspace jest run', (t) => {
const selected = {
api: NONE,
'packages/api': mode('FILES', ['packages/api/src/agents/run.integration.spec.ts']),
'packages/data-provider': NONE,
'packages/data-schemas': NONE,
};
const touched = select(t, 'backend-review.yml', {
changed: modified('packages/api/src/agents/run.ts'),
selected,
});
assert.equal(touched.pkgapi_run, 'true');
assert.equal(touched.pkgapi_files, 'src/guard.spec.ts');
const untouched = select(t, 'backend-review.yml', {
changed: modified('packages/api/package.json'),
selected,
});
assert.equal(untouched.pkgapi_run, 'false');
assert.equal(untouched.pkgapi_files, undefined);
});
test('a selection whose files are all gone gets no guards, so the test job still runs FULL', (t) => {
const backend = select(t, 'backend-review.yml', {
changed: modified('api/server/routes/old.js'),
selected: {
api: mode('FILES', ['api/server/routes/old.spec.js']),
'packages/api': FULL,
'packages/data-provider': FULL,
'packages/data-schemas': FULL,
},
stale: true,
});
assert.equal(backend.api_files, 'server/routes/old.spec.js');
const frontend = select(t, 'frontend-review.yml', {
changed: modified('client/src/old.ts'),
selected: { client: mode('FILES', ['client/src/old.spec.ts']), 'packages/client': FULL },
stale: true,
});
assert.equal(frontend.client_files, 'src/old.spec.ts');
});
test('backend: a guard list that cannot be read runs the narrowed workspaces FULL', (t) => {
const request = {
changed: modified('api/server/routes/new.js'),
selected: {
api: mode('FILES', ['api/server/routes/new.spec.js']),
'packages/api': NONE,
'packages/data-provider': FULL,
'packages/data-schemas': NONE,
},
};
const unsafe = { guards: { 'api/server/a b.spec.js': ['api/'] } };
for (const list of [null, '{', '{"guards": []}', unsafe]) {
const outputs = select(t, 'backend-review.yml', { ...request, list });
assert.deepEqual(outputs, {
api_run: 'true',
pkgapi_run: 'true',
dataprovider_run: 'true',
dataschemas_run: 'true',
decided: 'true',
});
}
});
test('backend: malformed decisions and an unavailable service still fail open', (t) => {
const changed = modified('api/server/routes/new.js');
const rest = {
'packages/api': FULL,
'packages/data-provider': FULL,
'packages/data-schemas': FULL,
};
for (const api of [
mode('FILES', []),
mode('FILES'),
mode('FILES', ['packages/api/src/misplaced.spec.ts']),
mode('FILES', ['api/server/has space.spec.js']),
mode('SOMETHING_NEW'),
]) {
const outputs = select(t, 'backend-review.yml', { changed, selected: { api, ...rest } });
assert.equal(outputs.api_run, 'true');
assert.equal(outputs.api_files, undefined);
}
const selected = { api: NONE, ...rest };
for (const env of [
{ URL: '' },
{ TOKEN: '' },
{ CURL_EXIT: '28' },
{ GH_EXIT: '1' },
{ CHANGED: '2' },
]) {
assert.deepEqual(select(t, 'backend-review.yml', { changed, selected, env }), {});
}
assert.deepEqual(select(t, 'backend-review.yml', { changed, selected: {} }), {});
});
test('frontend: guards join a narrowed run and promote a skip the change makes relevant', (t) => {
const narrowed = select(t, 'frontend-review.yml', {
changed: modified('client/src/components/Chat/Header.tsx'),
selected: {
client: mode('FILES', ['client/src/components/Chat/Header.spec.tsx']),
'packages/client': NONE,
},
});
assert.deepEqual(narrowed, {
client_run: 'true',
client_files: 'src/components/Chat/Header.spec.tsx src/guard.spec.ts',
clientpkg_run: 'false',
decided: 'true',
});
const promoted = select(t, 'frontend-review.yml', {
changed: modified('packages/chat/src/components/composer/Bar.tsx'),
selected: { client: NONE, 'packages/client': NONE },
});
assert.deepEqual(promoted, {
client_run: 'true',
client_files: 'src/guard.spec.ts',
clientpkg_run: 'false',
decided: 'true',
});
});
test('frontend: an unreadable guard list, malformed decisions and outages fail open', (t) => {
const changed = modified('client/src/a.ts');
const unreadable = select(t, 'frontend-review.yml', {
changed,
selected: { client: mode('FILES', ['client/src/a.spec.ts']), 'packages/client': NONE },
list: null,
});
assert.deepEqual(unreadable, { client_run: 'true', clientpkg_run: 'true', decided: 'true' });
const malformed = select(t, 'frontend-review.yml', {
changed,
selected: { client: mode('FILES', []), 'packages/client': FULL },
});
assert.deepEqual(malformed, { client_run: 'true', clientpkg_run: 'true', decided: 'true' });
const outage = select(t, 'frontend-review.yml', {
changed,
selected: { client: NONE, 'packages/client': NONE },
env: { CURL_EXIT: '28' },
});
assert.deepEqual(outage, {});
});