diff --git a/.github/MAIN_PROMOTION.md b/.github/MAIN_PROMOTION.md index c562757892..fdebb23308 100644 --- a/.github/MAIN_PROMOTION.md +++ b/.github/MAIN_PROMOTION.md @@ -34,9 +34,12 @@ for the exact input SHA, from the repository's existing backend and frontend wor IDs and paths. All build, typecheck, test-shard and circular-dependency jobs must be present and successful. Only the PR-only **Codegraph select** job may be skipped. An older successful run cannot conceal a newer failed or pending full run. PR checks -and checks from a different commit do not qualify. This gate does not imply that -Lighthouse, Playwright or integration lanes that only run on PRs tested the resulting -post-merge SHA; maintainers still assess those PR results and release readiness. +and checks from a different commit do not qualify. This gate does not read Lighthouse, +Playwright or integration lanes. Lighthouse and integration lanes run only on PRs. +Playwright also runs after merges to `dev`, where merges that land during a run share +one run of the newest commit, so the `dev` tip has a run of its own unless it changed +only Markdown or other workflows. Maintainers still assess those results and release +readiness; a manual **Playwright E2E Tests** run on `dev` covers both stores in full. The candidate must be the current `dev` tip, and the reviewed baseline must still be the current `main` tip. `main` must be an ancestor of the candidate. Ref and CI diff --git a/.github/guard-specs.json b/.github/guard-specs.json new file mode 100644 index 0000000000..7505565a95 --- /dev/null +++ b/.github/guard-specs.json @@ -0,0 +1,105 @@ +{ + "description": "Specs that read repository files through fs instead of importing them. Codegraph test selection follows imports, so it cannot tell when a change makes one of these relevant; backend-review.yml and frontend-review.yml add them to every narrowed run (.github/scripts/guard-specs.mjs). Keys are spec paths. A guard lists the repository paths it reads: a directory ends with a slash and covers everything under it. An exempt spec matches the detector without reading tracked files and says why.", + "guards": { + "api/server/controllers/__tests__/deleteUserResourceCoverage.spec.js": [ + "api/server/controllers/UserController.js", + "packages/api/src/user/erasure.ts" + ], + "api/server/experimental.spec.js": ["api/server/experimental.js", "api/server/index.js"], + "api/server/index.spec.js": ["api/server/index.js", "api/server/experimental.js"], + "api/server/routes/agents/__tests__/openapi.contract.spec.js": [ + "packages/api/openapi/agents.openapi.json" + ], + "api/server/services/__tests__/toolCapabilityGates.spec.js": [ + "api/app/", + "api/server/", + "packages/api/src/", + "packages/data-provider/src/" + ], + "api/server/utils/import/importers.spec.js": ["api/server/utils/import/__data__/"], + "client/src/Providers/__tests__/bootDirection.spec.ts": ["client/index.html"], + "client/src/Providers/__tests__/bootScript.spec.tsx": ["client/index.html"], + "client/src/Providers/__tests__/handoff.spec.tsx": ["client/index.html"], + "client/src/__tests__/hosts.test.tsx": ["client/src/App.jsx"], + "client/src/__tests__/loadingCanvas.spec.ts": ["client/index.html"], + "client/src/components/Chat/Input/Composer/__tests__/semanticTokens.spec.ts": [ + "client/src/components/Chat/", + "client/src/hooks/Input/usePaletteEntries.tsx", + "packages/chat/src/components/composer/" + ], + "client/src/components/Chat/Input/__tests__/appearanceTokens.spec.ts": [ + "client/src/components/Chat/Input/", + "packages/chat/src/components/composer/" + ], + "client/src/components/Chat/Input/__tests__/panelRadius.spec.ts": [ + "client/src/components/", + "packages/chat/src/components/", + "packages/client/src/components/" + ], + "client/src/lib/assets/__tests__/recovery.spec.tsx": ["client/index.html"], + "client/src/style.spec.ts": [ + "client/src/style.css", + "client/tailwind.config.cjs", + "packages/client/src/theme/", + "packages/client/tailwind.preset.cjs", + "package-lock.json" + ], + "client/src/utils/__tests__/mcpSandbox.spec.ts": ["client/public/mcp-sandbox.html"], + "packages/api/src/admin/meta.spec.ts": ["packages/data-provider/package.json"], + "packages/api/src/admin/secrets.runtime.spec.ts": [ + "packages/api/src/app/", + "packages/api/src/langfuse/utils.ts", + "packages/api/src/web/web.ts" + ], + "packages/api/src/agents/tools.spec.ts": ["e2e/setup/fake-model.js"], + "packages/api/src/channels/deployment.spec.ts": [".env.example"], + "packages/api/src/code/agent.spec.ts": ["api/", "packages/api/src/"], + "packages/api/src/code/setup/bridge.spec.ts": ["vendor/code/"], + "packages/api/src/files/documents/html.spec.ts": ["packages/api/src/files/documents/"], + "packages/api/src/files/documents/libreoffice.spec.ts": ["packages/api/src/files/documents/"], + "packages/api/src/files/documents/spreadsheet.spec.ts": ["packages/api/src/files/documents/"], + "packages/api/src/mcp/__tests__/sandbox.test.ts": ["client/public/mcp-sandbox.html"], + "packages/client/src/components/Progress.spec.tsx": ["packages/client/src/theme/tokens.css"], + "packages/client/src/components/SegmentedMeter.spec.tsx": [ + "packages/client/src/theme/tokens.css" + ], + "packages/client/src/components/__tests__/radixLayers.spec.ts": ["package-lock.json"], + "packages/client/src/theme/defaults.spec.ts": ["packages/client/src/theme/defaults.css"], + "packages/client/src/theme/primitives.spec.ts": [ + "eslint-suppressions.json", + "packages/client/src/components/" + ], + "packages/client/src/theme/registry.spec.ts": ["package-lock.json"], + "packages/client/src/theme/semanticTokens.spec.ts": [ + "client/src/style.css", + "packages/client/src/components/", + "packages/client/src/theme/defaults.css" + ], + "packages/client/src/theme/tailwind.spec.js": [ + "client/src/style.css", + "client/tailwind.config.cjs", + "packages/client/", + "package-lock.json" + ], + "packages/client/src/theme/themes/clickhouse.spec.tsx": ["packages/client/src/theme/fonts.css"], + "packages/client/src/theme/themes/clickui.spec.ts": [ + "client/src/style.css", + "packages/client/src/" + ], + "packages/client/src/theme/tokens.spec.ts": [ + "packages/client/src/theme/", + "packages/client/tailwind.preset.cjs", + "package-lock.json" + ], + "packages/data-provider/specs/config-schemas.spec.ts": ["librechat.example.yaml"], + "packages/data-schemas/src/methods/documentdb.spec.ts": [ + "api/", + "packages/api/src/", + "packages/data-schemas/src/" + ] + }, + "exempt": { + "packages/api/src/code/environment.live.spec.ts": "Skips unless LIBRECHAT_CODE_TEST_PACKAGE is set, and then reads only the temp directory it creates.", + "packages/api/src/plugins/runtime.spec.ts": "Reads only logs it writes under os.tmpdir(); process.cwd() appears inside an inline hook command." + } +} diff --git a/.github/scripts/expressions.mjs b/.github/scripts/expressions.mjs new file mode 100644 index 0000000000..66f3ef3c77 --- /dev/null +++ b/.github/scripts/expressions.mjs @@ -0,0 +1,169 @@ +/** + * Evaluates the subset of GitHub Actions expressions the workflows here use, so a test can + * state what a job condition, matrix or concurrency group resolves to for a given event + * instead of matching the expression's text. Covers literals, context lookups, `!`, `==`, + * `!=`, `&&`, `||`, parentheses and the functions passed in `functions`. + */ + +const TOKEN = + /\s*(?:('(?:[^']|'')*')|(-?\d+(?:\.\d+)?)|(==|!=|&&|\|\||[!(),])|([A-Za-z_][\w-]*(?:\.[\w-]+)*))/y; + +function tokenize(source) { + const tokens = []; + TOKEN.lastIndex = 0; + while (TOKEN.lastIndex < source.length) { + const start = TOKEN.lastIndex; + const match = TOKEN.exec(source); + if (!match) { + if (source.slice(start).trim() === '') break; + throw new Error(`Unsupported expression syntax at: ${source.slice(start, start + 40)}`); + } + const [, string, number, operator, name] = match; + if (string !== undefined) { + tokens.push({ type: 'value', value: string.slice(1, -1).replaceAll("''", "'") }); + } else if (number !== undefined) { + tokens.push({ type: 'value', value: Number(number) }); + } else if (operator !== undefined) { + tokens.push({ type: operator }); + } else { + tokens.push({ type: 'name', name }); + } + } + return tokens; +} + +const truthy = (value) => + !(value === false || value === 0 || value === '' || value == null || Number.isNaN(value)); + +function toNumber(value) { + if (value == null || value === '') return 0; + if (typeof value === 'boolean') return value ? 1 : 0; + if (typeof value === 'number') return value; + return typeof value === 'string' ? Number(value) : NaN; +} + +/** Same-type values compare directly (strings ignoring case); mixed types compare as numbers. */ +function equal(left, right) { + if (typeof left === 'string' && typeof right === 'string') { + return left.toLowerCase() === right.toLowerCase(); + } + if (left === right) return true; + const sameType = typeof left === typeof right && left != null && right != null; + return sameType ? false : toNumber(left) === toNumber(right); +} + +const BUILTINS = { + fromJSON: (text) => JSON.parse(text), + contains: (haystack, needle) => + Array.isArray(haystack) + ? haystack.some((item) => equal(item, needle)) + : String(haystack ?? '') + .toLowerCase() + .includes(String(needle ?? '').toLowerCase()), +}; + +const LITERALS = new Map([ + ['true', true], + ['false', false], + ['null', null], +]); + +/** A missing property is null, as in Actions. */ +const lookup = (context, path) => + path.split('.').reduce((value, key) => (value == null ? null : (value[key] ?? null)), context); + +export function evaluate(source, context = {}, functions = {}) { + const tokens = tokenize(source); + const calls = { ...BUILTINS, ...functions }; + let position = 0; + const peek = () => tokens[position]?.type; + const take = (type) => { + const token = tokens[position]; + if (!token || (type && token.type !== type)) { + throw new Error(`Expected ${type ?? 'a token'} in: ${source}`); + } + position += 1; + return token; + }; + + /** `evaluating` is false on the short-circuited side, which is parsed but never called. */ + function primary(evaluating) { + if (peek() === '(') { + take('('); + const value = or(evaluating); + take(')'); + return value; + } + if (peek() === 'value') return take('value').value; + const { name } = take('name'); + if (peek() !== '(') return LITERALS.has(name) ? LITERALS.get(name) : lookup(context, name); + take('('); + const args = []; + while (peek() !== ')') { + args.push(or(evaluating)); + if (peek() === ',') take(','); + } + take(')'); + if (!evaluating) return null; + if (!calls[name]) throw new Error(`Unsupported function ${name}() in: ${source}`); + return calls[name](...args); + } + + function unary(evaluating) { + if (peek() !== '!') return primary(evaluating); + take('!'); + return !truthy(unary(evaluating)); + } + + function comparison(evaluating) { + let left = unary(evaluating); + while (peek() === '==' || peek() === '!=') { + const { type } = take(); + const right = unary(evaluating); + left = type === '==' ? equal(left, right) : !equal(left, right); + } + return left; + } + + function and(evaluating) { + let left = comparison(evaluating); + while (peek() === '&&') { + take('&&'); + const right = comparison(evaluating && truthy(left)); + left = truthy(left) ? right : left; + } + return left; + } + + function or(evaluating) { + let left = and(evaluating); + while (peek() === '||') { + take('||'); + const right = and(evaluating && !truthy(left)); + left = truthy(left) ? left : right; + } + return left; + } + + const value = or(true); + if (position !== tokens.length) throw new Error(`Trailing tokens in: ${source}`); + return value; +} + +/** A job-level `if` is an expression whether or not it is wrapped in `${{ }}`. */ +export function condition(source, context, functions) { + const text = String(source).trim(); + const wrapped = /^\$\{\{([\s\S]*)\}\}$/.exec(text); + return truthy(evaluate(wrapped ? wrapped[1] : text, context, functions)); +} + +/** Resolves every `${{ }}` in a string; a value that is only an expression keeps its type. */ +export function interpolate(template, context, functions) { + const text = String(template).trim(); + const whole = /^\$\{\{((?:(?!\}\})[\s\S])*)\}\}$/.exec(text); + if (whole) return evaluate(whole[1], context, functions); + return text.replace(/\$\{\{((?:(?!\}\})[\s\S])*)\}\}/g, (_, expression) => { + const value = evaluate(expression, context, functions); + return value == null ? '' : String(value); + }); +} diff --git a/.github/scripts/guard-specs.mjs b/.github/scripts/guard-specs.mjs new file mode 100644 index 0000000000..384f69b64b --- /dev/null +++ b/.github/scripts/guard-specs.mjs @@ -0,0 +1,158 @@ +/** + * Guard specs under codegraph test selection. + * + * Selection (the `codegraph_select` jobs in backend-review.yml and frontend-review.yml) narrows a + * pull request run to the test files an import graph reaches from the change. A spec that reads + * source with `fs` has no import edge to what it reads: toolCapabilityGates.spec.js scans four + * source trees, was not selected when #17864 changed one of the files it counts, and then failed + * on dev. `.github/guard-specs.json` lists every such spec with the paths it reads, and this + * script adds them to a workspace's decision: + * + * FILES every guard of the workspace joins the selected files, unless none of the selected + * files exists in this checkout: the test job runs FULL on such a stale selection, and + * guards that do exist would hide it; + * NONE the workspace runs its guards alone when the change touches a path one of them reads, + * renames a file, or edits the list or this script; otherwise it stays skipped. + * + * Usage: SELECTED="" node guard-specs.mjs [ignore] + * Run from the root of a checkout of the commit under test. Prints the workspace-relative files + * to run, space separated; nothing means skip. Any failure exits non-zero and the caller runs + * the workspace FULL. + */ +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { readFileSync, statSync } from 'node:fs'; + +export const MANIFEST_PATH = '.github/guard-specs.json'; +const SCRIPT_PATH = '.github/scripts/guard-specs.mjs'; +const MANIFEST_URL = new URL('../guard-specs.json', import.meta.url); + +/** A spec file, by the suffixes jest matches in the gated workspaces. */ +export const SPEC_FILE = /\.(?:spec|test)\.[cm]?[jt]sx?$/; + +/** + * What marks a spec as reading the repository: a filesystem read or walk, together with a path + * anchored at the spec's own location or the working directory. A spec that only uses temp + * directories has no anchor; one that only imports has no read. + */ +const FS_READ = + /\b(?:read(?:File|dir)(?:Sync)?|opendir(?:Sync)?|createReadStream|glob(?:Sync)?|globby)\s*\(/; +const REPO_ANCHOR = + /__dirname|__filename|process\.cwd\(\)|import\.meta\.(?:url|dirname|filename)|require\.resolve/; + +export const readsRepository = (source) => FS_READ.test(source) && REPO_ANCHOR.test(source); + +/** + * Listed paths end up unquoted in the test jobs' shell (`--runTestsByPath $KEEP`), so anything + * that could split, glob or climb out of the repository is refused here. + */ +const SAFE_PATH = /^[A-Za-z0-9_.-]+(?:\/[A-Za-z0-9_.-]+)*\/?$/; +const isSafePath = (path) => + typeof path === 'string' && SAFE_PATH.test(path) && !path.split('/').includes('..'); + +const isRecord = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); + +export function parseManifest(text) { + const { guards, exempt = {} } = JSON.parse(text); + if (!isRecord(guards) || !isRecord(exempt)) { + throw new Error(`${MANIFEST_PATH}: "guards" and "exempt" must be objects`); + } + for (const [spec, scans] of Object.entries(guards)) { + if (!isSafePath(spec) || !SPEC_FILE.test(spec)) { + throw new Error(`${MANIFEST_PATH}: guard is not a plain spec path: ${spec}`); + } + if (!Array.isArray(scans) || scans.length === 0 || !scans.every(isSafePath)) { + throw new Error(`${MANIFEST_PATH}: ${spec} must list the repository paths it reads`); + } + if (spec in exempt) { + throw new Error(`${MANIFEST_PATH}: ${spec} is both a guard and exempt`); + } + } + for (const [spec, reason] of Object.entries(exempt)) { + if (!isSafePath(spec) || typeof reason !== 'string' || reason.trim() === '') { + throw new Error(`${MANIFEST_PATH}: exempt spec needs a plain path and a reason: ${spec}`); + } + } + return { guards, exempt }; +} + +export const loadManifest = () => parseManifest(readFileSync(MANIFEST_URL, 'utf8')); + +const covers = (scan, path) => (scan.endsWith('/') ? path.startsWith(scan) : path === scan); + +/** + * A rename reports only its new path, so one can move a file out of a scanned tree unseen; and + * a change to the list or to this script can change which guards exist. Both count for all. + */ +const touchesEveryGuard = ({ path, status }) => + status === 'renamed' || path === MANIFEST_PATH || path === SCRIPT_PATH; + +/** + * @param {object} options + * @param {{ guards: Record }} options.manifest + * @param {'FILES' | 'NONE'} options.mode The workspace's decision after the caller's own checks. + * @param {string} options.workspace Repository-relative workspace directory, e.g. `packages/api`. + * @param {{ path: string, status?: string }[]} options.changed The pull request's changed files. + * @param {string[]} [options.selected] Workspace-relative files already selected (FILES only). + * @param {(file: string) => boolean} options.exists Whether a workspace-relative file is in the + * checkout the tests will run against. + * @param {string} [options.ignore] Regex of paths the workspace's own jest run excludes. + * @returns {string[]} Workspace-relative files to run; empty means the workspace stays skipped. + */ +export function withGuards({ + manifest, + mode, + workspace, + changed, + exists, + selected = [], + ignore = '', +}) { + if (mode !== 'FILES' && mode !== 'NONE') throw new Error(`Unsupported mode: ${mode}`); + if (typeof exists !== 'function') throw new Error('An exists(file) check is required'); + if (!isSafePath(workspace) || workspace.endsWith('/')) { + throw new Error(`Unsupported workspace: ${workspace}`); + } + if (!Array.isArray(changed) || !changed.every((file) => typeof file?.path === 'string')) { + throw new Error('The changed-file list must be an array of { path }'); + } + const prefix = `${workspace}/`; + const excluded = ignore === '' ? null : new RegExp(ignore); + const guards = Object.entries(manifest.guards).filter( + ([spec]) => spec.startsWith(prefix) && !excluded?.test(spec), + ); + const files = guards.map(([spec]) => spec.slice(prefix.length)); + if (mode === 'FILES') { + return selected.some(exists) ? [...new Set([...selected, ...files])] : selected; + } + const relevant = changed.some( + (file) => + touchesEveryGuard(file) || + guards.some( + ([spec, scans]) => file.path === spec || scans.some((scan) => covers(scan, file.path)), + ), + ); + return relevant ? files : []; +} + +function main([mode, workspace, changedFile, ignore = '']) { + const files = withGuards({ + manifest: loadManifest(), + mode, + workspace, + changed: JSON.parse(readFileSync(changedFile, 'utf8')), + exists: (file) => statSync(join(workspace, file), { throwIfNoEntry: false })?.isFile() === true, + selected: (process.env.SELECTED ?? '').split(/\s+/).filter(Boolean), + ignore, + }); + process.stdout.write(files.join(' ')); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + try { + main(process.argv.slice(2)); + } catch (error) { + console.error(`guard-specs: ${error.message}`); + process.exit(1); + } +} diff --git a/.github/scripts/guard-specs.test.mjs b/.github/scripts/guard-specs.test.mjs new file mode 100644 index 0000000000..b330bbb0a1 --- /dev/null +++ b/.github/scripts/guard-specs.test.mjs @@ -0,0 +1,540 @@ +import yaml from 'js-yaml'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { condition } from './expressions.mjs'; +import { + loadManifest, + MANIFEST_PATH, + parseManifest, + readsRepository, + SPEC_FILE, + withGuards, +} from './guard-specs.mjs'; + +const scriptsDir = fileURLToPath(new URL('.', import.meta.url)); +const repoRoot = fileURLToPath(new URL('../..', import.meta.url)); +const manifest = loadManifest(); + +const WORKFLOWS = ['backend-review.yml', 'frontend-review.yml']; +const workflows = new Map( + WORKFLOWS.map((file) => { + const workflow = yaml.load(readFileSync(join(repoRoot, '.github/workflows', file), 'utf8')); + const selector = workflow.jobs.codegraph_select.steps.find((step) => step.id === 'sel'); + return [file, { workflow, selector }]; + }), +); + +/** `emit [ignore]` is how a selector names each workspace it gates. */ +const gated = new Map( + WORKFLOWS.flatMap((file) => + Array.from( + workflows.get(file).selector.run.matchAll(/^\s*emit \w+ (\S+)(?: (?:'[^']*'|""))?$/gm), + ([, workspace]) => [workspace, { file }], + ), + ), +); + +const SKIPPED_DIRECTORIES = new Set(['node_modules', 'dist', 'build', 'coverage', '.turbo']); + +function specsUnder(directory) { + return readdirSync(join(repoRoot, directory), { withFileTypes: true }).flatMap((entry) => { + const path = `${directory}/${entry.name}`; + if (entry.isDirectory()) return SKIPPED_DIRECTORIES.has(entry.name) ? [] : specsUnder(path); + return SPEC_FILE.test(entry.name) ? [path] : []; + }); +} + +const workspaceOf = (spec) => + [...gated.keys()].find((workspace) => spec.startsWith(`${workspace}/`)); + +/** GitHub's path filter, for the shapes these triggers use: exact paths and `dir/**`. */ +const triggers = (pattern, path) => + pattern.endsWith('/**') ? path.startsWith(pattern.slice(0, -2)) : pattern === path; + +test('both selectors gate the workspaces this test walks', () => { + assert.deepEqual( + [...gated].map(([workspace, { file }]) => [workspace, file]), + [ + ['api', 'backend-review.yml'], + ['packages/api', 'backend-review.yml'], + ['packages/data-provider', 'backend-review.yml'], + ['packages/data-schemas', 'backend-review.yml'], + ['client', 'frontend-review.yml'], + ['packages/client', 'frontend-review.yml'], + ], + ); +}); + +test('only a pull request synchronize is narrowed; every other event runs each workspace FULL', () => { + const event = (event_name, action) => ({ github: { event_name, event: { action } }, vars: {} }); + const unselected = { build: { result: 'success' }, codegraph_select: { outputs: {} } }; + for (const [file, { workflow }] of workflows) { + const selects = (context) => condition(workflow.jobs.codegraph_select.if, context); + assert.equal(selects(event('pull_request', 'synchronize')), true, file); + for (const context of [ + event('pull_request', 'opened'), + event('pull_request', 'reopened'), + event('push'), + event('workflow_dispatch'), + event('merge_group', 'checks_requested'), + ]) { + assert.equal(selects(context), false, `${file} ${context.github.event_name}`); + } + const gatedJobs = Object.values(workflow.jobs).filter((job) => + [job.needs].flat().includes('codegraph_select'), + ); + assert.ok(gatedJobs.length > 0, file); + for (const job of gatedJobs) { + assert.equal( + condition(job.if, { needs: unselected }, { cancelled: () => false }), + true, + `${file} ${job.name}`, + ); + } + } +}); + +test('every spec that reads the repository through fs is listed as a guard or exempt', () => { + const listed = new Set([...Object.keys(manifest.guards), ...Object.keys(manifest.exempt)]); + const unlisted = [...gated.keys()] + .flatMap(specsUnder) + .filter((spec) => !listed.has(spec)) + .filter((spec) => readsRepository(readFileSync(join(repoRoot, spec), 'utf8'))); + assert.deepEqual( + unlisted, + [], + `These specs read files with fs from a path anchored in the repository, so codegraph test ` + + `selection cannot tell when a change affects them. Add each to "guards" in ` + + `${MANIFEST_PATH} with the repository paths it reads, or to "exempt" with the reason it ` + + `reads no tracked file.`, + ); +}); + +test('every listed spec exists, sits in a gated workspace and reads paths that exist', () => { + for (const spec of [...Object.keys(manifest.guards), ...Object.keys(manifest.exempt)]) { + assert.ok(existsSync(join(repoRoot, spec)), `${spec} is listed but does not exist`); + assert.ok(workspaceOf(spec), `${spec} is not in a workspace that test selection gates`); + } + for (const [spec, scans] of Object.entries(manifest.guards)) { + for (const scan of scans) { + const target = join(repoRoot, scan); + assert.ok(existsSync(target), `${spec} lists ${scan}, which does not exist`); + assert.equal( + statSync(target).isDirectory(), + scan.endsWith('/'), + `${spec} lists ${scan}: a directory ends with a slash and a file does not`, + ); + } + } +}); + +test('a workflow runs whenever a path one of its guard specs reads changes', () => { + for (const [spec, scans] of Object.entries(manifest.guards)) { + const { file } = gated.get(workspaceOf(spec)); + const { workflow } = workflows.get(file); + for (const event of ['pull_request', 'push']) { + const patterns = workflow.on[event].paths.filter((pattern) => !pattern.startsWith('!')); + for (const scan of scans) { + assert.ok( + patterns.some((pattern) => triggers(pattern, scan)), + `${spec} reads ${scan}, but ${file} does not run on ${event} when it changes; ` + + `add it to that trigger's paths`, + ); + } + } + } +}); + +test('the detector flags repository reads and ignores temp-only and import-only specs', () => { + assert.ok(readsRepository("fs.readFileSync(path.join(__dirname, '../index.js'), 'utf8')")); + assert.ok(readsRepository('for (const e of readdirSync(join(process.cwd(), "src"))) {}')); + assert.ok(readsRepository('await readFile(new URL("./a.css", import.meta.url))')); + assert.ok(readsRepository("globSync('src/**/*.ts', { cwd: __dirname })")); + assert.ok(!readsRepository("fs.readFileSync(path.join(os.tmpdir(), 'out.txt'))")); + assert.ok(!readsRepository("const { run } = require(path.join(__dirname, 'run.js'))")); + assert.ok(!readsRepository("jest.mock('fs'); expect(reader.readFile).toBeDefined()")); +}); + +test('a manifest is refused when a path could split, glob or leave the repository', () => { + const guards = (entries) => JSON.stringify({ guards: entries }); + assert.throws(() => parseManifest('{}')); + assert.throws(() => parseManifest(guards({ 'api/a b.spec.js': ['api/'] }))); + assert.throws(() => parseManifest(guards({ 'api/*.spec.js': ['api/'] }))); + assert.throws(() => parseManifest(guards({ 'api/$(id).spec.js': ['api/'] }))); + assert.throws(() => parseManifest(guards({ 'api/../a.spec.js': ['api/'] }))); + assert.throws(() => parseManifest(guards({ '/api/a.spec.js': ['api/'] }))); + assert.throws(() => parseManifest(guards({ 'api/a.js': ['api/'] }))); + assert.throws(() => parseManifest(guards({ 'api/a.spec.js': [] }))); + assert.throws(() => parseManifest(guards({ 'api/a.spec.js': ['api/ server/'] }))); + assert.throws(() => parseManifest(guards({ 'api/a.spec.js': 'api/' }))); + assert.throws(() => + parseManifest( + JSON.stringify({ guards: { 'api/a.spec.js': ['api/'] }, exempt: { 'api/a.spec.js': 'x' } }), + ), + ); + assert.throws(() => + parseManifest(JSON.stringify({ guards: {}, exempt: { 'api/a.spec.js': '' } })), + ); + assert.deepEqual(parseManifest(guards({ 'api/a.spec.js': ['api/', 'b.json'] })).guards, { + 'api/a.spec.js': ['api/', 'b.json'], + }); +}); + +const FIXTURE = { + guards: { + 'api/server/guard.spec.js': ['api/server/', 'packages/api/src/'], + 'packages/api/src/guard.spec.ts': ['packages/api/src/'], + 'packages/api/src/scan.integration.spec.ts': ['packages/api/src/'], + 'packages/data-schemas/src/guard.spec.ts': ['api/'], + 'client/src/guard.spec.ts': ['client/index.html', 'packages/chat/src/'], + 'packages/client/src/guard.spec.tsx': ['packages/client/src/theme/'], + }, +}; + +test('a rename, or a change to the list or its reader, counts for every guard', () => { + const run = (changed) => + withGuards({ + manifest: FIXTURE, + mode: 'NONE', + workspace: 'packages/client', + changed, + exists: () => true, + }); + assert.deepEqual(run([{ path: 'docs/a.txt', status: 'modified' }]), []); + assert.deepEqual(run([{ path: 'docs/b.txt', status: 'renamed' }]), ['src/guard.spec.tsx']); + assert.deepEqual(run([{ path: MANIFEST_PATH, status: 'modified' }]), ['src/guard.spec.tsx']); + assert.deepEqual(run([{ path: '.github/scripts/guard-specs.mjs' }]), ['src/guard.spec.tsx']); + assert.deepEqual(run([{ path: 'packages/client/src/guard.spec.tsx' }]), ['src/guard.spec.tsx']); + assert.deepEqual(run([{ path: 'packages/client/src/theme/a.css' }]), ['src/guard.spec.tsx']); + assert.deepEqual(run([{ path: 'packages/client/src/themes.ts' }]), []); + assert.throws(() => run([{ filename: 'a' }])); + assert.throws(() => run({})); +}); + +test('a selection keeps its guards only while one of its files is in the checkout', () => { + const selected = ['src/gone.spec.tsx', 'src/kept.spec.tsx']; + const run = (exists) => + withGuards({ + manifest: FIXTURE, + mode: 'FILES', + workspace: 'packages/client', + changed: [], + selected, + exists, + }); + assert.deepEqual( + run((file) => file === 'src/kept.spec.tsx'), + [...selected, 'src/guard.spec.tsx'], + ); + assert.deepEqual( + run(() => false), + selected, + ); + assert.throws(() => run(undefined)); +}); + +const mode = (value, files) => ({ mode: value, ...(files ? { files } : {}) }); +const NONE = mode('NONE'); +const FULL = mode('FULL'); + +/** + * Runs a workflow's real select step against a canned codegraph response. `gh` and `curl` are + * stubs on PATH; the guard spec reader is the real script beside a fixture list. Every selected + * file exists in the scratch checkout unless `stale` says the selection outlived its files. + */ +function select(t, file, { selected, changed, list = FIXTURE, env = {}, stale = false }) { + const { selector } = workflows.get(file); + const scratch = mkdtempSync(join(scriptsDir, '.guard-specs-test-')); + t.after(() => rmSync(scratch, { recursive: true, force: true })); + const write = (name, content, options) => { + mkdirSync(dirname(join(scratch, name)), { recursive: true }); + writeFileSync(join(scratch, name), content, options); + }; + write('response.json', JSON.stringify({ selected })); + write('changed.ndjson', changed.map((entry) => JSON.stringify(entry)).join('\n') + '\n'); + write('output', ''); + write('selector.sh', selector.run); + write('gh', '#!/usr/bin/env bash\ncat changed.ndjson\nexit "${GH_EXIT:-0}"\n', { mode: 0o700 }); + write('curl', '#!/usr/bin/env bash\ncat response.json\nexit "${CURL_EXIT:-0}"\n', { + mode: 0o700, + }); + if (list !== null) { + write(MANIFEST_PATH, typeof list === 'string' ? list : JSON.stringify(list)); + } + if (!stale) { + Object.values(selected) + .flatMap((decision) => decision.files ?? []) + .forEach((path) => write(path, '')); + } + mkdirSync(join(scratch, '.github/scripts'), { recursive: true }); + copyFileSync( + join(scriptsDir, 'guard-specs.mjs'), + join(scratch, '.github/scripts/guard-specs.mjs'), + ); + const result = spawnSync('bash', ['selector.sh'], { + cwd: scratch, + encoding: 'utf8', + timeout: 20000, + env: { + PATH: `${scratch}:${process.env.PATH}`, + URL: 'https://selector.invalid', + TOKEN: 'test-fixture', + REPO: 'fixture/repo', + PR: '1', + BASE_SHA: 'base', + HEAD_SHA: 'head', + CHANGED: String(changed.length), + GITHUB_OUTPUT: join(scratch, 'output'), + GITHUB_STEP_SUMMARY: join(scratch, 'summary'), + ...env, + }, + }); + assert.equal(result.status, 0, result.stdout + result.stderr); + return Object.fromEntries( + readFileSync(join(scratch, 'output'), 'utf8') + .split('\n') + .filter(Boolean) + .map((line) => { + const separator = line.indexOf('='); + return [line.slice(0, separator), line.slice(separator + 1)]; + }), + ); +} + +const modified = (...paths) => paths.map((path) => ({ path, status: 'modified' })); + +test('backend: a narrowed workspace runs its guards with the selection (the #17864 shape)', (t) => { + const outputs = select(t, 'backend-review.yml', { + changed: modified('packages/api/src/agents/tools.ts'), + selected: { + api: mode('FILES', ['api/server/services/ToolService.spec.js']), + 'packages/api': mode('FILES', [ + 'packages/api/src/agents/tools.spec.ts', + 'packages/api/src/guard.spec.ts', + ]), + 'packages/data-provider': NONE, + 'packages/data-schemas': NONE, + }, + }); + assert.deepEqual(outputs, { + api_run: 'true', + api_files: 'server/services/ToolService.spec.js server/guard.spec.js', + pkgapi_run: 'true', + pkgapi_files: 'src/agents/tools.spec.ts src/guard.spec.ts', + dataprovider_run: 'false', + dataschemas_run: 'false', + decided: 'true', + }); +}); + +test('backend: a skipped workspace runs its guards alone when the change touches what they read', (t) => { + const outputs = select(t, 'backend-review.yml', { + changed: modified('api/server/routes/new.js'), + selected: { + api: NONE, + 'packages/api': NONE, + 'packages/data-provider': NONE, + 'packages/data-schemas': NONE, + }, + }); + assert.deepEqual(outputs, { + api_run: 'true', + api_files: 'server/guard.spec.js', + pkgapi_run: 'false', + dataprovider_run: 'false', + dataschemas_run: 'true', + dataschemas_files: 'src/guard.spec.ts', + decided: 'true', + }); +}); + +test('backend: a skip stands when nothing a guard reads changed, and FULL is left alone', (t) => { + const skipped = select(t, 'backend-review.yml', { + changed: modified('packages/client/src/components/Button.tsx'), + selected: { + api: NONE, + 'packages/api': NONE, + 'packages/data-provider': NONE, + 'packages/data-schemas': NONE, + }, + }); + assert.deepEqual(skipped, { + api_run: 'false', + pkgapi_run: 'false', + dataprovider_run: 'false', + dataschemas_run: 'false', + decided: 'true', + }); + const full = select(t, 'backend-review.yml', { + changed: modified('package-lock.json'), + selected: { + api: FULL, + 'packages/api': FULL, + 'packages/data-provider': FULL, + 'packages/data-schemas': FULL, + }, + }); + assert.deepEqual(full, { + api_run: 'true', + pkgapi_run: 'true', + dataprovider_run: 'true', + dataschemas_run: 'true', + decided: 'true', + }); +}); + +test('backend: guards obey the exclusions of the workspace jest run', (t) => { + const selected = { + api: NONE, + 'packages/api': mode('FILES', ['packages/api/src/agents/run.integration.spec.ts']), + 'packages/data-provider': NONE, + 'packages/data-schemas': NONE, + }; + const touched = select(t, 'backend-review.yml', { + changed: modified('packages/api/src/agents/run.ts'), + selected, + }); + assert.equal(touched.pkgapi_run, 'true'); + assert.equal(touched.pkgapi_files, 'src/guard.spec.ts'); + const untouched = select(t, 'backend-review.yml', { + changed: modified('packages/api/package.json'), + selected, + }); + assert.equal(untouched.pkgapi_run, 'false'); + assert.equal(untouched.pkgapi_files, undefined); +}); + +test('a selection whose files are all gone gets no guards, so the test job still runs FULL', (t) => { + const backend = select(t, 'backend-review.yml', { + changed: modified('api/server/routes/old.js'), + selected: { + api: mode('FILES', ['api/server/routes/old.spec.js']), + 'packages/api': FULL, + 'packages/data-provider': FULL, + 'packages/data-schemas': FULL, + }, + stale: true, + }); + assert.equal(backend.api_files, 'server/routes/old.spec.js'); + const frontend = select(t, 'frontend-review.yml', { + changed: modified('client/src/old.ts'), + selected: { client: mode('FILES', ['client/src/old.spec.ts']), 'packages/client': FULL }, + stale: true, + }); + assert.equal(frontend.client_files, 'src/old.spec.ts'); +}); + +test('backend: a guard list that cannot be read runs the narrowed workspaces FULL', (t) => { + const request = { + changed: modified('api/server/routes/new.js'), + selected: { + api: mode('FILES', ['api/server/routes/new.spec.js']), + 'packages/api': NONE, + 'packages/data-provider': FULL, + 'packages/data-schemas': NONE, + }, + }; + const unsafe = { guards: { 'api/server/a b.spec.js': ['api/'] } }; + for (const list of [null, '{', '{"guards": []}', unsafe]) { + const outputs = select(t, 'backend-review.yml', { ...request, list }); + assert.deepEqual(outputs, { + api_run: 'true', + pkgapi_run: 'true', + dataprovider_run: 'true', + dataschemas_run: 'true', + decided: 'true', + }); + } +}); + +test('backend: malformed decisions and an unavailable service still fail open', (t) => { + const changed = modified('api/server/routes/new.js'); + const rest = { + 'packages/api': FULL, + 'packages/data-provider': FULL, + 'packages/data-schemas': FULL, + }; + for (const api of [ + mode('FILES', []), + mode('FILES'), + mode('FILES', ['packages/api/src/misplaced.spec.ts']), + mode('FILES', ['api/server/has space.spec.js']), + mode('SOMETHING_NEW'), + ]) { + const outputs = select(t, 'backend-review.yml', { changed, selected: { api, ...rest } }); + assert.equal(outputs.api_run, 'true'); + assert.equal(outputs.api_files, undefined); + } + const selected = { api: NONE, ...rest }; + for (const env of [ + { URL: '' }, + { TOKEN: '' }, + { CURL_EXIT: '28' }, + { GH_EXIT: '1' }, + { CHANGED: '2' }, + ]) { + assert.deepEqual(select(t, 'backend-review.yml', { changed, selected, env }), {}); + } + assert.deepEqual(select(t, 'backend-review.yml', { changed, selected: {} }), {}); +}); + +test('frontend: guards join a narrowed run and promote a skip the change makes relevant', (t) => { + const narrowed = select(t, 'frontend-review.yml', { + changed: modified('client/src/components/Chat/Header.tsx'), + selected: { + client: mode('FILES', ['client/src/components/Chat/Header.spec.tsx']), + 'packages/client': NONE, + }, + }); + assert.deepEqual(narrowed, { + client_run: 'true', + client_files: 'src/components/Chat/Header.spec.tsx src/guard.spec.ts', + clientpkg_run: 'false', + decided: 'true', + }); + const promoted = select(t, 'frontend-review.yml', { + changed: modified('packages/chat/src/components/composer/Bar.tsx'), + selected: { client: NONE, 'packages/client': NONE }, + }); + assert.deepEqual(promoted, { + client_run: 'true', + client_files: 'src/guard.spec.ts', + clientpkg_run: 'false', + decided: 'true', + }); +}); + +test('frontend: an unreadable guard list, malformed decisions and outages fail open', (t) => { + const changed = modified('client/src/a.ts'); + const unreadable = select(t, 'frontend-review.yml', { + changed, + selected: { client: mode('FILES', ['client/src/a.spec.ts']), 'packages/client': NONE }, + list: null, + }); + assert.deepEqual(unreadable, { client_run: 'true', clientpkg_run: 'true', decided: 'true' }); + const malformed = select(t, 'frontend-review.yml', { + changed, + selected: { client: mode('FILES', []), 'packages/client': FULL }, + }); + assert.deepEqual(malformed, { client_run: 'true', clientpkg_run: 'true', decided: 'true' }); + const outage = select(t, 'frontend-review.yml', { + changed, + selected: { client: NONE, 'packages/client': NONE }, + env: { CURL_EXIT: '28' }, + }); + assert.deepEqual(outage, {}); +}); diff --git a/.github/scripts/playwright-matrix.test.mjs b/.github/scripts/playwright-matrix.test.mjs index ab7bc27e1d..69f2953bb8 100644 --- a/.github/scripts/playwright-matrix.test.mjs +++ b/.github/scripts/playwright-matrix.test.mjs @@ -5,6 +5,7 @@ import assert from 'node:assert/strict'; import { fileURLToPath } from 'node:url'; import { spawnSync } from 'node:child_process'; import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { condition, interpolate } from './expressions.mjs'; const scriptsDir = fileURLToPath(new URL('.', import.meta.url)); const workflow = yaml.load( @@ -85,6 +86,55 @@ function decision(redis, mcp = true) { return { matrix: { 'playwright-mock': { redis_transport: redis, mcp_tool_list_changed: mcp } } }; } +const REPOSITORY = 'LibreChat-AI/LibreChat'; +const event = (event_name, ref, payload = {}) => ({ + event_name, + ref, + repository: REPOSITORY, + event: payload, +}); +const pushToDev = event('push', 'refs/heads/dev'); +const nightly = event('schedule', 'refs/heads/main'); +const manual = event('workflow_dispatch', 'refs/heads/dev'); +const pullRequest = (action, association = 'MEMBER', head = REPOSITORY) => + event('pull_request', 'refs/pull/1/merge', { + action, + pull_request: { author_association: association, head: { repo: { full_name: head } } }, + }); + +/** What the workflow's own expressions resolve to for one event, before any job has run. */ +function resolve(github, { inputs = {}, select = {} } = {}) { + const context = { github, inputs, vars: {}, needs: { codegraph_select: { outputs: select } } }; + const functions = { cancelled: () => false }; + const runs = (job) => condition(workflow.jobs[job].if, context, functions); + return { + jobs: { + select: runs('codegraph_select'), + shards: runs('e2e_shards'), + mcp: runs('mcp_tool_list_changed'), + apps: runs('mcp_apps'), + gate: runs('e2e'), + }, + matrix: interpolate(shards.strategy.matrix, context, functions), + group: interpolate(workflow.concurrency.group, context, functions), + cancels: interpolate(workflow.concurrency['cancel-in-progress'], context, functions), + }; +} + +const everyLane = { select: true, shards: true, mcp: true, apps: true, gate: true }; + +/** True when the gate's only step runs, which is the step that fails the required check. */ +function gateFails(results, mcpRun = '') { + return condition(workflow.jobs.e2e.steps[0].if, { + needs: { + e2e_shards: { result: results.shards ?? 'success' }, + mcp_apps: { result: results.apps ?? 'success' }, + mcp_tool_list_changed: { result: results.mcp ?? 'success' }, + codegraph_select: { outputs: { mcp_run: mcpRun } }, + }, + }); +} + test('PR selection and fail-open defaults use ten memory shards and two Redis transport shards', () => { assert.equal(defaults.length, 2); assert.deepEqual(defaults[0], prMatrix); @@ -115,10 +165,8 @@ test('PR and nightly matrices preserve both registered-email change modes on one test('manual PR coverage opts into the PR fallback matrix without bypassing the PR author gate', () => { assert.deepEqual(workflow.on.workflow_dispatch.inputs.coverage.options, ['full', 'pr']); - assert.match( - shards.strategy.matrix, - /github\.event_name == 'pull_request' \|\| inputs\.coverage == 'pr'/, - ); + assert.deepEqual(resolve(manual, { inputs: { coverage: 'pr' } }).matrix, prMatrix); + assert.deepEqual(resolve(manual, { inputs: { coverage: 'full' } }).matrix, defaults[1]); assert.match(shards.if, /github\.event_name == 'workflow_dispatch'/); assert.match(shards.if, /github\.event\.pull_request\.author_association/); }); @@ -218,5 +266,69 @@ test('the required gate still waits for all shards and rejects failures without assert.match(gate.steps[0].if, /needs\.mcp_tool_list_changed\.result != 'success'/); assert.match(gate.steps[0].if, /needs\.codegraph_select\.outputs\.mcp_run == 'false'/); assert.equal(gate.steps[0].run, 'exit 1'); - assert.equal(workflow.concurrency['cancel-in-progress'], true); + assert.equal(resolve(pullRequest('synchronize')).cancels, true); +}); + +test('a push to dev runs every lane on the pull request matrix with nothing narrowed', () => { + assert.deepEqual(workflow.on.push.branches, ['dev']); + assert.deepEqual(workflow.on.push.paths, workflow.on.pull_request.paths); + const { jobs, matrix } = resolve(pushToDev); + assert.deepEqual(jobs, { ...everyLane, select: false }); + assert.deepEqual(matrix, prMatrix); + assert.equal(gateFails({}), false); + assert.equal(gateFails({ shards: 'failure' }), true); + assert.equal(gateFails({ apps: 'failure' }), true); + assert.equal(gateFails({ mcp: 'failure' }), true); + /** No selector ran on a push, so a skipped MCP lane is a fault there, not a decision. */ + assert.equal(gateFails({ mcp: 'skipped' }), true); + assert.equal(gateFails({ mcp: 'skipped' }, 'false'), false); +}); + +test('dev pushes never cancel each other and share no group with any other event', () => { + const pushed = resolve(pushToDev); + assert.equal(pushed.group, 'playwright-mock-post-merge-refs/heads/dev'); + assert.equal(pushed.cancels, false); + const others = [pullRequest('opened'), pullRequest('synchronize'), nightly, manual].map( + (github) => resolve(github), + ); + assert.deepEqual( + others.map(({ group }) => group), + [ + 'playwright-mock-refs/pull/1/merge', + 'playwright-mock-refs/pull/1/merge', + 'playwright-mock-refs/heads/main', + 'playwright-mock-refs/heads/dev', + ], + ); + assert.ok(others.every(({ cancels }) => cancels === true)); +}); + +test('pull request, nightly and manual runs resolve as they did without the push trigger', () => { + const opened = resolve(pullRequest('opened')); + assert.deepEqual(opened.jobs, { ...everyLane, select: false }); + assert.deepEqual(opened.matrix, prMatrix); + + const memoryOnly = { include: prMatrix.include.filter((lane) => lane.suite === 'full') }; + const narrowed = resolve(pullRequest('synchronize'), { + select: { e2e_include: JSON.stringify(memoryOnly), mcp_run: 'false' }, + }); + assert.deepEqual(narrowed.jobs, { ...everyLane, mcp: false }); + assert.deepEqual(narrowed.matrix, memoryOnly); + + const outsider = resolve(pullRequest('synchronize', 'CONTRIBUTOR', 'someone/LibreChat')); + assert.deepEqual(outsider.jobs, { + select: false, + shards: false, + mcp: false, + apps: false, + gate: false, + }); + const sameRepositoryBot = resolve(pullRequest('opened', 'NONE')); + assert.deepEqual(sameRepositoryBot.jobs, { ...everyLane, select: false }); + + for (const github of [nightly, manual]) { + const run = resolve(github); + assert.deepEqual(run.jobs, { ...everyLane, select: false }); + assert.deepEqual(run.matrix, defaults[1]); + } }); diff --git a/.github/workflows/backend-review.yml b/.github/workflows/backend-review.yml index 9d11d37419..f5f9a6a28a 100644 --- a/.github/workflows/backend-review.yml +++ b/.github/workflows/backend-review.yml @@ -15,6 +15,15 @@ on: - '.bun-version' - 'config/circular-deps.mjs' - '.github/workflows/backend-review.yml' + # The guard spec list, its reader and their test, then every path a backend guard spec + # reads outside the trees above; guard-specs.test.mjs fails when one is missing here. + - '.github/guard-specs.json' + - '.github/scripts/guard-specs*.mjs' + - '.env.example' + - 'librechat.example.yaml' + - 'client/public/mcp-sandbox.html' + - 'e2e/setup/fake-model.js' + - 'vendor/code/**' - '!**.md' pull_request: paths: @@ -25,6 +34,15 @@ on: - '.bun-version' - 'config/circular-deps.mjs' - '.github/workflows/backend-review.yml' + # The guard spec list, its reader and their test, then every path a backend guard spec + # reads outside the trees above; guard-specs.test.mjs fails when one is missing here. + - '.github/guard-specs.json' + - '.github/scripts/guard-specs*.mjs' + - '.env.example' + - 'librechat.example.yaml' + - 'client/public/mcp-sandbox.html' + - 'e2e/setup/fake-model.js' + - 'vendor/code/**' - '!**.md' # Promotion requires full CI at the exact dev SHA, including path-filtered commits. workflow_dispatch: @@ -140,6 +158,9 @@ jobs: # graph, unclassifiable change, root/lockfile floors => mode FULL per workspace), and this # job emits nothing unless the response parses end to end — the worst case at every layer # is "CI runs everything", which is the pre-gating behavior. + # + # The graph follows imports, so a spec that reads source with fs is invisible to it. Those + # guard specs are listed in .github/guard-specs.json and run with every narrowed decision. codegraph_select: name: Codegraph select runs-on: ubuntu-latest @@ -159,6 +180,14 @@ jobs: dataschemas_run: ${{ steps.sel.outputs.dataschemas_run }} dataschemas_files: ${{ steps.sel.outputs.dataschemas_files }} steps: + # The commit under test, for the guard spec list, its reader and the reader's check that + # a selection is not stale. Best effort: without it the select step runs every narrowed + # workspace FULL, so a failed checkout never fails this job. + - uses: actions/checkout@v5 + continue-on-error: true + with: + persist-credentials: false + - name: Select tests, fail open on any doubt id: sel env: @@ -227,6 +256,23 @@ jobs: if [ "$mode" = "FILES" ] && [ -z "$files" ]; then mode="NONE"; fi fi fi + # Guard specs read source with fs, so no import edge selects them. A narrowed run + # carries its workspace's guards, and a skip becomes a guards-only run when the + # change touches what they read (.github/guard-specs.json; the rules are in + # .github/scripts/guard-specs.mjs). A list that cannot be read runs FULL. A selection + # with no file left in this checkout gets no guards: the test job runs FULL on a + # stale selection, and guards that exist would turn that into a guards-only run. + guard_note="" + if [ "$mode" = "FILES" ] || [ "$mode" = "NONE" ]; then + guarded=$(SELECTED="$files" node .github/scripts/guard-specs.mjs "$mode" "$ws" files.json "$ignore"); grc=$? + if [ "$grc" -ne 0 ]; then + mode="FULL"; files="" + note "| $ws | guard specs unreadable (exit $grc); running FULL |" + elif [ -n "$guarded" ]; then + if [ "$mode" = "NONE" ]; then guard_note=", guard specs only"; fi + mode="FILES"; files="$guarded" + fi + fi if [ "$mode" = "NONE" ]; then echo "${key}_run=false" >> "$GITHUB_OUTPUT" note "| $ws | skip (no reachable tests) |" @@ -234,7 +280,7 @@ jobs: n=$(echo "$files" | wc -w | tr -d ' ') echo "${key}_run=true" >> "$GITHUB_OUTPUT" echo "${key}_files=$files" >> "$GITHUB_OUTPUT" - note "| $ws | $n selected files |" + note "| $ws | $n selected files$guard_note |" else echo "${key}_run=true" >> "$GITHUB_OUTPUT" note "| $ws | FULL |" @@ -357,6 +403,11 @@ jobs: - name: Detect circular dependencies run: node config/circular-deps.mjs + # Here because this job has node_modules, always runs and gates no other job. + - name: Test guard spec selection + if: ${{ !cancelled() }} + run: node --test .github/scripts/guard-specs.test.mjs + test-api: name: 'Tests: api (shard ${{ matrix.shard }}/3)' needs: [build, codegraph_select] diff --git a/.github/workflows/codegraph-e2e-votes.yml b/.github/workflows/codegraph-e2e-votes.yml index 4ce57125ef..d1e40e0e9d 100644 --- a/.github/workflows/codegraph-e2e-votes.yml +++ b/.github/workflows/codegraph-e2e-votes.yml @@ -1,9 +1,10 @@ # Codegraph e2e VOTES — observe-only, post-merge, time-boxed. # -# Playwright never runs on pushes to dev, so evidence for the e2e skip election would -# otherwise wait on rare organic PR spec failures. This workflow runs the FULL mock suite after -# merges: each run is one graduation trial for every spec it executes, and doubles as the -# post-merge safety net the jest workflows already have via their dev-push triggers. +# Evidence for the e2e skip election would otherwise wait on rare organic PR spec failures, so +# this workflow runs the FULL mock suite after merges: each run is one graduation trial for +# every spec it executes. It is not the post-merge safety net: its test step cannot fail (see +# below), so a regression on dev leaves it green. The run that goes red is playwright-mock.yml's +# own dev-push trigger. # # By default every merge votes (the push trigger). Each run costs about 154 job-minutes on ten # runners, so with repo variable CODEGRAPH_E2E_VOTE_MIN_HOURS set to N > 0 a push run votes diff --git a/.github/workflows/frontend-review.yml b/.github/workflows/frontend-review.yml index 5bddf5bd18..f5d2df96ef 100644 --- a/.github/workflows/frontend-review.yml +++ b/.github/workflows/frontend-review.yml @@ -13,6 +13,11 @@ on: - 'package-lock.json' - '.bun-version' - '.github/workflows/frontend-review.yml' + # The guard spec list, its reader and their test, then every path a frontend guard spec + # reads outside the trees above; guard-specs.test.mjs fails when one is missing here. + - '.github/guard-specs.json' + - '.github/scripts/guard-specs*.mjs' + - 'eslint-suppressions.json' - '!**.md' # Post-merge safety net and full-run baseline for gated selection (same rationale as # backend-review.yml stage 1): every dev merge touching frontend paths runs the full suite. @@ -30,6 +35,11 @@ on: - 'package-lock.json' - '.bun-version' - '.github/workflows/frontend-review.yml' + # The guard spec list, its reader and their test, then every path a frontend guard spec + # reads outside the trees above; guard-specs.test.mjs fails when one is missing here. + - '.github/guard-specs.json' + - '.github/scripts/guard-specs*.mjs' + - 'eslint-suppressions.json' # Promotion requires full CI at the exact dev SHA, including path-filtered commits. workflow_dispatch: @@ -102,6 +112,10 @@ jobs: - run: npm run test:client-build env: PLAYWRIGHT_CHANNEL: chrome + # Here because this job has node_modules, always runs and gates no other job. + - name: Test guard spec selection + if: ${{ !cancelled() }} + run: node --test .github/scripts/guard-specs.test.mjs # Stage 1.5 of codegraph gating (stage 1 = backend jest, #15132; stage 2 = matrix lanes, # #15136). Same mechanism, same record: across 604 finalized shadow receipts the frontend @@ -110,6 +124,9 @@ jobs: # every dev push, and on any doubt. A workspace skips ONLY on an explicit NONE. Kill switch: # repo variable CODEGRAPH_GATING=off. Neither frontend workspace defines # testPathIgnorePatterns, so --runTestsByPath needs no exclude mirroring here. + # + # The graph follows imports, so a spec that reads source with fs is invisible to it. Those + # guard specs are listed in .github/guard-specs.json and run with every narrowed decision. codegraph_select: name: Codegraph select runs-on: ubuntu-latest @@ -125,6 +142,14 @@ jobs: clientpkg_run: ${{ steps.sel.outputs.clientpkg_run }} clientpkg_files: ${{ steps.sel.outputs.clientpkg_files }} steps: + # The commit under test, for the guard spec list, its reader and the reader's check that + # a selection is not stale. Best effort: without it the select step runs every narrowed + # workspace FULL, so a failed checkout never fails this job. + - uses: actions/checkout@v5 + continue-on-error: true + with: + persist-credentials: false + - name: Select tests, fail open on any doubt id: sel env: @@ -180,6 +205,23 @@ jobs: if [ "$spaced" != "0" ]; then mode="FULL"; files=""; fi fi fi + # Guard specs read source with fs, so no import edge selects them. A narrowed run + # carries its workspace's guards, and a skip becomes a guards-only run when the + # change touches what they read (.github/guard-specs.json; the rules are in + # .github/scripts/guard-specs.mjs). A list that cannot be read runs FULL. A selection + # with no file left in this checkout gets no guards: the test job runs FULL on a + # stale selection, and guards that exist would turn that into a guards-only run. + guard_note="" + if [ "$mode" = "FILES" ] || [ "$mode" = "NONE" ]; then + guarded=$(SELECTED="$files" node .github/scripts/guard-specs.mjs "$mode" "$ws" files.json ""); grc=$? + if [ "$grc" -ne 0 ]; then + mode="FULL"; files="" + note "| $ws | guard specs unreadable (exit $grc); running FULL |" + elif [ -n "$guarded" ]; then + if [ "$mode" = "NONE" ]; then guard_note=", guard specs only"; fi + mode="FILES"; files="$guarded" + fi + fi if [ "$mode" = "NONE" ]; then echo "${key}_run=false" >> "$GITHUB_OUTPUT" note "| $ws | skip (no reachable tests) |" @@ -187,7 +229,7 @@ jobs: n=$(echo "$files" | wc -w | tr -d ' ') echo "${key}_run=true" >> "$GITHUB_OUTPUT" echo "${key}_files=$files" >> "$GITHUB_OUTPUT" - note "| $ws | $n selected files |" + note "| $ws | $n selected files$guard_note |" else echo "${key}_run=true" >> "$GITHUB_OUTPUT" note "| $ws | FULL |" diff --git a/.github/workflows/playwright-mock.yml b/.github/workflows/playwright-mock.yml index d071052ec6..13a3c64af3 100644 --- a/.github/workflows/playwright-mock.yml +++ b/.github/workflows/playwright-mock.yml @@ -7,6 +7,19 @@ on: - '!**.md' - '!.github/workflows/**' - '.github/workflows/playwright-mock.yml' + # Post-merge safety net, as backend-review.yml and frontend-review.yml have for jest. A + # regression that reaches dev fails this suite on every open pull request, and without a run + # on dev itself nothing names the merge that caused it. A push runs the pull request profile + # with no lane narrowed, so a red run on dev is what the next pull request would see. The + # nightly stays on the default branch, where it covers the full suite on both stores. + push: + branches: + - dev + paths: + - '**' + - '!**.md' + - '!.github/workflows/**' + - '.github/workflows/playwright-mock.yml' schedule: - cron: '0 5 * * *' workflow_dispatch: @@ -27,8 +40,16 @@ permissions: contents: read concurrency: - group: playwright-mock-${{ github.ref }} - cancel-in-progress: true + # Pull requests, the nightly and manual runs keep a per-ref group in which a newer run cancels + # the older one. Pushes to dev take a group of their own that never cancels, so the run in + # progress always finishes; GitHub holds one pending run per group and a newer one replaces + # it, so merges that land meanwhile collapse into a single run of the newest commit. A + # cancelling group would rarely finish on a busy day (nine merges landed within 89 seconds on + # 2026-11-09) and a run per commit, which the jest workflows use, costs about 190 + # runner-minutes on 15 runners each time. A red run therefore names a range of merges, from + # the commit of the last finished run to its own, and the dev tip always gets a run. + group: playwright-mock-${{ github.event_name == 'push' && 'post-merge-' || '' }}${{ github.ref }} + cancel-in-progress: ${{ github.event_name != 'push' }} env: NODE_OPTIONS: '--max-old-space-size=${{ secrets.NODE_MAX_OLD_SPACE_SIZE || 6144 }}' @@ -39,9 +60,9 @@ jobs: # two matrix lanes whose relevance is a reachability question: the redis-transport shards (the # stream-boundary specs under a real Redis round-trip) and the MCP list_changed lanes. Memory # shards always run. Monotone and fail-open: a lane is dropped ONLY on an explicit `false` from - # the service; unavailable/unconfigured/non-synchronize events keep the full matrix. Lock - # attribution rides along so a backend dependency bump (reaches nothing in the graph) still - # earns both lanes. Kill switch: repo variable CODEGRAPH_GATING=off. + # the service; unavailable/unconfigured/non-synchronize events (dev pushes among them) keep the + # full matrix. Lock attribution rides along so a backend dependency bump (reaches nothing in + # the graph) still earns both lanes. Kill switch: repo variable CODEGRAPH_GATING=off. codegraph_select: name: Codegraph select runs-on: ubuntu-latest @@ -153,7 +174,7 @@ jobs: fi echo "decided=true" >> "$GITHUB_OUTPUT" note "" - note "memory shards always run · kill switch: repo variable \`CODEGRAPH_GATING=off\` · full matrix on PR open and nightly" + note "memory shards always run · kill switch: repo variable \`CODEGRAPH_GATING=off\` · full matrix on PR open, on every dev push and nightly" exit 0 e2e_shards: @@ -163,6 +184,7 @@ jobs: !cancelled() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || + github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request != null && (contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) || @@ -177,7 +199,7 @@ jobs: ${{ (github.event_name == 'pull_request' && needs.codegraph_select.outputs.e2e_include != '' && fromJSON(needs.codegraph_select.outputs.e2e_include)) || - ((github.event_name == 'pull_request' || inputs.coverage == 'pr') && + ((github.event_name == 'pull_request' || github.event_name == 'push' || inputs.coverage == 'pr') && fromJSON('{"include":[{"name":"memory, shard 1/10","stream_store":"memory","redis_image":"","suite":"full","shard":"1/10","artifact":"memory-1-of-10","email_change":"true"},{"name":"memory, shard 2/10","stream_store":"memory","redis_image":"","suite":"full","shard":"2/10","artifact":"memory-2-of-10"},{"name":"memory, shard 3/10","stream_store":"memory","redis_image":"","suite":"full","shard":"3/10","artifact":"memory-3-of-10"},{"name":"memory, shard 4/10","stream_store":"memory","redis_image":"","suite":"full","shard":"4/10","artifact":"memory-4-of-10"},{"name":"memory, shard 5/10","stream_store":"memory","redis_image":"","suite":"full","shard":"5/10","artifact":"memory-5-of-10"},{"name":"memory, shard 6/10","stream_store":"memory","redis_image":"","suite":"full","shard":"6/10","artifact":"memory-6-of-10"},{"name":"memory, shard 7/10","stream_store":"memory","redis_image":"","suite":"full","shard":"7/10","artifact":"memory-7-of-10"},{"name":"memory, shard 8/10","stream_store":"memory","redis_image":"","suite":"full","shard":"8/10","artifact":"memory-8-of-10"},{"name":"memory, shard 9/10","stream_store":"memory","redis_image":"","suite":"full","shard":"9/10","artifact":"memory-9-of-10"},{"name":"memory, shard 10/10","stream_store":"memory","redis_image":"","suite":"full","shard":"10/10","artifact":"memory-10-of-10"},{"name":"redis transport, shard 1/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"1/2","artifact":"redis-transport-1-of-2"},{"name":"redis transport, shard 2/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"2/2","artifact":"redis-transport-2-of-2"}]}')) || fromJSON('{"include":[{"name":"memory, shard 1/10","stream_store":"memory","redis_image":"","suite":"full","shard":"1/10","artifact":"memory-1-of-10","email_change":"true"},{"name":"memory, shard 2/10","stream_store":"memory","redis_image":"","suite":"full","shard":"2/10","artifact":"memory-2-of-10"},{"name":"memory, shard 3/10","stream_store":"memory","redis_image":"","suite":"full","shard":"3/10","artifact":"memory-3-of-10"},{"name":"memory, shard 4/10","stream_store":"memory","redis_image":"","suite":"full","shard":"4/10","artifact":"memory-4-of-10"},{"name":"memory, shard 5/10","stream_store":"memory","redis_image":"","suite":"full","shard":"5/10","artifact":"memory-5-of-10"},{"name":"memory, shard 6/10","stream_store":"memory","redis_image":"","suite":"full","shard":"6/10","artifact":"memory-6-of-10"},{"name":"memory, shard 7/10","stream_store":"memory","redis_image":"","suite":"full","shard":"7/10","artifact":"memory-7-of-10"},{"name":"memory, shard 8/10","stream_store":"memory","redis_image":"","suite":"full","shard":"8/10","artifact":"memory-8-of-10"},{"name":"memory, shard 9/10","stream_store":"memory","redis_image":"","suite":"full","shard":"9/10","artifact":"memory-9-of-10"},{"name":"memory, shard 10/10","stream_store":"memory","redis_image":"","suite":"full","shard":"10/10","artifact":"memory-10-of-10"},{"name":"redis, shard 1/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"1/10","artifact":"redis-1-of-10"},{"name":"redis, shard 2/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"2/10","artifact":"redis-2-of-10"},{"name":"redis, shard 3/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"3/10","artifact":"redis-3-of-10"},{"name":"redis, shard 4/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"4/10","artifact":"redis-4-of-10"},{"name":"redis, shard 5/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"5/10","artifact":"redis-5-of-10"},{"name":"redis, shard 6/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"6/10","artifact":"redis-6-of-10"},{"name":"redis, shard 7/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"7/10","artifact":"redis-7-of-10"},{"name":"redis, shard 8/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"8/10","artifact":"redis-8-of-10"},{"name":"redis, shard 9/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"9/10","artifact":"redis-9-of-10"},{"name":"redis, shard 10/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"10/10","artifact":"redis-10-of-10"}]}') }} @@ -438,6 +460,7 @@ jobs: needs.codegraph_select.outputs.mcp_run != 'false' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || + github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request != null && (contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) || @@ -588,11 +611,12 @@ jobs: mcp_apps: name: MCP Apps (browser and server) - # Always run the dedicated fixture on eligible PRs, schedule, and manual runs. The regular - # mock shards skip it unless E2E_MCP_APPS=true; codegraph gating must not omit this lane. + # Always run the dedicated fixture on eligible PRs, dev pushes, schedule, and manual runs. The + # regular mock shards skip it unless E2E_MCP_APPS=true; codegraph gating must not omit this lane. if: >- github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || + github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request != null && (contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) || @@ -675,6 +699,7 @@ jobs: !cancelled() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || + github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request != null && (contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) ||