From 97f7fcdbacecf7baa48ec9bf0316e0c878647971 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 11 Nov 2026 10:30:09 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=A5=85=20ci:=20Run=20Playwright=20After?= =?UTF-8?q?=20Merges=20to=20Dev=20and=20Guard=20Specs=20Under=20Test=20Sel?= =?UTF-8?q?ection=20(#18049)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🥅 ci: Run Playwright After Every Merge to Dev The mock e2e suite ran on pull requests, nightly on main and on demand, so when a spec broke on dev no run on dev could go red: the vote workflow that runs the suite after merges cannot fail by design. Add a push trigger for dev with the pull request path filter. A push runs the pull request profile with no lane narrowed (10 memory shards, 2 Redis transport shards, both MCP lanes and MCP Apps), and the gate job reports it. Pushes take their own concurrency group that never cancels: the run in progress finishes, and GitHub keeps one pending run per group, so merges that land meanwhile collapse into one run of the newest commit. Pull request, nightly and manual runs keep their per-ref cancelling group. The matrix test now evaluates the workflow's own expressions per event rather than matching their text. * 🔭 ci: Always Run Source-Scanning Guard Specs Under Test Selection Codegraph test selection narrows a pull request run to the test files an import graph reaches. A spec that reads source with fs has no import edge to what it reads: toolCapabilityGates.spec.js scans four source trees, was not selected when #17864 changed one of the files it counts, and failed on dev afterwards. List every such spec, with the paths it reads, in .github/guard-specs.json. Both selectors now add a workspace's guards to a FILES decision and turn a NONE into a guards-only run when the change touches a path one of them reads, renames a file, or edits the list. A list that cannot be read runs the workspace FULL. FULL decisions and every earlier fail-open path are unchanged. The rule keys off the decision, not the event. A selection with no file left in the checkout gets no guards, so the test job still runs FULL on a stale selection instead of running only the guards. guard-specs.test.mjs fails when a spec in a gated workspace reads the repository through fs without being listed, when a listed path is missing, and when a workflow would not run for a path one of its guards reads. That last check adds the five backend and one frontend paths the triggers lacked. The test also runs both real select steps against canned codegraph responses. --- .github/MAIN_PROMOTION.md | 9 +- .github/guard-specs.json | 105 ++++ .github/scripts/expressions.mjs | 169 +++++++ .github/scripts/guard-specs.mjs | 158 ++++++ .github/scripts/guard-specs.test.mjs | 540 +++++++++++++++++++++ .github/scripts/playwright-matrix.test.mjs | 122 ++++- .github/workflows/backend-review.yml | 53 +- .github/workflows/codegraph-e2e-votes.yml | 9 +- .github/workflows/frontend-review.yml | 44 +- .github/workflows/playwright-mock.yml | 43 +- 10 files changed, 1229 insertions(+), 23 deletions(-) create mode 100644 .github/guard-specs.json create mode 100644 .github/scripts/expressions.mjs create mode 100644 .github/scripts/guard-specs.mjs create mode 100644 .github/scripts/guard-specs.test.mjs diff --git a/.github/MAIN_PROMOTION.md b/.github/MAIN_PROMOTION.md index c562757892..fdebb23308 100644 --- a/.github/MAIN_PROMOTION.md +++ b/.github/MAIN_PROMOTION.md @@ -34,9 +34,12 @@ for the exact input SHA, from the repository's existing backend and frontend wor IDs and paths. All build, typecheck, test-shard and circular-dependency jobs must be present and successful. Only the PR-only **Codegraph select** job may be skipped. An older successful run cannot conceal a newer failed or pending full run. PR checks -and checks from a different commit do not qualify. This gate does not imply that -Lighthouse, Playwright or integration lanes that only run on PRs tested the resulting -post-merge SHA; maintainers still assess those PR results and release readiness. +and checks from a different commit do not qualify. This gate does not read Lighthouse, +Playwright or integration lanes. Lighthouse and integration lanes run only on PRs. +Playwright also runs after merges to `dev`, where merges that land during a run share +one run of the newest commit, so the `dev` tip has a run of its own unless it changed +only Markdown or other workflows. Maintainers still assess those results and release +readiness; a manual **Playwright E2E Tests** run on `dev` covers both stores in full. The candidate must be the current `dev` tip, and the reviewed baseline must still be the current `main` tip. `main` must be an ancestor of the candidate. Ref and CI diff --git a/.github/guard-specs.json b/.github/guard-specs.json new file mode 100644 index 0000000000..7505565a95 --- /dev/null +++ b/.github/guard-specs.json @@ -0,0 +1,105 @@ +{ + "description": "Specs that read repository files through fs instead of importing them. Codegraph test selection follows imports, so it cannot tell when a change makes one of these relevant; backend-review.yml and frontend-review.yml add them to every narrowed run (.github/scripts/guard-specs.mjs). Keys are spec paths. A guard lists the repository paths it reads: a directory ends with a slash and covers everything under it. An exempt spec matches the detector without reading tracked files and says why.", + "guards": { + "api/server/controllers/__tests__/deleteUserResourceCoverage.spec.js": [ + "api/server/controllers/UserController.js", + "packages/api/src/user/erasure.ts" + ], + "api/server/experimental.spec.js": ["api/server/experimental.js", "api/server/index.js"], + "api/server/index.spec.js": ["api/server/index.js", "api/server/experimental.js"], + "api/server/routes/agents/__tests__/openapi.contract.spec.js": [ + "packages/api/openapi/agents.openapi.json" + ], + "api/server/services/__tests__/toolCapabilityGates.spec.js": [ + "api/app/", + "api/server/", + "packages/api/src/", + "packages/data-provider/src/" + ], + "api/server/utils/import/importers.spec.js": ["api/server/utils/import/__data__/"], + "client/src/Providers/__tests__/bootDirection.spec.ts": ["client/index.html"], + "client/src/Providers/__tests__/bootScript.spec.tsx": ["client/index.html"], + "client/src/Providers/__tests__/handoff.spec.tsx": ["client/index.html"], + "client/src/__tests__/hosts.test.tsx": ["client/src/App.jsx"], + "client/src/__tests__/loadingCanvas.spec.ts": ["client/index.html"], + "client/src/components/Chat/Input/Composer/__tests__/semanticTokens.spec.ts": [ + "client/src/components/Chat/", + "client/src/hooks/Input/usePaletteEntries.tsx", + "packages/chat/src/components/composer/" + ], + "client/src/components/Chat/Input/__tests__/appearanceTokens.spec.ts": [ + "client/src/components/Chat/Input/", + "packages/chat/src/components/composer/" + ], + "client/src/components/Chat/Input/__tests__/panelRadius.spec.ts": [ + "client/src/components/", + "packages/chat/src/components/", + "packages/client/src/components/" + ], + "client/src/lib/assets/__tests__/recovery.spec.tsx": ["client/index.html"], + "client/src/style.spec.ts": [ + "client/src/style.css", + "client/tailwind.config.cjs", + "packages/client/src/theme/", + "packages/client/tailwind.preset.cjs", + "package-lock.json" + ], + "client/src/utils/__tests__/mcpSandbox.spec.ts": ["client/public/mcp-sandbox.html"], + "packages/api/src/admin/meta.spec.ts": ["packages/data-provider/package.json"], + "packages/api/src/admin/secrets.runtime.spec.ts": [ + "packages/api/src/app/", + "packages/api/src/langfuse/utils.ts", + "packages/api/src/web/web.ts" + ], + "packages/api/src/agents/tools.spec.ts": ["e2e/setup/fake-model.js"], + "packages/api/src/channels/deployment.spec.ts": [".env.example"], + "packages/api/src/code/agent.spec.ts": ["api/", "packages/api/src/"], + "packages/api/src/code/setup/bridge.spec.ts": ["vendor/code/"], + "packages/api/src/files/documents/html.spec.ts": ["packages/api/src/files/documents/"], + "packages/api/src/files/documents/libreoffice.spec.ts": ["packages/api/src/files/documents/"], + "packages/api/src/files/documents/spreadsheet.spec.ts": ["packages/api/src/files/documents/"], + "packages/api/src/mcp/__tests__/sandbox.test.ts": ["client/public/mcp-sandbox.html"], + "packages/client/src/components/Progress.spec.tsx": ["packages/client/src/theme/tokens.css"], + "packages/client/src/components/SegmentedMeter.spec.tsx": [ + "packages/client/src/theme/tokens.css" + ], + "packages/client/src/components/__tests__/radixLayers.spec.ts": ["package-lock.json"], + "packages/client/src/theme/defaults.spec.ts": ["packages/client/src/theme/defaults.css"], + "packages/client/src/theme/primitives.spec.ts": [ + "eslint-suppressions.json", + "packages/client/src/components/" + ], + "packages/client/src/theme/registry.spec.ts": ["package-lock.json"], + "packages/client/src/theme/semanticTokens.spec.ts": [ + "client/src/style.css", + "packages/client/src/components/", + "packages/client/src/theme/defaults.css" + ], + "packages/client/src/theme/tailwind.spec.js": [ + "client/src/style.css", + "client/tailwind.config.cjs", + "packages/client/", + "package-lock.json" + ], + "packages/client/src/theme/themes/clickhouse.spec.tsx": ["packages/client/src/theme/fonts.css"], + "packages/client/src/theme/themes/clickui.spec.ts": [ + "client/src/style.css", + "packages/client/src/" + ], + "packages/client/src/theme/tokens.spec.ts": [ + "packages/client/src/theme/", + "packages/client/tailwind.preset.cjs", + "package-lock.json" + ], + "packages/data-provider/specs/config-schemas.spec.ts": ["librechat.example.yaml"], + "packages/data-schemas/src/methods/documentdb.spec.ts": [ + "api/", + "packages/api/src/", + "packages/data-schemas/src/" + ] + }, + "exempt": { + "packages/api/src/code/environment.live.spec.ts": "Skips unless LIBRECHAT_CODE_TEST_PACKAGE is set, and then reads only the temp directory it creates.", + "packages/api/src/plugins/runtime.spec.ts": "Reads only logs it writes under os.tmpdir(); process.cwd() appears inside an inline hook command." + } +} diff --git a/.github/scripts/expressions.mjs b/.github/scripts/expressions.mjs new file mode 100644 index 0000000000..66f3ef3c77 --- /dev/null +++ b/.github/scripts/expressions.mjs @@ -0,0 +1,169 @@ +/** + * Evaluates the subset of GitHub Actions expressions the workflows here use, so a test can + * state what a job condition, matrix or concurrency group resolves to for a given event + * instead of matching the expression's text. Covers literals, context lookups, `!`, `==`, + * `!=`, `&&`, `||`, parentheses and the functions passed in `functions`. + */ + +const TOKEN = + /\s*(?:('(?:[^']|'')*')|(-?\d+(?:\.\d+)?)|(==|!=|&&|\|\||[!(),])|([A-Za-z_][\w-]*(?:\.[\w-]+)*))/y; + +function tokenize(source) { + const tokens = []; + TOKEN.lastIndex = 0; + while (TOKEN.lastIndex < source.length) { + const start = TOKEN.lastIndex; + const match = TOKEN.exec(source); + if (!match) { + if (source.slice(start).trim() === '') break; + throw new Error(`Unsupported expression syntax at: ${source.slice(start, start + 40)}`); + } + const [, string, number, operator, name] = match; + if (string !== undefined) { + tokens.push({ type: 'value', value: string.slice(1, -1).replaceAll("''", "'") }); + } else if (number !== undefined) { + tokens.push({ type: 'value', value: Number(number) }); + } else if (operator !== undefined) { + tokens.push({ type: operator }); + } else { + tokens.push({ type: 'name', name }); + } + } + return tokens; +} + +const truthy = (value) => + !(value === false || value === 0 || value === '' || value == null || Number.isNaN(value)); + +function toNumber(value) { + if (value == null || value === '') return 0; + if (typeof value === 'boolean') return value ? 1 : 0; + if (typeof value === 'number') return value; + return typeof value === 'string' ? Number(value) : NaN; +} + +/** Same-type values compare directly (strings ignoring case); mixed types compare as numbers. */ +function equal(left, right) { + if (typeof left === 'string' && typeof right === 'string') { + return left.toLowerCase() === right.toLowerCase(); + } + if (left === right) return true; + const sameType = typeof left === typeof right && left != null && right != null; + return sameType ? false : toNumber(left) === toNumber(right); +} + +const BUILTINS = { + fromJSON: (text) => JSON.parse(text), + contains: (haystack, needle) => + Array.isArray(haystack) + ? haystack.some((item) => equal(item, needle)) + : String(haystack ?? '') + .toLowerCase() + .includes(String(needle ?? '').toLowerCase()), +}; + +const LITERALS = new Map([ + ['true', true], + ['false', false], + ['null', null], +]); + +/** A missing property is null, as in Actions. */ +const lookup = (context, path) => + path.split('.').reduce((value, key) => (value == null ? null : (value[key] ?? null)), context); + +export function evaluate(source, context = {}, functions = {}) { + const tokens = tokenize(source); + const calls = { ...BUILTINS, ...functions }; + let position = 0; + const peek = () => tokens[position]?.type; + const take = (type) => { + const token = tokens[position]; + if (!token || (type && token.type !== type)) { + throw new Error(`Expected ${type ?? 'a token'} in: ${source}`); + } + position += 1; + return token; + }; + + /** `evaluating` is false on the short-circuited side, which is parsed but never called. */ + function primary(evaluating) { + if (peek() === '(') { + take('('); + const value = or(evaluating); + take(')'); + return value; + } + if (peek() === 'value') return take('value').value; + const { name } = take('name'); + if (peek() !== '(') return LITERALS.has(name) ? LITERALS.get(name) : lookup(context, name); + take('('); + const args = []; + while (peek() !== ')') { + args.push(or(evaluating)); + if (peek() === ',') take(','); + } + take(')'); + if (!evaluating) return null; + if (!calls[name]) throw new Error(`Unsupported function ${name}() in: ${source}`); + return calls[name](...args); + } + + function unary(evaluating) { + if (peek() !== '!') return primary(evaluating); + take('!'); + return !truthy(unary(evaluating)); + } + + function comparison(evaluating) { + let left = unary(evaluating); + while (peek() === '==' || peek() === '!=') { + const { type } = take(); + const right = unary(evaluating); + left = type === '==' ? equal(left, right) : !equal(left, right); + } + return left; + } + + function and(evaluating) { + let left = comparison(evaluating); + while (peek() === '&&') { + take('&&'); + const right = comparison(evaluating && truthy(left)); + left = truthy(left) ? right : left; + } + return left; + } + + function or(evaluating) { + let left = and(evaluating); + while (peek() === '||') { + take('||'); + const right = and(evaluating && !truthy(left)); + left = truthy(left) ? left : right; + } + return left; + } + + const value = or(true); + if (position !== tokens.length) throw new Error(`Trailing tokens in: ${source}`); + return value; +} + +/** A job-level `if` is an expression whether or not it is wrapped in `${{ }}`. */ +export function condition(source, context, functions) { + const text = String(source).trim(); + const wrapped = /^\$\{\{([\s\S]*)\}\}$/.exec(text); + return truthy(evaluate(wrapped ? wrapped[1] : text, context, functions)); +} + +/** Resolves every `${{ }}` in a string; a value that is only an expression keeps its type. */ +export function interpolate(template, context, functions) { + const text = String(template).trim(); + const whole = /^\$\{\{((?:(?!\}\})[\s\S])*)\}\}$/.exec(text); + if (whole) return evaluate(whole[1], context, functions); + return text.replace(/\$\{\{((?:(?!\}\})[\s\S])*)\}\}/g, (_, expression) => { + const value = evaluate(expression, context, functions); + return value == null ? '' : String(value); + }); +} diff --git a/.github/scripts/guard-specs.mjs b/.github/scripts/guard-specs.mjs new file mode 100644 index 0000000000..384f69b64b --- /dev/null +++ b/.github/scripts/guard-specs.mjs @@ -0,0 +1,158 @@ +/** + * Guard specs under codegraph test selection. + * + * Selection (the `codegraph_select` jobs in backend-review.yml and frontend-review.yml) narrows a + * pull request run to the test files an import graph reaches from the change. A spec that reads + * source with `fs` has no import edge to what it reads: toolCapabilityGates.spec.js scans four + * source trees, was not selected when #17864 changed one of the files it counts, and then failed + * on dev. `.github/guard-specs.json` lists every such spec with the paths it reads, and this + * script adds them to a workspace's decision: + * + * FILES every guard of the workspace joins the selected files, unless none of the selected + * files exists in this checkout: the test job runs FULL on such a stale selection, and + * guards that do exist would hide it; + * NONE the workspace runs its guards alone when the change touches a path one of them reads, + * renames a file, or edits the list or this script; otherwise it stays skipped. + * + * Usage: SELECTED="" node guard-specs.mjs [ignore] + * Run from the root of a checkout of the commit under test. Prints the workspace-relative files + * to run, space separated; nothing means skip. Any failure exits non-zero and the caller runs + * the workspace FULL. + */ +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { readFileSync, statSync } from 'node:fs'; + +export const MANIFEST_PATH = '.github/guard-specs.json'; +const SCRIPT_PATH = '.github/scripts/guard-specs.mjs'; +const MANIFEST_URL = new URL('../guard-specs.json', import.meta.url); + +/** A spec file, by the suffixes jest matches in the gated workspaces. */ +export const SPEC_FILE = /\.(?:spec|test)\.[cm]?[jt]sx?$/; + +/** + * What marks a spec as reading the repository: a filesystem read or walk, together with a path + * anchored at the spec's own location or the working directory. A spec that only uses temp + * directories has no anchor; one that only imports has no read. + */ +const FS_READ = + /\b(?:read(?:File|dir)(?:Sync)?|opendir(?:Sync)?|createReadStream|glob(?:Sync)?|globby)\s*\(/; +const REPO_ANCHOR = + /__dirname|__filename|process\.cwd\(\)|import\.meta\.(?:url|dirname|filename)|require\.resolve/; + +export const readsRepository = (source) => FS_READ.test(source) && REPO_ANCHOR.test(source); + +/** + * Listed paths end up unquoted in the test jobs' shell (`--runTestsByPath $KEEP`), so anything + * that could split, glob or climb out of the repository is refused here. + */ +const SAFE_PATH = /^[A-Za-z0-9_.-]+(?:\/[A-Za-z0-9_.-]+)*\/?$/; +const isSafePath = (path) => + typeof path === 'string' && SAFE_PATH.test(path) && !path.split('/').includes('..'); + +const isRecord = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); + +export function parseManifest(text) { + const { guards, exempt = {} } = JSON.parse(text); + if (!isRecord(guards) || !isRecord(exempt)) { + throw new Error(`${MANIFEST_PATH}: "guards" and "exempt" must be objects`); + } + for (const [spec, scans] of Object.entries(guards)) { + if (!isSafePath(spec) || !SPEC_FILE.test(spec)) { + throw new Error(`${MANIFEST_PATH}: guard is not a plain spec path: ${spec}`); + } + if (!Array.isArray(scans) || scans.length === 0 || !scans.every(isSafePath)) { + throw new Error(`${MANIFEST_PATH}: ${spec} must list the repository paths it reads`); + } + if (spec in exempt) { + throw new Error(`${MANIFEST_PATH}: ${spec} is both a guard and exempt`); + } + } + for (const [spec, reason] of Object.entries(exempt)) { + if (!isSafePath(spec) || typeof reason !== 'string' || reason.trim() === '') { + throw new Error(`${MANIFEST_PATH}: exempt spec needs a plain path and a reason: ${spec}`); + } + } + return { guards, exempt }; +} + +export const loadManifest = () => parseManifest(readFileSync(MANIFEST_URL, 'utf8')); + +const covers = (scan, path) => (scan.endsWith('/') ? path.startsWith(scan) : path === scan); + +/** + * A rename reports only its new path, so one can move a file out of a scanned tree unseen; and + * a change to the list or to this script can change which guards exist. Both count for all. + */ +const touchesEveryGuard = ({ path, status }) => + status === 'renamed' || path === MANIFEST_PATH || path === SCRIPT_PATH; + +/** + * @param {object} options + * @param {{ guards: Record }} options.manifest + * @param {'FILES' | 'NONE'} options.mode The workspace's decision after the caller's own checks. + * @param {string} options.workspace Repository-relative workspace directory, e.g. `packages/api`. + * @param {{ path: string, status?: string }[]} options.changed The pull request's changed files. + * @param {string[]} [options.selected] Workspace-relative files already selected (FILES only). + * @param {(file: string) => boolean} options.exists Whether a workspace-relative file is in the + * checkout the tests will run against. + * @param {string} [options.ignore] Regex of paths the workspace's own jest run excludes. + * @returns {string[]} Workspace-relative files to run; empty means the workspace stays skipped. + */ +export function withGuards({ + manifest, + mode, + workspace, + changed, + exists, + selected = [], + ignore = '', +}) { + if (mode !== 'FILES' && mode !== 'NONE') throw new Error(`Unsupported mode: ${mode}`); + if (typeof exists !== 'function') throw new Error('An exists(file) check is required'); + if (!isSafePath(workspace) || workspace.endsWith('/')) { + throw new Error(`Unsupported workspace: ${workspace}`); + } + if (!Array.isArray(changed) || !changed.every((file) => typeof file?.path === 'string')) { + throw new Error('The changed-file list must be an array of { path }'); + } + const prefix = `${workspace}/`; + const excluded = ignore === '' ? null : new RegExp(ignore); + const guards = Object.entries(manifest.guards).filter( + ([spec]) => spec.startsWith(prefix) && !excluded?.test(spec), + ); + const files = guards.map(([spec]) => spec.slice(prefix.length)); + if (mode === 'FILES') { + return selected.some(exists) ? [...new Set([...selected, ...files])] : selected; + } + const relevant = changed.some( + (file) => + touchesEveryGuard(file) || + guards.some( + ([spec, scans]) => file.path === spec || scans.some((scan) => covers(scan, file.path)), + ), + ); + return relevant ? files : []; +} + +function main([mode, workspace, changedFile, ignore = '']) { + const files = withGuards({ + manifest: loadManifest(), + mode, + workspace, + changed: JSON.parse(readFileSync(changedFile, 'utf8')), + exists: (file) => statSync(join(workspace, file), { throwIfNoEntry: false })?.isFile() === true, + selected: (process.env.SELECTED ?? '').split(/\s+/).filter(Boolean), + ignore, + }); + process.stdout.write(files.join(' ')); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + try { + main(process.argv.slice(2)); + } catch (error) { + console.error(`guard-specs: ${error.message}`); + process.exit(1); + } +} diff --git a/.github/scripts/guard-specs.test.mjs b/.github/scripts/guard-specs.test.mjs new file mode 100644 index 0000000000..b330bbb0a1 --- /dev/null +++ b/.github/scripts/guard-specs.test.mjs @@ -0,0 +1,540 @@ +import yaml from 'js-yaml'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { condition } from './expressions.mjs'; +import { + loadManifest, + MANIFEST_PATH, + parseManifest, + readsRepository, + SPEC_FILE, + withGuards, +} from './guard-specs.mjs'; + +const scriptsDir = fileURLToPath(new URL('.', import.meta.url)); +const repoRoot = fileURLToPath(new URL('../..', import.meta.url)); +const manifest = loadManifest(); + +const WORKFLOWS = ['backend-review.yml', 'frontend-review.yml']; +const workflows = new Map( + WORKFLOWS.map((file) => { + const workflow = yaml.load(readFileSync(join(repoRoot, '.github/workflows', file), 'utf8')); + const selector = workflow.jobs.codegraph_select.steps.find((step) => step.id === 'sel'); + return [file, { workflow, selector }]; + }), +); + +/** `emit [ignore]` is how a selector names each workspace it gates. */ +const gated = new Map( + WORKFLOWS.flatMap((file) => + Array.from( + workflows.get(file).selector.run.matchAll(/^\s*emit \w+ (\S+)(?: (?:'[^']*'|""))?$/gm), + ([, workspace]) => [workspace, { file }], + ), + ), +); + +const SKIPPED_DIRECTORIES = new Set(['node_modules', 'dist', 'build', 'coverage', '.turbo']); + +function specsUnder(directory) { + return readdirSync(join(repoRoot, directory), { withFileTypes: true }).flatMap((entry) => { + const path = `${directory}/${entry.name}`; + if (entry.isDirectory()) return SKIPPED_DIRECTORIES.has(entry.name) ? [] : specsUnder(path); + return SPEC_FILE.test(entry.name) ? [path] : []; + }); +} + +const workspaceOf = (spec) => + [...gated.keys()].find((workspace) => spec.startsWith(`${workspace}/`)); + +/** GitHub's path filter, for the shapes these triggers use: exact paths and `dir/**`. */ +const triggers = (pattern, path) => + pattern.endsWith('/**') ? path.startsWith(pattern.slice(0, -2)) : pattern === path; + +test('both selectors gate the workspaces this test walks', () => { + assert.deepEqual( + [...gated].map(([workspace, { file }]) => [workspace, file]), + [ + ['api', 'backend-review.yml'], + ['packages/api', 'backend-review.yml'], + ['packages/data-provider', 'backend-review.yml'], + ['packages/data-schemas', 'backend-review.yml'], + ['client', 'frontend-review.yml'], + ['packages/client', 'frontend-review.yml'], + ], + ); +}); + +test('only a pull request synchronize is narrowed; every other event runs each workspace FULL', () => { + const event = (event_name, action) => ({ github: { event_name, event: { action } }, vars: {} }); + const unselected = { build: { result: 'success' }, codegraph_select: { outputs: {} } }; + for (const [file, { workflow }] of workflows) { + const selects = (context) => condition(workflow.jobs.codegraph_select.if, context); + assert.equal(selects(event('pull_request', 'synchronize')), true, file); + for (const context of [ + event('pull_request', 'opened'), + event('pull_request', 'reopened'), + event('push'), + event('workflow_dispatch'), + event('merge_group', 'checks_requested'), + ]) { + assert.equal(selects(context), false, `${file} ${context.github.event_name}`); + } + const gatedJobs = Object.values(workflow.jobs).filter((job) => + [job.needs].flat().includes('codegraph_select'), + ); + assert.ok(gatedJobs.length > 0, file); + for (const job of gatedJobs) { + assert.equal( + condition(job.if, { needs: unselected }, { cancelled: () => false }), + true, + `${file} ${job.name}`, + ); + } + } +}); + +test('every spec that reads the repository through fs is listed as a guard or exempt', () => { + const listed = new Set([...Object.keys(manifest.guards), ...Object.keys(manifest.exempt)]); + const unlisted = [...gated.keys()] + .flatMap(specsUnder) + .filter((spec) => !listed.has(spec)) + .filter((spec) => readsRepository(readFileSync(join(repoRoot, spec), 'utf8'))); + assert.deepEqual( + unlisted, + [], + `These specs read files with fs from a path anchored in the repository, so codegraph test ` + + `selection cannot tell when a change affects them. Add each to "guards" in ` + + `${MANIFEST_PATH} with the repository paths it reads, or to "exempt" with the reason it ` + + `reads no tracked file.`, + ); +}); + +test('every listed spec exists, sits in a gated workspace and reads paths that exist', () => { + for (const spec of [...Object.keys(manifest.guards), ...Object.keys(manifest.exempt)]) { + assert.ok(existsSync(join(repoRoot, spec)), `${spec} is listed but does not exist`); + assert.ok(workspaceOf(spec), `${spec} is not in a workspace that test selection gates`); + } + for (const [spec, scans] of Object.entries(manifest.guards)) { + for (const scan of scans) { + const target = join(repoRoot, scan); + assert.ok(existsSync(target), `${spec} lists ${scan}, which does not exist`); + assert.equal( + statSync(target).isDirectory(), + scan.endsWith('/'), + `${spec} lists ${scan}: a directory ends with a slash and a file does not`, + ); + } + } +}); + +test('a workflow runs whenever a path one of its guard specs reads changes', () => { + for (const [spec, scans] of Object.entries(manifest.guards)) { + const { file } = gated.get(workspaceOf(spec)); + const { workflow } = workflows.get(file); + for (const event of ['pull_request', 'push']) { + const patterns = workflow.on[event].paths.filter((pattern) => !pattern.startsWith('!')); + for (const scan of scans) { + assert.ok( + patterns.some((pattern) => triggers(pattern, scan)), + `${spec} reads ${scan}, but ${file} does not run on ${event} when it changes; ` + + `add it to that trigger's paths`, + ); + } + } + } +}); + +test('the detector flags repository reads and ignores temp-only and import-only specs', () => { + assert.ok(readsRepository("fs.readFileSync(path.join(__dirname, '../index.js'), 'utf8')")); + assert.ok(readsRepository('for (const e of readdirSync(join(process.cwd(), "src"))) {}')); + assert.ok(readsRepository('await readFile(new URL("./a.css", import.meta.url))')); + assert.ok(readsRepository("globSync('src/**/*.ts', { cwd: __dirname })")); + assert.ok(!readsRepository("fs.readFileSync(path.join(os.tmpdir(), 'out.txt'))")); + assert.ok(!readsRepository("const { run } = require(path.join(__dirname, 'run.js'))")); + assert.ok(!readsRepository("jest.mock('fs'); expect(reader.readFile).toBeDefined()")); +}); + +test('a manifest is refused when a path could split, glob or leave the repository', () => { + const guards = (entries) => JSON.stringify({ guards: entries }); + assert.throws(() => parseManifest('{}')); + assert.throws(() => parseManifest(guards({ 'api/a b.spec.js': ['api/'] }))); + assert.throws(() => parseManifest(guards({ 'api/*.spec.js': ['api/'] }))); + assert.throws(() => parseManifest(guards({ 'api/$(id).spec.js': ['api/'] }))); + assert.throws(() => parseManifest(guards({ 'api/../a.spec.js': ['api/'] }))); + assert.throws(() => parseManifest(guards({ '/api/a.spec.js': ['api/'] }))); + assert.throws(() => parseManifest(guards({ 'api/a.js': ['api/'] }))); + assert.throws(() => parseManifest(guards({ 'api/a.spec.js': [] }))); + assert.throws(() => parseManifest(guards({ 'api/a.spec.js': ['api/ server/'] }))); + assert.throws(() => parseManifest(guards({ 'api/a.spec.js': 'api/' }))); + assert.throws(() => + parseManifest( + JSON.stringify({ guards: { 'api/a.spec.js': ['api/'] }, exempt: { 'api/a.spec.js': 'x' } }), + ), + ); + assert.throws(() => + parseManifest(JSON.stringify({ guards: {}, exempt: { 'api/a.spec.js': '' } })), + ); + assert.deepEqual(parseManifest(guards({ 'api/a.spec.js': ['api/', 'b.json'] })).guards, { + 'api/a.spec.js': ['api/', 'b.json'], + }); +}); + +const FIXTURE = { + guards: { + 'api/server/guard.spec.js': ['api/server/', 'packages/api/src/'], + 'packages/api/src/guard.spec.ts': ['packages/api/src/'], + 'packages/api/src/scan.integration.spec.ts': ['packages/api/src/'], + 'packages/data-schemas/src/guard.spec.ts': ['api/'], + 'client/src/guard.spec.ts': ['client/index.html', 'packages/chat/src/'], + 'packages/client/src/guard.spec.tsx': ['packages/client/src/theme/'], + }, +}; + +test('a rename, or a change to the list or its reader, counts for every guard', () => { + const run = (changed) => + withGuards({ + manifest: FIXTURE, + mode: 'NONE', + workspace: 'packages/client', + changed, + exists: () => true, + }); + assert.deepEqual(run([{ path: 'docs/a.txt', status: 'modified' }]), []); + assert.deepEqual(run([{ path: 'docs/b.txt', status: 'renamed' }]), ['src/guard.spec.tsx']); + assert.deepEqual(run([{ path: MANIFEST_PATH, status: 'modified' }]), ['src/guard.spec.tsx']); + assert.deepEqual(run([{ path: '.github/scripts/guard-specs.mjs' }]), ['src/guard.spec.tsx']); + assert.deepEqual(run([{ path: 'packages/client/src/guard.spec.tsx' }]), ['src/guard.spec.tsx']); + assert.deepEqual(run([{ path: 'packages/client/src/theme/a.css' }]), ['src/guard.spec.tsx']); + assert.deepEqual(run([{ path: 'packages/client/src/themes.ts' }]), []); + assert.throws(() => run([{ filename: 'a' }])); + assert.throws(() => run({})); +}); + +test('a selection keeps its guards only while one of its files is in the checkout', () => { + const selected = ['src/gone.spec.tsx', 'src/kept.spec.tsx']; + const run = (exists) => + withGuards({ + manifest: FIXTURE, + mode: 'FILES', + workspace: 'packages/client', + changed: [], + selected, + exists, + }); + assert.deepEqual( + run((file) => file === 'src/kept.spec.tsx'), + [...selected, 'src/guard.spec.tsx'], + ); + assert.deepEqual( + run(() => false), + selected, + ); + assert.throws(() => run(undefined)); +}); + +const mode = (value, files) => ({ mode: value, ...(files ? { files } : {}) }); +const NONE = mode('NONE'); +const FULL = mode('FULL'); + +/** + * Runs a workflow's real select step against a canned codegraph response. `gh` and `curl` are + * stubs on PATH; the guard spec reader is the real script beside a fixture list. Every selected + * file exists in the scratch checkout unless `stale` says the selection outlived its files. + */ +function select(t, file, { selected, changed, list = FIXTURE, env = {}, stale = false }) { + const { selector } = workflows.get(file); + const scratch = mkdtempSync(join(scriptsDir, '.guard-specs-test-')); + t.after(() => rmSync(scratch, { recursive: true, force: true })); + const write = (name, content, options) => { + mkdirSync(dirname(join(scratch, name)), { recursive: true }); + writeFileSync(join(scratch, name), content, options); + }; + write('response.json', JSON.stringify({ selected })); + write('changed.ndjson', changed.map((entry) => JSON.stringify(entry)).join('\n') + '\n'); + write('output', ''); + write('selector.sh', selector.run); + write('gh', '#!/usr/bin/env bash\ncat changed.ndjson\nexit "${GH_EXIT:-0}"\n', { mode: 0o700 }); + write('curl', '#!/usr/bin/env bash\ncat response.json\nexit "${CURL_EXIT:-0}"\n', { + mode: 0o700, + }); + if (list !== null) { + write(MANIFEST_PATH, typeof list === 'string' ? list : JSON.stringify(list)); + } + if (!stale) { + Object.values(selected) + .flatMap((decision) => decision.files ?? []) + .forEach((path) => write(path, '')); + } + mkdirSync(join(scratch, '.github/scripts'), { recursive: true }); + copyFileSync( + join(scriptsDir, 'guard-specs.mjs'), + join(scratch, '.github/scripts/guard-specs.mjs'), + ); + const result = spawnSync('bash', ['selector.sh'], { + cwd: scratch, + encoding: 'utf8', + timeout: 20000, + env: { + PATH: `${scratch}:${process.env.PATH}`, + URL: 'https://selector.invalid', + TOKEN: 'test-fixture', + REPO: 'fixture/repo', + PR: '1', + BASE_SHA: 'base', + HEAD_SHA: 'head', + CHANGED: String(changed.length), + GITHUB_OUTPUT: join(scratch, 'output'), + GITHUB_STEP_SUMMARY: join(scratch, 'summary'), + ...env, + }, + }); + assert.equal(result.status, 0, result.stdout + result.stderr); + return Object.fromEntries( + readFileSync(join(scratch, 'output'), 'utf8') + .split('\n') + .filter(Boolean) + .map((line) => { + const separator = line.indexOf('='); + return [line.slice(0, separator), line.slice(separator + 1)]; + }), + ); +} + +const modified = (...paths) => paths.map((path) => ({ path, status: 'modified' })); + +test('backend: a narrowed workspace runs its guards with the selection (the #17864 shape)', (t) => { + const outputs = select(t, 'backend-review.yml', { + changed: modified('packages/api/src/agents/tools.ts'), + selected: { + api: mode('FILES', ['api/server/services/ToolService.spec.js']), + 'packages/api': mode('FILES', [ + 'packages/api/src/agents/tools.spec.ts', + 'packages/api/src/guard.spec.ts', + ]), + 'packages/data-provider': NONE, + 'packages/data-schemas': NONE, + }, + }); + assert.deepEqual(outputs, { + api_run: 'true', + api_files: 'server/services/ToolService.spec.js server/guard.spec.js', + pkgapi_run: 'true', + pkgapi_files: 'src/agents/tools.spec.ts src/guard.spec.ts', + dataprovider_run: 'false', + dataschemas_run: 'false', + decided: 'true', + }); +}); + +test('backend: a skipped workspace runs its guards alone when the change touches what they read', (t) => { + const outputs = select(t, 'backend-review.yml', { + changed: modified('api/server/routes/new.js'), + selected: { + api: NONE, + 'packages/api': NONE, + 'packages/data-provider': NONE, + 'packages/data-schemas': NONE, + }, + }); + assert.deepEqual(outputs, { + api_run: 'true', + api_files: 'server/guard.spec.js', + pkgapi_run: 'false', + dataprovider_run: 'false', + dataschemas_run: 'true', + dataschemas_files: 'src/guard.spec.ts', + decided: 'true', + }); +}); + +test('backend: a skip stands when nothing a guard reads changed, and FULL is left alone', (t) => { + const skipped = select(t, 'backend-review.yml', { + changed: modified('packages/client/src/components/Button.tsx'), + selected: { + api: NONE, + 'packages/api': NONE, + 'packages/data-provider': NONE, + 'packages/data-schemas': NONE, + }, + }); + assert.deepEqual(skipped, { + api_run: 'false', + pkgapi_run: 'false', + dataprovider_run: 'false', + dataschemas_run: 'false', + decided: 'true', + }); + const full = select(t, 'backend-review.yml', { + changed: modified('package-lock.json'), + selected: { + api: FULL, + 'packages/api': FULL, + 'packages/data-provider': FULL, + 'packages/data-schemas': FULL, + }, + }); + assert.deepEqual(full, { + api_run: 'true', + pkgapi_run: 'true', + dataprovider_run: 'true', + dataschemas_run: 'true', + decided: 'true', + }); +}); + +test('backend: guards obey the exclusions of the workspace jest run', (t) => { + const selected = { + api: NONE, + 'packages/api': mode('FILES', ['packages/api/src/agents/run.integration.spec.ts']), + 'packages/data-provider': NONE, + 'packages/data-schemas': NONE, + }; + const touched = select(t, 'backend-review.yml', { + changed: modified('packages/api/src/agents/run.ts'), + selected, + }); + assert.equal(touched.pkgapi_run, 'true'); + assert.equal(touched.pkgapi_files, 'src/guard.spec.ts'); + const untouched = select(t, 'backend-review.yml', { + changed: modified('packages/api/package.json'), + selected, + }); + assert.equal(untouched.pkgapi_run, 'false'); + assert.equal(untouched.pkgapi_files, undefined); +}); + +test('a selection whose files are all gone gets no guards, so the test job still runs FULL', (t) => { + const backend = select(t, 'backend-review.yml', { + changed: modified('api/server/routes/old.js'), + selected: { + api: mode('FILES', ['api/server/routes/old.spec.js']), + 'packages/api': FULL, + 'packages/data-provider': FULL, + 'packages/data-schemas': FULL, + }, + stale: true, + }); + assert.equal(backend.api_files, 'server/routes/old.spec.js'); + const frontend = select(t, 'frontend-review.yml', { + changed: modified('client/src/old.ts'), + selected: { client: mode('FILES', ['client/src/old.spec.ts']), 'packages/client': FULL }, + stale: true, + }); + assert.equal(frontend.client_files, 'src/old.spec.ts'); +}); + +test('backend: a guard list that cannot be read runs the narrowed workspaces FULL', (t) => { + const request = { + changed: modified('api/server/routes/new.js'), + selected: { + api: mode('FILES', ['api/server/routes/new.spec.js']), + 'packages/api': NONE, + 'packages/data-provider': FULL, + 'packages/data-schemas': NONE, + }, + }; + const unsafe = { guards: { 'api/server/a b.spec.js': ['api/'] } }; + for (const list of [null, '{', '{"guards": []}', unsafe]) { + const outputs = select(t, 'backend-review.yml', { ...request, list }); + assert.deepEqual(outputs, { + api_run: 'true', + pkgapi_run: 'true', + dataprovider_run: 'true', + dataschemas_run: 'true', + decided: 'true', + }); + } +}); + +test('backend: malformed decisions and an unavailable service still fail open', (t) => { + const changed = modified('api/server/routes/new.js'); + const rest = { + 'packages/api': FULL, + 'packages/data-provider': FULL, + 'packages/data-schemas': FULL, + }; + for (const api of [ + mode('FILES', []), + mode('FILES'), + mode('FILES', ['packages/api/src/misplaced.spec.ts']), + mode('FILES', ['api/server/has space.spec.js']), + mode('SOMETHING_NEW'), + ]) { + const outputs = select(t, 'backend-review.yml', { changed, selected: { api, ...rest } }); + assert.equal(outputs.api_run, 'true'); + assert.equal(outputs.api_files, undefined); + } + const selected = { api: NONE, ...rest }; + for (const env of [ + { URL: '' }, + { TOKEN: '' }, + { CURL_EXIT: '28' }, + { GH_EXIT: '1' }, + { CHANGED: '2' }, + ]) { + assert.deepEqual(select(t, 'backend-review.yml', { changed, selected, env }), {}); + } + assert.deepEqual(select(t, 'backend-review.yml', { changed, selected: {} }), {}); +}); + +test('frontend: guards join a narrowed run and promote a skip the change makes relevant', (t) => { + const narrowed = select(t, 'frontend-review.yml', { + changed: modified('client/src/components/Chat/Header.tsx'), + selected: { + client: mode('FILES', ['client/src/components/Chat/Header.spec.tsx']), + 'packages/client': NONE, + }, + }); + assert.deepEqual(narrowed, { + client_run: 'true', + client_files: 'src/components/Chat/Header.spec.tsx src/guard.spec.ts', + clientpkg_run: 'false', + decided: 'true', + }); + const promoted = select(t, 'frontend-review.yml', { + changed: modified('packages/chat/src/components/composer/Bar.tsx'), + selected: { client: NONE, 'packages/client': NONE }, + }); + assert.deepEqual(promoted, { + client_run: 'true', + client_files: 'src/guard.spec.ts', + clientpkg_run: 'false', + decided: 'true', + }); +}); + +test('frontend: an unreadable guard list, malformed decisions and outages fail open', (t) => { + const changed = modified('client/src/a.ts'); + const unreadable = select(t, 'frontend-review.yml', { + changed, + selected: { client: mode('FILES', ['client/src/a.spec.ts']), 'packages/client': NONE }, + list: null, + }); + assert.deepEqual(unreadable, { client_run: 'true', clientpkg_run: 'true', decided: 'true' }); + const malformed = select(t, 'frontend-review.yml', { + changed, + selected: { client: mode('FILES', []), 'packages/client': FULL }, + }); + assert.deepEqual(malformed, { client_run: 'true', clientpkg_run: 'true', decided: 'true' }); + const outage = select(t, 'frontend-review.yml', { + changed, + selected: { client: NONE, 'packages/client': NONE }, + env: { CURL_EXIT: '28' }, + }); + assert.deepEqual(outage, {}); +}); diff --git a/.github/scripts/playwright-matrix.test.mjs b/.github/scripts/playwright-matrix.test.mjs index ab7bc27e1d..69f2953bb8 100644 --- a/.github/scripts/playwright-matrix.test.mjs +++ b/.github/scripts/playwright-matrix.test.mjs @@ -5,6 +5,7 @@ import assert from 'node:assert/strict'; import { fileURLToPath } from 'node:url'; import { spawnSync } from 'node:child_process'; import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { condition, interpolate } from './expressions.mjs'; const scriptsDir = fileURLToPath(new URL('.', import.meta.url)); const workflow = yaml.load( @@ -85,6 +86,55 @@ function decision(redis, mcp = true) { return { matrix: { 'playwright-mock': { redis_transport: redis, mcp_tool_list_changed: mcp } } }; } +const REPOSITORY = 'LibreChat-AI/LibreChat'; +const event = (event_name, ref, payload = {}) => ({ + event_name, + ref, + repository: REPOSITORY, + event: payload, +}); +const pushToDev = event('push', 'refs/heads/dev'); +const nightly = event('schedule', 'refs/heads/main'); +const manual = event('workflow_dispatch', 'refs/heads/dev'); +const pullRequest = (action, association = 'MEMBER', head = REPOSITORY) => + event('pull_request', 'refs/pull/1/merge', { + action, + pull_request: { author_association: association, head: { repo: { full_name: head } } }, + }); + +/** What the workflow's own expressions resolve to for one event, before any job has run. */ +function resolve(github, { inputs = {}, select = {} } = {}) { + const context = { github, inputs, vars: {}, needs: { codegraph_select: { outputs: select } } }; + const functions = { cancelled: () => false }; + const runs = (job) => condition(workflow.jobs[job].if, context, functions); + return { + jobs: { + select: runs('codegraph_select'), + shards: runs('e2e_shards'), + mcp: runs('mcp_tool_list_changed'), + apps: runs('mcp_apps'), + gate: runs('e2e'), + }, + matrix: interpolate(shards.strategy.matrix, context, functions), + group: interpolate(workflow.concurrency.group, context, functions), + cancels: interpolate(workflow.concurrency['cancel-in-progress'], context, functions), + }; +} + +const everyLane = { select: true, shards: true, mcp: true, apps: true, gate: true }; + +/** True when the gate's only step runs, which is the step that fails the required check. */ +function gateFails(results, mcpRun = '') { + return condition(workflow.jobs.e2e.steps[0].if, { + needs: { + e2e_shards: { result: results.shards ?? 'success' }, + mcp_apps: { result: results.apps ?? 'success' }, + mcp_tool_list_changed: { result: results.mcp ?? 'success' }, + codegraph_select: { outputs: { mcp_run: mcpRun } }, + }, + }); +} + test('PR selection and fail-open defaults use ten memory shards and two Redis transport shards', () => { assert.equal(defaults.length, 2); assert.deepEqual(defaults[0], prMatrix); @@ -115,10 +165,8 @@ test('PR and nightly matrices preserve both registered-email change modes on one test('manual PR coverage opts into the PR fallback matrix without bypassing the PR author gate', () => { assert.deepEqual(workflow.on.workflow_dispatch.inputs.coverage.options, ['full', 'pr']); - assert.match( - shards.strategy.matrix, - /github\.event_name == 'pull_request' \|\| inputs\.coverage == 'pr'/, - ); + assert.deepEqual(resolve(manual, { inputs: { coverage: 'pr' } }).matrix, prMatrix); + assert.deepEqual(resolve(manual, { inputs: { coverage: 'full' } }).matrix, defaults[1]); assert.match(shards.if, /github\.event_name == 'workflow_dispatch'/); assert.match(shards.if, /github\.event\.pull_request\.author_association/); }); @@ -218,5 +266,69 @@ test('the required gate still waits for all shards and rejects failures without assert.match(gate.steps[0].if, /needs\.mcp_tool_list_changed\.result != 'success'/); assert.match(gate.steps[0].if, /needs\.codegraph_select\.outputs\.mcp_run == 'false'/); assert.equal(gate.steps[0].run, 'exit 1'); - assert.equal(workflow.concurrency['cancel-in-progress'], true); + assert.equal(resolve(pullRequest('synchronize')).cancels, true); +}); + +test('a push to dev runs every lane on the pull request matrix with nothing narrowed', () => { + assert.deepEqual(workflow.on.push.branches, ['dev']); + assert.deepEqual(workflow.on.push.paths, workflow.on.pull_request.paths); + const { jobs, matrix } = resolve(pushToDev); + assert.deepEqual(jobs, { ...everyLane, select: false }); + assert.deepEqual(matrix, prMatrix); + assert.equal(gateFails({}), false); + assert.equal(gateFails({ shards: 'failure' }), true); + assert.equal(gateFails({ apps: 'failure' }), true); + assert.equal(gateFails({ mcp: 'failure' }), true); + /** No selector ran on a push, so a skipped MCP lane is a fault there, not a decision. */ + assert.equal(gateFails({ mcp: 'skipped' }), true); + assert.equal(gateFails({ mcp: 'skipped' }, 'false'), false); +}); + +test('dev pushes never cancel each other and share no group with any other event', () => { + const pushed = resolve(pushToDev); + assert.equal(pushed.group, 'playwright-mock-post-merge-refs/heads/dev'); + assert.equal(pushed.cancels, false); + const others = [pullRequest('opened'), pullRequest('synchronize'), nightly, manual].map( + (github) => resolve(github), + ); + assert.deepEqual( + others.map(({ group }) => group), + [ + 'playwright-mock-refs/pull/1/merge', + 'playwright-mock-refs/pull/1/merge', + 'playwright-mock-refs/heads/main', + 'playwright-mock-refs/heads/dev', + ], + ); + assert.ok(others.every(({ cancels }) => cancels === true)); +}); + +test('pull request, nightly and manual runs resolve as they did without the push trigger', () => { + const opened = resolve(pullRequest('opened')); + assert.deepEqual(opened.jobs, { ...everyLane, select: false }); + assert.deepEqual(opened.matrix, prMatrix); + + const memoryOnly = { include: prMatrix.include.filter((lane) => lane.suite === 'full') }; + const narrowed = resolve(pullRequest('synchronize'), { + select: { e2e_include: JSON.stringify(memoryOnly), mcp_run: 'false' }, + }); + assert.deepEqual(narrowed.jobs, { ...everyLane, mcp: false }); + assert.deepEqual(narrowed.matrix, memoryOnly); + + const outsider = resolve(pullRequest('synchronize', 'CONTRIBUTOR', 'someone/LibreChat')); + assert.deepEqual(outsider.jobs, { + select: false, + shards: false, + mcp: false, + apps: false, + gate: false, + }); + const sameRepositoryBot = resolve(pullRequest('opened', 'NONE')); + assert.deepEqual(sameRepositoryBot.jobs, { ...everyLane, select: false }); + + for (const github of [nightly, manual]) { + const run = resolve(github); + assert.deepEqual(run.jobs, { ...everyLane, select: false }); + assert.deepEqual(run.matrix, defaults[1]); + } }); diff --git a/.github/workflows/backend-review.yml b/.github/workflows/backend-review.yml index 9d11d37419..f5f9a6a28a 100644 --- a/.github/workflows/backend-review.yml +++ b/.github/workflows/backend-review.yml @@ -15,6 +15,15 @@ on: - '.bun-version' - 'config/circular-deps.mjs' - '.github/workflows/backend-review.yml' + # The guard spec list, its reader and their test, then every path a backend guard spec + # reads outside the trees above; guard-specs.test.mjs fails when one is missing here. + - '.github/guard-specs.json' + - '.github/scripts/guard-specs*.mjs' + - '.env.example' + - 'librechat.example.yaml' + - 'client/public/mcp-sandbox.html' + - 'e2e/setup/fake-model.js' + - 'vendor/code/**' - '!**.md' pull_request: paths: @@ -25,6 +34,15 @@ on: - '.bun-version' - 'config/circular-deps.mjs' - '.github/workflows/backend-review.yml' + # The guard spec list, its reader and their test, then every path a backend guard spec + # reads outside the trees above; guard-specs.test.mjs fails when one is missing here. + - '.github/guard-specs.json' + - '.github/scripts/guard-specs*.mjs' + - '.env.example' + - 'librechat.example.yaml' + - 'client/public/mcp-sandbox.html' + - 'e2e/setup/fake-model.js' + - 'vendor/code/**' - '!**.md' # Promotion requires full CI at the exact dev SHA, including path-filtered commits. workflow_dispatch: @@ -140,6 +158,9 @@ jobs: # graph, unclassifiable change, root/lockfile floors => mode FULL per workspace), and this # job emits nothing unless the response parses end to end — the worst case at every layer # is "CI runs everything", which is the pre-gating behavior. + # + # The graph follows imports, so a spec that reads source with fs is invisible to it. Those + # guard specs are listed in .github/guard-specs.json and run with every narrowed decision. codegraph_select: name: Codegraph select runs-on: ubuntu-latest @@ -159,6 +180,14 @@ jobs: dataschemas_run: ${{ steps.sel.outputs.dataschemas_run }} dataschemas_files: ${{ steps.sel.outputs.dataschemas_files }} steps: + # The commit under test, for the guard spec list, its reader and the reader's check that + # a selection is not stale. Best effort: without it the select step runs every narrowed + # workspace FULL, so a failed checkout never fails this job. + - uses: actions/checkout@v5 + continue-on-error: true + with: + persist-credentials: false + - name: Select tests, fail open on any doubt id: sel env: @@ -227,6 +256,23 @@ jobs: if [ "$mode" = "FILES" ] && [ -z "$files" ]; then mode="NONE"; fi fi fi + # Guard specs read source with fs, so no import edge selects them. A narrowed run + # carries its workspace's guards, and a skip becomes a guards-only run when the + # change touches what they read (.github/guard-specs.json; the rules are in + # .github/scripts/guard-specs.mjs). A list that cannot be read runs FULL. A selection + # with no file left in this checkout gets no guards: the test job runs FULL on a + # stale selection, and guards that exist would turn that into a guards-only run. + guard_note="" + if [ "$mode" = "FILES" ] || [ "$mode" = "NONE" ]; then + guarded=$(SELECTED="$files" node .github/scripts/guard-specs.mjs "$mode" "$ws" files.json "$ignore"); grc=$? + if [ "$grc" -ne 0 ]; then + mode="FULL"; files="" + note "| $ws | guard specs unreadable (exit $grc); running FULL |" + elif [ -n "$guarded" ]; then + if [ "$mode" = "NONE" ]; then guard_note=", guard specs only"; fi + mode="FILES"; files="$guarded" + fi + fi if [ "$mode" = "NONE" ]; then echo "${key}_run=false" >> "$GITHUB_OUTPUT" note "| $ws | skip (no reachable tests) |" @@ -234,7 +280,7 @@ jobs: n=$(echo "$files" | wc -w | tr -d ' ') echo "${key}_run=true" >> "$GITHUB_OUTPUT" echo "${key}_files=$files" >> "$GITHUB_OUTPUT" - note "| $ws | $n selected files |" + note "| $ws | $n selected files$guard_note |" else echo "${key}_run=true" >> "$GITHUB_OUTPUT" note "| $ws | FULL |" @@ -357,6 +403,11 @@ jobs: - name: Detect circular dependencies run: node config/circular-deps.mjs + # Here because this job has node_modules, always runs and gates no other job. + - name: Test guard spec selection + if: ${{ !cancelled() }} + run: node --test .github/scripts/guard-specs.test.mjs + test-api: name: 'Tests: api (shard ${{ matrix.shard }}/3)' needs: [build, codegraph_select] diff --git a/.github/workflows/codegraph-e2e-votes.yml b/.github/workflows/codegraph-e2e-votes.yml index 4ce57125ef..d1e40e0e9d 100644 --- a/.github/workflows/codegraph-e2e-votes.yml +++ b/.github/workflows/codegraph-e2e-votes.yml @@ -1,9 +1,10 @@ # Codegraph e2e VOTES — observe-only, post-merge, time-boxed. # -# Playwright never runs on pushes to dev, so evidence for the e2e skip election would -# otherwise wait on rare organic PR spec failures. This workflow runs the FULL mock suite after -# merges: each run is one graduation trial for every spec it executes, and doubles as the -# post-merge safety net the jest workflows already have via their dev-push triggers. +# Evidence for the e2e skip election would otherwise wait on rare organic PR spec failures, so +# this workflow runs the FULL mock suite after merges: each run is one graduation trial for +# every spec it executes. It is not the post-merge safety net: its test step cannot fail (see +# below), so a regression on dev leaves it green. The run that goes red is playwright-mock.yml's +# own dev-push trigger. # # By default every merge votes (the push trigger). Each run costs about 154 job-minutes on ten # runners, so with repo variable CODEGRAPH_E2E_VOTE_MIN_HOURS set to N > 0 a push run votes diff --git a/.github/workflows/frontend-review.yml b/.github/workflows/frontend-review.yml index 5bddf5bd18..f5d2df96ef 100644 --- a/.github/workflows/frontend-review.yml +++ b/.github/workflows/frontend-review.yml @@ -13,6 +13,11 @@ on: - 'package-lock.json' - '.bun-version' - '.github/workflows/frontend-review.yml' + # The guard spec list, its reader and their test, then every path a frontend guard spec + # reads outside the trees above; guard-specs.test.mjs fails when one is missing here. + - '.github/guard-specs.json' + - '.github/scripts/guard-specs*.mjs' + - 'eslint-suppressions.json' - '!**.md' # Post-merge safety net and full-run baseline for gated selection (same rationale as # backend-review.yml stage 1): every dev merge touching frontend paths runs the full suite. @@ -30,6 +35,11 @@ on: - 'package-lock.json' - '.bun-version' - '.github/workflows/frontend-review.yml' + # The guard spec list, its reader and their test, then every path a frontend guard spec + # reads outside the trees above; guard-specs.test.mjs fails when one is missing here. + - '.github/guard-specs.json' + - '.github/scripts/guard-specs*.mjs' + - 'eslint-suppressions.json' # Promotion requires full CI at the exact dev SHA, including path-filtered commits. workflow_dispatch: @@ -102,6 +112,10 @@ jobs: - run: npm run test:client-build env: PLAYWRIGHT_CHANNEL: chrome + # Here because this job has node_modules, always runs and gates no other job. + - name: Test guard spec selection + if: ${{ !cancelled() }} + run: node --test .github/scripts/guard-specs.test.mjs # Stage 1.5 of codegraph gating (stage 1 = backend jest, #15132; stage 2 = matrix lanes, # #15136). Same mechanism, same record: across 604 finalized shadow receipts the frontend @@ -110,6 +124,9 @@ jobs: # every dev push, and on any doubt. A workspace skips ONLY on an explicit NONE. Kill switch: # repo variable CODEGRAPH_GATING=off. Neither frontend workspace defines # testPathIgnorePatterns, so --runTestsByPath needs no exclude mirroring here. + # + # The graph follows imports, so a spec that reads source with fs is invisible to it. Those + # guard specs are listed in .github/guard-specs.json and run with every narrowed decision. codegraph_select: name: Codegraph select runs-on: ubuntu-latest @@ -125,6 +142,14 @@ jobs: clientpkg_run: ${{ steps.sel.outputs.clientpkg_run }} clientpkg_files: ${{ steps.sel.outputs.clientpkg_files }} steps: + # The commit under test, for the guard spec list, its reader and the reader's check that + # a selection is not stale. Best effort: without it the select step runs every narrowed + # workspace FULL, so a failed checkout never fails this job. + - uses: actions/checkout@v5 + continue-on-error: true + with: + persist-credentials: false + - name: Select tests, fail open on any doubt id: sel env: @@ -180,6 +205,23 @@ jobs: if [ "$spaced" != "0" ]; then mode="FULL"; files=""; fi fi fi + # Guard specs read source with fs, so no import edge selects them. A narrowed run + # carries its workspace's guards, and a skip becomes a guards-only run when the + # change touches what they read (.github/guard-specs.json; the rules are in + # .github/scripts/guard-specs.mjs). A list that cannot be read runs FULL. A selection + # with no file left in this checkout gets no guards: the test job runs FULL on a + # stale selection, and guards that exist would turn that into a guards-only run. + guard_note="" + if [ "$mode" = "FILES" ] || [ "$mode" = "NONE" ]; then + guarded=$(SELECTED="$files" node .github/scripts/guard-specs.mjs "$mode" "$ws" files.json ""); grc=$? + if [ "$grc" -ne 0 ]; then + mode="FULL"; files="" + note "| $ws | guard specs unreadable (exit $grc); running FULL |" + elif [ -n "$guarded" ]; then + if [ "$mode" = "NONE" ]; then guard_note=", guard specs only"; fi + mode="FILES"; files="$guarded" + fi + fi if [ "$mode" = "NONE" ]; then echo "${key}_run=false" >> "$GITHUB_OUTPUT" note "| $ws | skip (no reachable tests) |" @@ -187,7 +229,7 @@ jobs: n=$(echo "$files" | wc -w | tr -d ' ') echo "${key}_run=true" >> "$GITHUB_OUTPUT" echo "${key}_files=$files" >> "$GITHUB_OUTPUT" - note "| $ws | $n selected files |" + note "| $ws | $n selected files$guard_note |" else echo "${key}_run=true" >> "$GITHUB_OUTPUT" note "| $ws | FULL |" diff --git a/.github/workflows/playwright-mock.yml b/.github/workflows/playwright-mock.yml index d071052ec6..13a3c64af3 100644 --- a/.github/workflows/playwright-mock.yml +++ b/.github/workflows/playwright-mock.yml @@ -7,6 +7,19 @@ on: - '!**.md' - '!.github/workflows/**' - '.github/workflows/playwright-mock.yml' + # Post-merge safety net, as backend-review.yml and frontend-review.yml have for jest. A + # regression that reaches dev fails this suite on every open pull request, and without a run + # on dev itself nothing names the merge that caused it. A push runs the pull request profile + # with no lane narrowed, so a red run on dev is what the next pull request would see. The + # nightly stays on the default branch, where it covers the full suite on both stores. + push: + branches: + - dev + paths: + - '**' + - '!**.md' + - '!.github/workflows/**' + - '.github/workflows/playwright-mock.yml' schedule: - cron: '0 5 * * *' workflow_dispatch: @@ -27,8 +40,16 @@ permissions: contents: read concurrency: - group: playwright-mock-${{ github.ref }} - cancel-in-progress: true + # Pull requests, the nightly and manual runs keep a per-ref group in which a newer run cancels + # the older one. Pushes to dev take a group of their own that never cancels, so the run in + # progress always finishes; GitHub holds one pending run per group and a newer one replaces + # it, so merges that land meanwhile collapse into a single run of the newest commit. A + # cancelling group would rarely finish on a busy day (nine merges landed within 89 seconds on + # 2026-11-09) and a run per commit, which the jest workflows use, costs about 190 + # runner-minutes on 15 runners each time. A red run therefore names a range of merges, from + # the commit of the last finished run to its own, and the dev tip always gets a run. + group: playwright-mock-${{ github.event_name == 'push' && 'post-merge-' || '' }}${{ github.ref }} + cancel-in-progress: ${{ github.event_name != 'push' }} env: NODE_OPTIONS: '--max-old-space-size=${{ secrets.NODE_MAX_OLD_SPACE_SIZE || 6144 }}' @@ -39,9 +60,9 @@ jobs: # two matrix lanes whose relevance is a reachability question: the redis-transport shards (the # stream-boundary specs under a real Redis round-trip) and the MCP list_changed lanes. Memory # shards always run. Monotone and fail-open: a lane is dropped ONLY on an explicit `false` from - # the service; unavailable/unconfigured/non-synchronize events keep the full matrix. Lock - # attribution rides along so a backend dependency bump (reaches nothing in the graph) still - # earns both lanes. Kill switch: repo variable CODEGRAPH_GATING=off. + # the service; unavailable/unconfigured/non-synchronize events (dev pushes among them) keep the + # full matrix. Lock attribution rides along so a backend dependency bump (reaches nothing in + # the graph) still earns both lanes. Kill switch: repo variable CODEGRAPH_GATING=off. codegraph_select: name: Codegraph select runs-on: ubuntu-latest @@ -153,7 +174,7 @@ jobs: fi echo "decided=true" >> "$GITHUB_OUTPUT" note "" - note "memory shards always run · kill switch: repo variable \`CODEGRAPH_GATING=off\` · full matrix on PR open and nightly" + note "memory shards always run · kill switch: repo variable \`CODEGRAPH_GATING=off\` · full matrix on PR open, on every dev push and nightly" exit 0 e2e_shards: @@ -163,6 +184,7 @@ jobs: !cancelled() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || + github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request != null && (contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) || @@ -177,7 +199,7 @@ jobs: ${{ (github.event_name == 'pull_request' && needs.codegraph_select.outputs.e2e_include != '' && fromJSON(needs.codegraph_select.outputs.e2e_include)) || - ((github.event_name == 'pull_request' || inputs.coverage == 'pr') && + ((github.event_name == 'pull_request' || github.event_name == 'push' || inputs.coverage == 'pr') && fromJSON('{"include":[{"name":"memory, shard 1/10","stream_store":"memory","redis_image":"","suite":"full","shard":"1/10","artifact":"memory-1-of-10","email_change":"true"},{"name":"memory, shard 2/10","stream_store":"memory","redis_image":"","suite":"full","shard":"2/10","artifact":"memory-2-of-10"},{"name":"memory, shard 3/10","stream_store":"memory","redis_image":"","suite":"full","shard":"3/10","artifact":"memory-3-of-10"},{"name":"memory, shard 4/10","stream_store":"memory","redis_image":"","suite":"full","shard":"4/10","artifact":"memory-4-of-10"},{"name":"memory, shard 5/10","stream_store":"memory","redis_image":"","suite":"full","shard":"5/10","artifact":"memory-5-of-10"},{"name":"memory, shard 6/10","stream_store":"memory","redis_image":"","suite":"full","shard":"6/10","artifact":"memory-6-of-10"},{"name":"memory, shard 7/10","stream_store":"memory","redis_image":"","suite":"full","shard":"7/10","artifact":"memory-7-of-10"},{"name":"memory, shard 8/10","stream_store":"memory","redis_image":"","suite":"full","shard":"8/10","artifact":"memory-8-of-10"},{"name":"memory, shard 9/10","stream_store":"memory","redis_image":"","suite":"full","shard":"9/10","artifact":"memory-9-of-10"},{"name":"memory, shard 10/10","stream_store":"memory","redis_image":"","suite":"full","shard":"10/10","artifact":"memory-10-of-10"},{"name":"redis transport, shard 1/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"1/2","artifact":"redis-transport-1-of-2"},{"name":"redis transport, shard 2/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"2/2","artifact":"redis-transport-2-of-2"}]}')) || fromJSON('{"include":[{"name":"memory, shard 1/10","stream_store":"memory","redis_image":"","suite":"full","shard":"1/10","artifact":"memory-1-of-10","email_change":"true"},{"name":"memory, shard 2/10","stream_store":"memory","redis_image":"","suite":"full","shard":"2/10","artifact":"memory-2-of-10"},{"name":"memory, shard 3/10","stream_store":"memory","redis_image":"","suite":"full","shard":"3/10","artifact":"memory-3-of-10"},{"name":"memory, shard 4/10","stream_store":"memory","redis_image":"","suite":"full","shard":"4/10","artifact":"memory-4-of-10"},{"name":"memory, shard 5/10","stream_store":"memory","redis_image":"","suite":"full","shard":"5/10","artifact":"memory-5-of-10"},{"name":"memory, shard 6/10","stream_store":"memory","redis_image":"","suite":"full","shard":"6/10","artifact":"memory-6-of-10"},{"name":"memory, shard 7/10","stream_store":"memory","redis_image":"","suite":"full","shard":"7/10","artifact":"memory-7-of-10"},{"name":"memory, shard 8/10","stream_store":"memory","redis_image":"","suite":"full","shard":"8/10","artifact":"memory-8-of-10"},{"name":"memory, shard 9/10","stream_store":"memory","redis_image":"","suite":"full","shard":"9/10","artifact":"memory-9-of-10"},{"name":"memory, shard 10/10","stream_store":"memory","redis_image":"","suite":"full","shard":"10/10","artifact":"memory-10-of-10"},{"name":"redis, shard 1/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"1/10","artifact":"redis-1-of-10"},{"name":"redis, shard 2/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"2/10","artifact":"redis-2-of-10"},{"name":"redis, shard 3/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"3/10","artifact":"redis-3-of-10"},{"name":"redis, shard 4/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"4/10","artifact":"redis-4-of-10"},{"name":"redis, shard 5/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"5/10","artifact":"redis-5-of-10"},{"name":"redis, shard 6/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"6/10","artifact":"redis-6-of-10"},{"name":"redis, shard 7/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"7/10","artifact":"redis-7-of-10"},{"name":"redis, shard 8/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"8/10","artifact":"redis-8-of-10"},{"name":"redis, shard 9/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"9/10","artifact":"redis-9-of-10"},{"name":"redis, shard 10/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"10/10","artifact":"redis-10-of-10"}]}') }} @@ -438,6 +460,7 @@ jobs: needs.codegraph_select.outputs.mcp_run != 'false' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || + github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request != null && (contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) || @@ -588,11 +611,12 @@ jobs: mcp_apps: name: MCP Apps (browser and server) - # Always run the dedicated fixture on eligible PRs, schedule, and manual runs. The regular - # mock shards skip it unless E2E_MCP_APPS=true; codegraph gating must not omit this lane. + # Always run the dedicated fixture on eligible PRs, dev pushes, schedule, and manual runs. The + # regular mock shards skip it unless E2E_MCP_APPS=true; codegraph gating must not omit this lane. if: >- github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || + github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request != null && (contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) || @@ -675,6 +699,7 @@ jobs: !cancelled() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || + github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request != null && (contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) ||