mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-21 23:43:35 +00:00
🥅 ci: Run Playwright After Merges to Dev and Guard Specs Under Test Selection (#18049)
* 🥅 ci: Run Playwright After Every Merge to Dev The mock e2e suite ran on pull requests, nightly on main and on demand, so when a spec broke on dev no run on dev could go red: the vote workflow that runs the suite after merges cannot fail by design. Add a push trigger for dev with the pull request path filter. A push runs the pull request profile with no lane narrowed (10 memory shards, 2 Redis transport shards, both MCP lanes and MCP Apps), and the gate job reports it. Pushes take their own concurrency group that never cancels: the run in progress finishes, and GitHub keeps one pending run per group, so merges that land meanwhile collapse into one run of the newest commit. Pull request, nightly and manual runs keep their per-ref cancelling group. The matrix test now evaluates the workflow's own expressions per event rather than matching their text. * 🔭 ci: Always Run Source-Scanning Guard Specs Under Test Selection Codegraph test selection narrows a pull request run to the test files an import graph reaches. A spec that reads source with fs has no import edge to what it reads: toolCapabilityGates.spec.js scans four source trees, was not selected when #17864 changed one of the files it counts, and failed on dev afterwards. List every such spec, with the paths it reads, in .github/guard-specs.json. Both selectors now add a workspace's guards to a FILES decision and turn a NONE into a guards-only run when the change touches a path one of them reads, renames a file, or edits the list. A list that cannot be read runs the workspace FULL. FULL decisions and every earlier fail-open path are unchanged. The rule keys off the decision, not the event. A selection with no file left in the checkout gets no guards, so the test job still runs FULL on a stale selection instead of running only the guards. guard-specs.test.mjs fails when a spec in a gated workspace reads the repository through fs without being listed, when a listed path is missing, and when a workflow would not run for a path one of its guards reads. That last check adds the five backend and one frontend paths the triggers lacked. The test also runs both real select steps against canned codegraph responses.
This commit is contained in:
parent
1469f5ad75
commit
97f7fcdbac
10 changed files with 1229 additions and 23 deletions
122
.github/scripts/playwright-matrix.test.mjs
vendored
122
.github/scripts/playwright-matrix.test.mjs
vendored
|
|
@ -5,6 +5,7 @@ import assert from 'node:assert/strict';
|
|||
import { fileURLToPath } from 'node:url';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { condition, interpolate } from './expressions.mjs';
|
||||
|
||||
const scriptsDir = fileURLToPath(new URL('.', import.meta.url));
|
||||
const workflow = yaml.load(
|
||||
|
|
@ -85,6 +86,55 @@ function decision(redis, mcp = true) {
|
|||
return { matrix: { 'playwright-mock': { redis_transport: redis, mcp_tool_list_changed: mcp } } };
|
||||
}
|
||||
|
||||
const REPOSITORY = 'LibreChat-AI/LibreChat';
|
||||
const event = (event_name, ref, payload = {}) => ({
|
||||
event_name,
|
||||
ref,
|
||||
repository: REPOSITORY,
|
||||
event: payload,
|
||||
});
|
||||
const pushToDev = event('push', 'refs/heads/dev');
|
||||
const nightly = event('schedule', 'refs/heads/main');
|
||||
const manual = event('workflow_dispatch', 'refs/heads/dev');
|
||||
const pullRequest = (action, association = 'MEMBER', head = REPOSITORY) =>
|
||||
event('pull_request', 'refs/pull/1/merge', {
|
||||
action,
|
||||
pull_request: { author_association: association, head: { repo: { full_name: head } } },
|
||||
});
|
||||
|
||||
/** What the workflow's own expressions resolve to for one event, before any job has run. */
|
||||
function resolve(github, { inputs = {}, select = {} } = {}) {
|
||||
const context = { github, inputs, vars: {}, needs: { codegraph_select: { outputs: select } } };
|
||||
const functions = { cancelled: () => false };
|
||||
const runs = (job) => condition(workflow.jobs[job].if, context, functions);
|
||||
return {
|
||||
jobs: {
|
||||
select: runs('codegraph_select'),
|
||||
shards: runs('e2e_shards'),
|
||||
mcp: runs('mcp_tool_list_changed'),
|
||||
apps: runs('mcp_apps'),
|
||||
gate: runs('e2e'),
|
||||
},
|
||||
matrix: interpolate(shards.strategy.matrix, context, functions),
|
||||
group: interpolate(workflow.concurrency.group, context, functions),
|
||||
cancels: interpolate(workflow.concurrency['cancel-in-progress'], context, functions),
|
||||
};
|
||||
}
|
||||
|
||||
const everyLane = { select: true, shards: true, mcp: true, apps: true, gate: true };
|
||||
|
||||
/** True when the gate's only step runs, which is the step that fails the required check. */
|
||||
function gateFails(results, mcpRun = '') {
|
||||
return condition(workflow.jobs.e2e.steps[0].if, {
|
||||
needs: {
|
||||
e2e_shards: { result: results.shards ?? 'success' },
|
||||
mcp_apps: { result: results.apps ?? 'success' },
|
||||
mcp_tool_list_changed: { result: results.mcp ?? 'success' },
|
||||
codegraph_select: { outputs: { mcp_run: mcpRun } },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
test('PR selection and fail-open defaults use ten memory shards and two Redis transport shards', () => {
|
||||
assert.equal(defaults.length, 2);
|
||||
assert.deepEqual(defaults[0], prMatrix);
|
||||
|
|
@ -115,10 +165,8 @@ test('PR and nightly matrices preserve both registered-email change modes on one
|
|||
|
||||
test('manual PR coverage opts into the PR fallback matrix without bypassing the PR author gate', () => {
|
||||
assert.deepEqual(workflow.on.workflow_dispatch.inputs.coverage.options, ['full', 'pr']);
|
||||
assert.match(
|
||||
shards.strategy.matrix,
|
||||
/github\.event_name == 'pull_request' \|\| inputs\.coverage == 'pr'/,
|
||||
);
|
||||
assert.deepEqual(resolve(manual, { inputs: { coverage: 'pr' } }).matrix, prMatrix);
|
||||
assert.deepEqual(resolve(manual, { inputs: { coverage: 'full' } }).matrix, defaults[1]);
|
||||
assert.match(shards.if, /github\.event_name == 'workflow_dispatch'/);
|
||||
assert.match(shards.if, /github\.event\.pull_request\.author_association/);
|
||||
});
|
||||
|
|
@ -218,5 +266,69 @@ test('the required gate still waits for all shards and rejects failures without
|
|||
assert.match(gate.steps[0].if, /needs\.mcp_tool_list_changed\.result != 'success'/);
|
||||
assert.match(gate.steps[0].if, /needs\.codegraph_select\.outputs\.mcp_run == 'false'/);
|
||||
assert.equal(gate.steps[0].run, 'exit 1');
|
||||
assert.equal(workflow.concurrency['cancel-in-progress'], true);
|
||||
assert.equal(resolve(pullRequest('synchronize')).cancels, true);
|
||||
});
|
||||
|
||||
test('a push to dev runs every lane on the pull request matrix with nothing narrowed', () => {
|
||||
assert.deepEqual(workflow.on.push.branches, ['dev']);
|
||||
assert.deepEqual(workflow.on.push.paths, workflow.on.pull_request.paths);
|
||||
const { jobs, matrix } = resolve(pushToDev);
|
||||
assert.deepEqual(jobs, { ...everyLane, select: false });
|
||||
assert.deepEqual(matrix, prMatrix);
|
||||
assert.equal(gateFails({}), false);
|
||||
assert.equal(gateFails({ shards: 'failure' }), true);
|
||||
assert.equal(gateFails({ apps: 'failure' }), true);
|
||||
assert.equal(gateFails({ mcp: 'failure' }), true);
|
||||
/** No selector ran on a push, so a skipped MCP lane is a fault there, not a decision. */
|
||||
assert.equal(gateFails({ mcp: 'skipped' }), true);
|
||||
assert.equal(gateFails({ mcp: 'skipped' }, 'false'), false);
|
||||
});
|
||||
|
||||
test('dev pushes never cancel each other and share no group with any other event', () => {
|
||||
const pushed = resolve(pushToDev);
|
||||
assert.equal(pushed.group, 'playwright-mock-post-merge-refs/heads/dev');
|
||||
assert.equal(pushed.cancels, false);
|
||||
const others = [pullRequest('opened'), pullRequest('synchronize'), nightly, manual].map(
|
||||
(github) => resolve(github),
|
||||
);
|
||||
assert.deepEqual(
|
||||
others.map(({ group }) => group),
|
||||
[
|
||||
'playwright-mock-refs/pull/1/merge',
|
||||
'playwright-mock-refs/pull/1/merge',
|
||||
'playwright-mock-refs/heads/main',
|
||||
'playwright-mock-refs/heads/dev',
|
||||
],
|
||||
);
|
||||
assert.ok(others.every(({ cancels }) => cancels === true));
|
||||
});
|
||||
|
||||
test('pull request, nightly and manual runs resolve as they did without the push trigger', () => {
|
||||
const opened = resolve(pullRequest('opened'));
|
||||
assert.deepEqual(opened.jobs, { ...everyLane, select: false });
|
||||
assert.deepEqual(opened.matrix, prMatrix);
|
||||
|
||||
const memoryOnly = { include: prMatrix.include.filter((lane) => lane.suite === 'full') };
|
||||
const narrowed = resolve(pullRequest('synchronize'), {
|
||||
select: { e2e_include: JSON.stringify(memoryOnly), mcp_run: 'false' },
|
||||
});
|
||||
assert.deepEqual(narrowed.jobs, { ...everyLane, mcp: false });
|
||||
assert.deepEqual(narrowed.matrix, memoryOnly);
|
||||
|
||||
const outsider = resolve(pullRequest('synchronize', 'CONTRIBUTOR', 'someone/LibreChat'));
|
||||
assert.deepEqual(outsider.jobs, {
|
||||
select: false,
|
||||
shards: false,
|
||||
mcp: false,
|
||||
apps: false,
|
||||
gate: false,
|
||||
});
|
||||
const sameRepositoryBot = resolve(pullRequest('opened', 'NONE'));
|
||||
assert.deepEqual(sameRepositoryBot.jobs, { ...everyLane, select: false });
|
||||
|
||||
for (const github of [nightly, manual]) {
|
||||
const run = resolve(github);
|
||||
assert.deepEqual(run.jobs, { ...everyLane, select: false });
|
||||
assert.deepEqual(run.matrix, defaults[1]);
|
||||
}
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue