LibreChat/.github/scripts/merge-gate.test.mjs
Marco Beretta e36598f019
⏩ ci: Record Playwright Shard Timings Automatically and Run Fifteen Shards (#18115)
* ci: Record Playwright Shard Timings From Green Dev Runs
* ci: Run the Playwright Mock Suite on Fifteen Memory Shards
* ci: Run the Playwright Workflow Script Tests Once per Run
* test: Make the Due Schedule E2E Wait for One Engine Tick Only
* test: Cover the Shard Timings Hand-off With Acceptance Scenarios
2026-11-14 01:25:48 +01:00

531 lines
22 KiB
JavaScript

import yaml from 'js-yaml';
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { fileURLToPath } from 'node:url';
import { spawnSync } from 'node:child_process';
import { readdirSync, readFileSync } from 'node:fs';
import { condition, interpolate } from './expressions.mjs';
import { SUITES, plan, stale, supersede, verify } from './merge-gate.mjs';
const REPO = 'LibreChat-AI/LibreChat';
const workflowsDir = new URL('../workflows/', import.meta.url);
const load = (file) => yaml.load(readFileSync(new URL(file, workflowsDir), 'utf8'));
const workflows = Object.fromEntries(
readdirSync(workflowsDir)
.filter((file) => file.endsWith('.yml'))
.map((file) => [file, load(file)]),
);
const unwrap = (text) =>
String(text)
.trim()
.replace(/^\$\{\{([\s\S]*)\}\}$/, '$1');
const STATUS_FUNCTION = /\b(?:always|cancelled|success|failure)\(/;
/** The status functions as GitHub resolves them for a job with these `needs`, in a live run. */
function statusOf(needs) {
const results = Object.values(needs).map((job) => job.result);
return {
always: () => true,
cancelled: () => false,
success: () => results.every((result) => result === 'success'),
failure: () => results.includes('failure'),
};
}
const needsOf = (job) => [job.needs ?? []].flat();
/**
* Resolves which jobs of a workflow run for one event, assuming every job that runs succeeds
* unless `forced` gives it another result. `outputs` supplies job outputs (the codegraph
* decision). Returns each job's result plus the Playwright matrix when the workflow has one.
*/
function simulate(workflow, github, { outputs = {}, forced = {}, inputs = {}, vars = {} } = {}) {
const jobs = {};
const context = { github: { repository: REPO, ...github }, inputs, vars };
const resolve = (id) => {
if (jobs[id]) return jobs[id];
const job = workflow.jobs[id];
const needs = Object.fromEntries(needsOf(job).map((need) => [need, resolve(need)]));
const scope = { ...context, needs };
const functions = statusOf(needs);
const written = job.if === undefined ? 'success()' : unwrap(job.if);
const guarded = STATUS_FUNCTION.test(written) ? written : `success() && (${written})`;
const runs = condition(guarded, scope, functions);
const result = runs ? (forced[id] ?? 'success') : 'skipped';
jobs[id] = { result, outputs: result === 'success' ? (outputs[id] ?? {}) : {} };
if (runs && typeof job.strategy?.matrix === 'string') {
jobs[id].matrix = interpolate(job.strategy.matrix, scope, functions);
}
return jobs[id];
};
Object.keys(workflow.jobs).forEach(resolve);
return jobs;
}
/** Whether a workflow's gate lets the simulated run through. A skipped check counts as passed. */
function gatePasses(suite, jobs) {
const [id, gate] = gateOf(suite);
if (jobs[id].result !== 'success') return jobs[id].result === 'skipped';
const needs = Object.fromEntries(
needsOf(gate).map((need) => [need, { result: jobs[need].result, outputs: jobs[need].outputs }]),
);
const functions = statusOf(needs);
const step = gate.steps.at(-1);
if (step.run === 'exit 1') {
return !condition(step.if, { needs }, functions);
}
assert.equal(step.run, 'node .github/scripts/merge-gate.mjs verify');
assert.equal(step.env.NEEDS, '${{ toJSON(needs) }}');
const allowed = String(interpolate(step.env.ALLOW_SKIPPED ?? '', { needs }, functions));
return (
verify(JSON.parse(JSON.stringify(needs)), allowed.split(/\s+/).filter(Boolean)).length === 0
);
}
function gateOf(suite) {
const entries = Object.entries(workflows[suite.workflow].jobs).filter(
([, job]) => job.name === suite.context,
);
assert.equal(entries.length, 1, `${suite.workflow} needs exactly one job named ${suite.context}`);
return entries[0];
}
const member = { author_association: 'MEMBER', head: { repo: { full_name: REPO } } };
const outsider = {
author_association: 'CONTRIBUTOR',
head: { repo: { full_name: 'fork/LibreChat' } },
};
const pullRequest = (action, pull_request = member) => ({
event_name: 'pull_request',
event: { action, pull_request },
});
const mergeGroup = (
entry = 'pr-104-929f8209d40f77f4abc622a499c93a83babdbe64',
sha = 'a'.repeat(40),
) => ({
event_name: 'merge_group',
ref: `refs/heads/gh-readonly-queue/dev/${entry}`,
sha,
event: {
action: 'checks_requested',
merge_group: {
head_ref: `refs/heads/gh-readonly-queue/dev/${entry}`,
base_ref: 'refs/heads/dev',
},
},
});
const SELECT = 'codegraph_select';
const ran = (jobs) => Object.keys(jobs).filter((id) => jobs[id].result === 'success');
const skipped = (jobs) => Object.keys(jobs).filter((id) => jobs[id].result === 'skipped');
test('every gated workflow starts for a merge group and closes in one gate over all of its jobs', () => {
for (const suite of SUITES) {
const workflow = workflows[suite.workflow];
assert.ok(workflow, `${suite.workflow} is missing`);
assert.deepEqual(workflow.on.merge_group, { types: ['checks_requested'] }, suite.workflow);
const [id, gate] = gateOf(suite);
const others = Object.keys(workflow.jobs).filter((job) => job !== id);
assert.deepEqual([...needsOf(gate)].sort(), others.sort(), `${suite.workflow} gate needs`);
assert.match(unwrap(gate.if), /^\s*!cancelled\(\)/, `${suite.workflow} gate condition`);
for (const [job, definition] of Object.entries(workflow.jobs)) {
assert.notEqual(
definition['continue-on-error'],
true,
`${suite.workflow} ${job} would report success to the gate after failing`,
);
}
}
});
test('an artifact upload can fail a gated job only when another job downloads the artifact', () => {
const runs = (step, action) => step.uses?.startsWith(`actions/${action}-artifact@`);
for (const suite of SUITES) {
const steps = Object.entries(workflows[suite.workflow].jobs).flatMap(([job, definition]) =>
definition.steps.map((step) => ({ job, step })),
);
const downloaded = new Set(
steps.filter(({ step }) => runs(step, 'download')).map(({ step }) => step.with.name),
);
for (const { job, step } of steps.filter(({ step }) => runs(step, 'upload'))) {
const label = `${suite.workflow} ${job} uploads ${step.with.name}`;
if (downloaded.has(step.with.name)) {
assert.equal(step['continue-on-error'], undefined, `${label}, which a later job needs`);
continue;
}
assert.equal(step['continue-on-error'], true, `${label}, which no job downloads`);
assert.ok(step['timeout-minutes'] > 0, `${label} without a bound on a stalled upload`);
}
}
});
test('a required context is the name of exactly one gate job and of nothing else', () => {
const contexts = SUITES.map((suite) => suite.context);
assert.equal(new Set(contexts).size, contexts.length);
const owners = contexts.map((context) =>
Object.entries(workflows).flatMap(([file, workflow]) =>
Object.values(workflow.jobs)
.filter((job) => job.name === context)
.map(() => file),
),
);
assert.deepEqual(
owners,
SUITES.map((suite) => [suite.workflow]),
);
});
test('a merge group runs every job in full and only the codegraph selector is skipped', () => {
for (const suite of SUITES) {
const workflow = workflows[suite.workflow];
const jobs = simulate(workflow, mergeGroup(), {
outputs: { [SELECT]: { api_run: 'false', client_run: 'false', mcp_run: 'false' } },
});
assert.deepEqual(skipped(jobs), SELECT in workflow.jobs ? [SELECT] : [], suite.workflow);
assert.equal(gatePasses(suite, jobs), true, suite.workflow);
}
});
test('a merge group gate fails when any job fails, is cancelled or is skipped', () => {
for (const suite of SUITES) {
const workflow = workflows[suite.workflow];
const [gate] = gateOf(suite);
const required = Object.keys(workflow.jobs).filter((id) => id !== gate && id !== SELECT);
assert.ok(required.length > 0, suite.workflow);
for (const id of required) {
for (const result of ['failure', 'cancelled', 'skipped']) {
const jobs = simulate(workflow, mergeGroup(), { forced: { [id]: result } });
jobs[id].result = result;
assert.equal(gatePasses(suite, jobs), false, `${suite.workflow} ${id} ${result}`);
}
}
}
});
test('the Playwright merge group matrix is the pull request profile with no lane narrowed', () => {
const workflow = workflows['playwright-mock.yml'];
const selector = workflow.jobs[SELECT].steps.find((step) => step.id === 'sel');
const profile = JSON.parse(selector.env.FULL_INCLUDE);
const queued = simulate(workflow, mergeGroup());
assert.deepEqual(queued.e2e_shards.matrix, profile);
assert.equal(profile.include.length, 17);
assert.deepEqual(ran(queued).sort(), [
'e2e',
'e2e_shards',
'mcp_apps',
'mcp_tool_list_changed',
'shard_timings',
]);
const opened = simulate(workflow, pullRequest('opened'));
assert.deepEqual(opened.e2e_shards.matrix, profile);
const nightly = simulate(workflow, { event_name: 'schedule', event: {} });
assert.equal(nightly.e2e_shards.matrix.include.length, 25);
});
test('pull request runs keep their selection: full when opened, narrowed on synchronize', () => {
const backend = workflows['backend-review.yml'];
const suite = SUITES.find((entry) => entry.workflow === 'backend-review.yml');
const opened = simulate(backend, pullRequest('opened'));
assert.deepEqual(skipped(opened), [SELECT]);
assert.equal(gatePasses(suite, opened), true);
const decision = { [SELECT]: { decided: 'true', api_run: 'false', pkgapi_run: 'true' } };
const narrowed = simulate(backend, pullRequest('synchronize'), { outputs: decision });
assert.deepEqual(skipped(narrowed), ['test-api']);
assert.equal(gatePasses(suite, narrowed), true);
const undecided = simulate(backend, pullRequest('synchronize'));
undecided['test-api'].result = 'skipped';
assert.equal(gatePasses(suite, undecided), false);
const killed = simulate(backend, pullRequest('synchronize'), {
outputs: decision,
vars: { CODEGRAPH_GATING: 'off' },
});
assert.deepEqual(skipped(killed), [SELECT]);
const frontend = workflows['frontend-review.yml'];
const frontendSuite = SUITES.find((entry) => entry.workflow === 'frontend-review.yml');
const trimmed = simulate(frontend, pullRequest('synchronize'), {
outputs: { [SELECT]: { client_run: 'false', clientpkg_run: 'false' } },
});
assert.deepEqual(skipped(trimmed), ['test-packages-client', 'test-ubuntu']);
assert.equal(gatePasses(frontendSuite, trimmed), true);
});
test('a failing job fails the pull request gate, which still runs to report it', () => {
for (const suite of SUITES) {
const workflow = workflows[suite.workflow];
const [gate] = gateOf(suite);
const first = Object.keys(workflow.jobs).find((id) => id !== gate && id !== SELECT);
const jobs = simulate(workflow, pullRequest('opened'), { forced: { [first]: 'failure' } });
assert.equal(jobs[gate].result, 'success', `${suite.workflow} gate must still run`);
assert.equal(gatePasses(suite, jobs), false, suite.workflow);
}
});
test('push and manual runs still succeed job by job, as main promotion requires', () => {
const events = {
push: { event_name: 'push', event: {} },
workflow_dispatch: { event_name: 'workflow_dispatch', event: {} },
};
for (const file of ['backend-review.yml', 'frontend-review.yml']) {
const suite = SUITES.find((entry) => entry.workflow === file);
for (const [name, github] of Object.entries(events)) {
assert.ok(name in workflows[file].on, `${file} ${name}`);
const jobs = simulate(workflows[file], github);
assert.deepEqual(skipped(jobs), [SELECT], `${file} ${name}`);
assert.equal(gatePasses(suite, jobs), true, `${file} ${name}`);
}
}
});
test('Playwright still skips outside contributors on pull requests and reports that as a skip', () => {
const workflow = workflows['playwright-mock.yml'];
const jobs = simulate(workflow, pullRequest('opened', outsider));
assert.deepEqual(ran(jobs), []);
assert.equal(
gatePasses(
SUITES.find((suite) => suite.context === 'e2e'),
jobs,
),
true,
);
assert.deepEqual(ran(simulate(workflow, mergeGroup())).includes('e2e_shards'), true);
});
test('merge group runs never share a concurrency group across queue entries', () => {
const first = mergeGroup('pr-104-929f8209d40f77f4abc622a499c93a83babdbe64', 'a'.repeat(40));
const second = mergeGroup('pr-105-2ffea6db159f6b6c47a24e778fb9ef40cf6b1c7d', 'b'.repeat(40));
const groups = new Set();
const files = [...SUITES.map((suite) => suite.workflow), 'merge-gate.yml'];
for (const file of files) {
const { concurrency, name } = workflows[file];
const group = (github) =>
interpolate(concurrency.group, { github: { workflow: name, ...github } }, statusOf({}));
assert.notEqual(group(first), group(second), file);
assert.equal(group(first), group(mergeGroup()), `${file} re-created group`);
for (const github of [first, second]) {
assert.equal(groups.has(group(github)), false, `${file} shares a group`);
groups.add(group(github));
}
const pull = { ...pullRequest('synchronize'), ref: 'refs/pull/7/merge', sha: 'c'.repeat(40) };
pull.event.pull_request = { ...member, number: 7 };
assert.equal(groups.has(group(pull)), false, `${file} pull request group`);
}
});
test('static checks read the merge group base wherever they read the pull request base', () => {
const text = readFileSync(new URL('static-checks.yml', workflowsDir), 'utf8');
const reads = text.match(/jq --raw-output .*GITHUB_EVENT_PATH.*/g);
assert.ok(reads.length > 0);
for (const read of reads) {
assert.match(read, /'\.pull_request\.base\.sha \/\/ \.merge_group\.base_sha'/);
}
});
test('the companion workflow reports a context only for a suite that did not start', () => {
const workflow = workflows['merge-gate.yml'];
assert.deepEqual(Object.keys(workflow.on).sort(), ['merge_group', 'pull_request']);
assert.equal(workflow.on.pull_request, null);
const { plan: planner, placeholder, supersede } = workflow.jobs;
assert.equal(placeholder.name, '${{ matrix.context }}');
assert.equal(placeholder.strategy.matrix.context, '${{ fromJSON(needs.plan.outputs.contexts) }}');
assert.equal(planner.outputs.contexts, '${{ steps.plan.outputs.contexts }}');
const pull = simulate(workflow, pullRequest('opened'), {
outputs: { plan: { contexts: '["backend","static"]' } },
});
assert.deepEqual(ran(pull).sort(), ['placeholder', 'plan']);
const none = simulate(workflow, pullRequest('opened'), { outputs: { plan: { contexts: '[]' } } });
assert.deepEqual(ran(none), ['plan']);
const failed = simulate(workflow, pullRequest('opened'), { forced: { plan: 'failure' } });
assert.deepEqual(skipped(failed).sort(), ['placeholder', 'supersede']);
const queued = simulate(workflow, mergeGroup());
assert.deepEqual(ran(queued), ['supersede']);
assert.deepEqual(supersede.permissions, { contents: 'read', actions: 'write' });
assert.deepEqual(planner.permissions, { contents: 'read', actions: 'read' });
});
test('verify passes success and listed skips only', () => {
const ok = { result: 'success', outputs: {} };
const skip = { result: 'skipped', outputs: {} };
assert.deepEqual(verify({ build: ok, test: ok }), []);
assert.deepEqual(verify({ build: ok, select: skip }, ['select']), []);
assert.deepEqual(verify({ build: ok, select: skip }), ['select: skipped']);
assert.deepEqual(
verify({ build: { result: 'failure' }, test: skip, lint: { result: 'cancelled' } }, ['test']),
['build: failure', 'lint: cancelled'],
);
assert.deepEqual(verify({ build: { result: 'neutral' } }), ['build: neutral']);
assert.throws(() => verify({}), /No job results/);
assert.throws(() => verify(null), /No job results/);
assert.throws(() => verify({ build: ok }, ['tests']), /does not depend on: tests/);
});
test('plan reports the suites GitHub did not start and waits for late runs', async () => {
const sleeps = [];
const sleep = async (ms) => sleeps.push(ms);
const polls = {};
const started = (answers) => async (workflow) => {
polls[workflow] = (polls[workflow] ?? 0) + 1;
const answer = answers[workflow] ?? true;
const value = Array.isArray(answer)
? answer[Math.min(polls[workflow], answer.length) - 1]
: answer;
if (value instanceof Error) throw value;
return value;
};
assert.deepEqual(await plan({ started: started({}), sleep }), []);
assert.deepEqual(sleeps, []);
const docsOnly = Object.fromEntries(SUITES.map((suite) => [suite.workflow, false]));
assert.deepEqual(
await plan({ started: started(docsOnly), sleep }),
SUITES.map((suite) => suite.context),
);
assert.deepEqual(sleeps, [10_000, 10_000]);
Object.keys(polls).forEach((key) => delete polls[key]);
const late = { 'backend-review.yml': [false, true], 'static-checks.yml': false };
assert.deepEqual(await plan({ started: started(late), sleep }), ['static']);
assert.equal(polls['backend-review.yml'], 2);
assert.equal(polls['static-checks.yml'], 3);
assert.equal(polls['frontend-review.yml'], 1);
Object.keys(polls).forEach((key) => delete polls[key]);
const flaky = { 'backend-review.yml': [new Error('HTTP 502'), true] };
assert.deepEqual(await plan({ started: started(flaky), sleep }), []);
Object.keys(polls).forEach((key) => delete polls[key]);
const down = { 'backend-review.yml': new Error('HTTP 502') };
await assert.rejects(plan({ started: started(down), sleep }), /HTTP 502/);
});
const queueRun = (id, head_branch, overrides = {}) => ({
id,
head_branch,
event: 'merge_group',
created_at: '2026-11-11T11:50:00Z',
...overrides,
});
test('stale finds only active merge group runs whose queue branch is gone', async () => {
const now = Date.parse('2026-11-11T12:00:00Z');
const current = 'gh-readonly-queue/dev/pr-9-new';
const runs = [
queueRun(1, 'gh-readonly-queue/dev/pr-9-old'),
queueRun(2, 'gh-readonly-queue/dev/pr-9-old'),
queueRun(3, 'gh-readonly-queue/dev/pr-8-live'),
queueRun(4, current),
queueRun(5, 'gh-readonly-queue/dev/pr-7-unknown'),
queueRun(6, 'gh-readonly-queue/dev/pr-6-fresh', { created_at: '2026-11-11T11:59:30Z' }),
queueRun(7, 'dev', { event: 'push' }),
queueRun(8, 'feature/gh-readonly-queue/dev/pr-5'),
queueRun(9, null),
queueRun(10, 'gh-readonly-queue/dev/pr-4-live', { created_at: '2026-11-11T11:59:50Z' }),
];
const lookups = [];
const exists = async (branch) => {
lookups.push(branch);
if (branch.endsWith('unknown')) throw new Error('HTTP 500');
return branch === current || branch.endsWith('live');
};
const { cancel, recheck } = await stale({ runs, current, exists, now });
assert.deepEqual(
cancel.map((entry) => entry.id),
[1, 2],
);
assert.deepEqual(
recheck.map((entry) => entry.id),
[6],
);
assert.deepEqual(lookups.sort(), [
'gh-readonly-queue/dev/pr-4-live',
'gh-readonly-queue/dev/pr-6-fresh',
'gh-readonly-queue/dev/pr-7-unknown',
'gh-readonly-queue/dev/pr-8-live',
current,
'gh-readonly-queue/dev/pr-9-old',
]);
assert.deepEqual(await stale({ runs: [], current, exists, now }), { cancel: [], recheck: [] });
const nothing = { cancel: [], recheck: [] };
const allMissing = async () => false;
const failing = async () => {
throw new Error('HTTP 503');
};
assert.deepEqual(await stale({ runs, current, exists: allMissing, now }), nothing);
assert.deepEqual(await stale({ runs, current, exists: failing, now }), nothing);
assert.deepEqual(await stale({ runs, current: undefined, exists: allMissing, now }), nothing);
});
test('supersede cancels each stale run once and revisits runs that were too young', async () => {
const current = 'gh-readonly-queue/dev/pr-9-new';
const old = queueRun(1, 'gh-readonly-queue/dev/pr-9-old');
const fresh = queueRun(2, 'gh-readonly-queue/dev/pr-8-fresh', {
created_at: '2026-11-11T11:59:40Z',
});
const reborn = queueRun(3, 'gh-readonly-queue/dev/pr-7-back', {
created_at: '2026-11-11T11:59:45Z',
});
const live = queueRun(4, 'gh-readonly-queue/dev/pr-6-live');
const harness = (runs, missing) => {
const state = { time: Date.parse('2026-11-11T12:00:00Z'), lists: 0, sleeps: [], cancelled: [] };
const options = {
current,
list: async () => {
state.lists += 1;
return runs;
},
exists: async (branch) => branch === current || !missing(branch),
cancel: async (run) => {
state.cancelled.push(run.id);
return `cancelled ${run.id}`;
},
sleep: async (ms) => {
state.sleeps.push(ms);
state.time += ms;
},
clock: () => state.time,
};
return { state, options };
};
const settled = harness([old, live], (branch) => branch.endsWith('old'));
assert.deepEqual(await supersede(settled.options), ['cancelled 1']);
assert.deepEqual([settled.state.lists, settled.state.sleeps], [1, []]);
const lookups = {};
const mixed = harness([old, fresh, reborn, live], (branch) => {
lookups[branch] = (lookups[branch] ?? 0) + 1;
if (branch.endsWith('back')) return lookups[branch] === 1;
return !branch.endsWith('live');
});
assert.deepEqual(await supersede(mixed.options), ['cancelled 1', 'cancelled 2']);
assert.deepEqual(mixed.state.cancelled, [1, 2]);
assert.deepEqual([mixed.state.lists, mixed.state.sleeps], [2, [60_000]]);
});
test('the verify command exits non-zero unless the gate passes', () => {
const script = fileURLToPath(new URL('./merge-gate.mjs', import.meta.url));
const run = (env) =>
spawnSync(process.execPath, [script, 'verify'], {
encoding: 'utf8',
env: { PATH: process.env.PATH, ...env },
});
const needs = {
build: { result: 'success', outputs: {} },
select: { result: 'skipped', outputs: {} },
};
const passed = run({ NEEDS: JSON.stringify(needs), ALLOW_SKIPPED: ' select ' });
assert.equal(passed.status, 0, passed.stdout);
const blocked = run({ NEEDS: JSON.stringify(needs) });
assert.equal(blocked.status, 1);
assert.match(blocked.stdout, /::error::select: skipped/);
assert.equal(run({ NEEDS: '' }).status, 1);
assert.equal(run({}).status, 1);
assert.equal(run({ NEEDS: '{}' }).status, 1);
assert.equal(spawnSync(process.execPath, [script, 'nonsense'], { encoding: 'utf8' }).status, 1);
});