mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-19 06:23:32 +00:00
🐙 ci: Spread Playwright Coverage Across Isolated Shards (#16358)
* 🐙 ci: Spread Playwright Coverage Across Isolated Shards
* ci: Allow Manual Validation of the PR Shard Profile
---------
Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
parent
3c08a7dd04
commit
24570b68cf
2 changed files with 218 additions and 6 deletions
200
.github/scripts/playwright-matrix.test.mjs
vendored
Normal file
200
.github/scripts/playwright-matrix.test.mjs
vendored
Normal file
|
|
@ -0,0 +1,200 @@
|
|||
import yaml from 'js-yaml';
|
||||
import { test } from 'node:test';
|
||||
import { join } from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
|
||||
const scriptsDir = fileURLToPath(new URL('.', import.meta.url));
|
||||
const workflow = yaml.load(
|
||||
readFileSync(new URL('../workflows/playwright-mock.yml', import.meta.url), 'utf8'),
|
||||
);
|
||||
const selector = workflow.jobs.codegraph_select.steps.find((step) => step.id === 'sel');
|
||||
const shards = workflow.jobs.e2e_shards;
|
||||
const defaults = [...shards.strategy.matrix.matchAll(/fromJSON\('([^']+)'\)/g)].map((match) =>
|
||||
JSON.parse(match[1]),
|
||||
);
|
||||
const prMatrix = JSON.parse(selector.env.FULL_INCLUDE);
|
||||
|
||||
function assertPartition(matrix, store, suite, count) {
|
||||
const lanes = matrix.include.filter(
|
||||
(lane) => lane.stream_store === store && lane.suite === suite,
|
||||
);
|
||||
assert.equal(lanes.length, count);
|
||||
assert.deepEqual(
|
||||
lanes.map((lane) => lane.shard),
|
||||
Array.from({ length: count }, (_, index) => `${index + 1}/${count}`),
|
||||
);
|
||||
assert.ok(
|
||||
lanes.every((lane) => lane.redis_image === (store === 'redis' ? 'redis:7-alpine' : '')),
|
||||
);
|
||||
}
|
||||
|
||||
function select(t, response, overrides = {}) {
|
||||
const scratch = mkdtempSync(join(scriptsDir, '.playwright-matrix-test-'));
|
||||
t.after(() => rmSync(scratch, { recursive: true, force: true }));
|
||||
writeFileSync(join(scratch, 'response.json'), JSON.stringify(response));
|
||||
writeFileSync(join(scratch, 'output'), '');
|
||||
writeFileSync(join(scratch, 'selector.sh'), selector.run);
|
||||
writeFileSync(
|
||||
join(scratch, 'gh'),
|
||||
'#!/usr/bin/env bash\nprintf \'%s\\n\' \'{"path":"client/src/example.ts","status":"modified"}\'\nexit "${GH_EXIT:-0}"\n',
|
||||
{ mode: 0o700 },
|
||||
);
|
||||
writeFileSync(
|
||||
join(scratch, 'curl'),
|
||||
'#!/usr/bin/env bash\ncat response.json\nexit "${CURL_EXIT:-0}"\n',
|
||||
{ mode: 0o700 },
|
||||
);
|
||||
const result = spawnSync('bash', ['selector.sh'], {
|
||||
cwd: scratch,
|
||||
encoding: 'utf8',
|
||||
timeout: 10000,
|
||||
env: {
|
||||
PATH: `${scratch}:${process.env.PATH}`,
|
||||
FULL_INCLUDE: selector.env.FULL_INCLUDE,
|
||||
URL: 'https://selector.invalid',
|
||||
TOKEN: 'test-fixture',
|
||||
REPO: 'fixture/repo',
|
||||
PR: '1',
|
||||
BASE_SHA: 'base',
|
||||
HEAD_SHA: 'head',
|
||||
CHANGED: '1',
|
||||
E2E_SKIP_ARMED: '',
|
||||
GITHUB_OUTPUT: join(scratch, 'output'),
|
||||
GITHUB_STEP_SUMMARY: join(scratch, 'summary'),
|
||||
...overrides,
|
||||
},
|
||||
});
|
||||
assert.equal(result.status, 0, result.stdout + result.stderr);
|
||||
const outputs = Object.fromEntries(
|
||||
readFileSync(join(scratch, 'output'), 'utf8')
|
||||
.trim()
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const separator = line.indexOf('=');
|
||||
return [line.slice(0, separator), line.slice(separator + 1)];
|
||||
}),
|
||||
);
|
||||
return { outputs, matrix: outputs.e2e_include ? JSON.parse(outputs.e2e_include) : defaults[0] };
|
||||
}
|
||||
|
||||
function decision(redis, mcp = true) {
|
||||
return { matrix: { 'playwright-mock': { redis_transport: redis, mcp_tool_list_changed: mcp } } };
|
||||
}
|
||||
|
||||
test('PR selection and fail-open defaults use six memory shards and two Redis transport shards', () => {
|
||||
assert.equal(defaults.length, 2);
|
||||
assert.deepEqual(defaults[0], prMatrix);
|
||||
assert.equal(prMatrix.include.length, 8);
|
||||
assertPartition(prMatrix, 'memory', 'full', 6);
|
||||
assertPartition(prMatrix, 'redis', 'transport', 2);
|
||||
});
|
||||
|
||||
test('nightly and default manual runs retain full-suite coverage on both stores', () => {
|
||||
assert.equal(workflow.on.workflow_dispatch.inputs.coverage.default, 'full');
|
||||
assert.equal(defaults[1].include.length, 4);
|
||||
assertPartition(defaults[1], 'memory', 'full', 2);
|
||||
assertPartition(defaults[1], 'redis', 'full', 2);
|
||||
});
|
||||
|
||||
test('manual PR coverage opts into the PR fallback matrix without bypassing the PR author gate', () => {
|
||||
assert.deepEqual(workflow.on.workflow_dispatch.inputs.coverage.options, ['full', 'pr']);
|
||||
assert.match(
|
||||
shards.strategy.matrix,
|
||||
/github\.event_name == 'pull_request' \|\| inputs\.coverage == 'pr'/,
|
||||
);
|
||||
assert.match(shards.if, /github\.event_name == 'workflow_dispatch'/);
|
||||
assert.match(shards.if, /github\.event\.pull_request\.author_association/);
|
||||
});
|
||||
|
||||
test('every shard has a unique job name and artifact namespace', () => {
|
||||
for (const matrix of [prMatrix, ...defaults]) {
|
||||
for (const key of ['name', 'artifact']) {
|
||||
assert.equal(new Set(matrix.include.map((lane) => lane[key])).size, matrix.include.length);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('both suite commands consume the matrix shard without increasing worker concurrency', () => {
|
||||
for (const suite of ['full', 'transport']) {
|
||||
const step = shards.steps.find((entry) => entry.if === `matrix.suite == '${suite}'`);
|
||||
assert.match(step.run, /--shard=\$\{\{ matrix\.shard \}\}/);
|
||||
assert.doesNotMatch(step.run, /--workers/);
|
||||
}
|
||||
const config = readFileSync(
|
||||
new URL('../../e2e/playwright.config.mock.ts', import.meta.url),
|
||||
'utf8',
|
||||
);
|
||||
assert.match(config, /workers: 1,/);
|
||||
assert.equal(shards['runs-on'], 'ubuntu-latest');
|
||||
});
|
||||
|
||||
test('an explicit Redis skip drops both transport shards and no memory shard', (t) => {
|
||||
const { matrix, outputs } = select(t, decision(false, false));
|
||||
assert.equal(outputs.decided, 'true');
|
||||
assert.equal(outputs.mcp_run, 'false');
|
||||
assert.equal(matrix.include.length, 6);
|
||||
assertPartition(matrix, 'memory', 'full', 6);
|
||||
});
|
||||
|
||||
test('missing or non-boolean Redis decisions retain both transport shards', (t) => {
|
||||
for (const redis of [true, 'false', null, undefined]) {
|
||||
const { matrix } = select(t, decision(redis));
|
||||
assert.deepEqual(matrix, prMatrix);
|
||||
assertPartition(matrix, 'redis', 'transport', 2);
|
||||
}
|
||||
});
|
||||
|
||||
test('unavailable configuration, failed requests, and incomplete file lists fall back to the full PR matrix', (t) => {
|
||||
for (const overrides of [
|
||||
{ URL: '' },
|
||||
{ TOKEN: '' },
|
||||
{ CURL_EXIT: '28' },
|
||||
{ GH_EXIT: '1' },
|
||||
{ CHANGED: '2' },
|
||||
]) {
|
||||
const { matrix, outputs } = select(t, decision(false, false), overrides);
|
||||
assert.equal(outputs.e2e_include, undefined);
|
||||
assert.equal(outputs.mcp_run, undefined);
|
||||
assert.deepEqual(matrix, prMatrix);
|
||||
}
|
||||
const { matrix, outputs } = select(t, {});
|
||||
assert.equal(outputs.e2e_include, undefined);
|
||||
assert.deepEqual(matrix, prMatrix);
|
||||
});
|
||||
|
||||
test('graduated spec skips keep the same pool across shards and remain opt-in', (t) => {
|
||||
const response = {
|
||||
...decision(true),
|
||||
e2e: { fail_open: false, graduated: ['e2e/specs/mock/sidebar.spec.ts'] },
|
||||
};
|
||||
const dark = select(t, response);
|
||||
assert.equal(dark.outputs.e2e_skip, '');
|
||||
const armed = select(t, response, { E2E_SKIP_ARMED: 'on' });
|
||||
assert.equal(armed.outputs.e2e_skip, 'specs/mock/sidebar.spec.ts');
|
||||
assert.deepEqual(armed.matrix, prMatrix);
|
||||
const fallback = select(
|
||||
t,
|
||||
{ ...response, e2e: { ...response.e2e, fail_open: true } },
|
||||
{ E2E_SKIP_ARMED: 'on' },
|
||||
);
|
||||
assert.equal(fallback.outputs.e2e_skip, '');
|
||||
assert.deepEqual(fallback.matrix, prMatrix);
|
||||
});
|
||||
|
||||
test('the required gate still waits for all shards and rejects failures without fighting cancellation', () => {
|
||||
const gate = workflow.jobs.e2e;
|
||||
assert.equal(gate.name, 'e2e');
|
||||
assert.deepEqual(gate.needs, ['codegraph_select', 'e2e_shards', 'mcp_tool_list_changed']);
|
||||
assert.equal(shards.strategy['fail-fast'], false);
|
||||
assert.match(gate.if, /!cancelled\(\)/);
|
||||
assert.match(shards.if, /!cancelled\(\)/);
|
||||
assert.match(gate.steps[0].if, /needs\.e2e_shards\.result != 'success'/);
|
||||
assert.match(gate.steps[0].if, /needs\.mcp_tool_list_changed\.result != 'success'/);
|
||||
assert.match(gate.steps[0].if, /needs\.codegraph_select\.outputs\.mcp_run == 'false'/);
|
||||
assert.equal(gate.steps[0].run, 'exit 1');
|
||||
assert.equal(workflow.concurrency['cancel-in-progress'], true);
|
||||
});
|
||||
24
.github/workflows/playwright-mock.yml
vendored
24
.github/workflows/playwright-mock.yml
vendored
|
|
@ -15,6 +15,13 @@ on:
|
|||
description: 'Reason for manual trigger'
|
||||
required: false
|
||||
default: 'Manual e2e run'
|
||||
coverage:
|
||||
description: 'Coverage profile: full suite on both stores, or the PR shards'
|
||||
type: choice
|
||||
options:
|
||||
- full
|
||||
- pr
|
||||
default: full
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
@ -29,7 +36,7 @@ env:
|
|||
|
||||
jobs:
|
||||
# Stage 2 of codegraph gating (stage 1 = backend jest in backend-review.yml). The graph decides
|
||||
# two matrix lanes whose relevance is a reachability question: the redis-transport lane (the ten
|
||||
# two matrix lanes whose relevance is a reachability question: the redis-transport shards (the
|
||||
# stream-boundary specs under a real Redis round-trip) and the MCP list_changed lanes. Memory
|
||||
# shards always run. Monotone and fail-open: a lane is dropped ONLY on an explicit `false` from
|
||||
# the service; unavailable/unconfigured/non-synchronize events keep the full matrix. Lock
|
||||
|
|
@ -63,7 +70,7 @@ jobs:
|
|||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
CHANGED: ${{ github.event.pull_request.changed_files }}
|
||||
E2E_SKIP_ARMED: ${{ vars.CODEGRAPH_E2E_SKIP }}
|
||||
FULL_INCLUDE: '{"include":[{"name":"memory, shard 1/3","stream_store":"memory","redis_image":"","suite":"full","shard":"1/3","artifact":"memory-1-of-3"},{"name":"memory, shard 2/3","stream_store":"memory","redis_image":"","suite":"full","shard":"2/3","artifact":"memory-2-of-3"},{"name":"memory, shard 3/3","stream_store":"memory","redis_image":"","suite":"full","shard":"3/3","artifact":"memory-3-of-3"},{"name":"redis transport","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"","artifact":"redis-transport"}]}'
|
||||
FULL_INCLUDE: '{"include":[{"name":"memory, shard 1/6","stream_store":"memory","redis_image":"","suite":"full","shard":"1/6","artifact":"memory-1-of-6"},{"name":"memory, shard 2/6","stream_store":"memory","redis_image":"","suite":"full","shard":"2/6","artifact":"memory-2-of-6"},{"name":"memory, shard 3/6","stream_store":"memory","redis_image":"","suite":"full","shard":"3/6","artifact":"memory-3-of-6"},{"name":"memory, shard 4/6","stream_store":"memory","redis_image":"","suite":"full","shard":"4/6","artifact":"memory-4-of-6"},{"name":"memory, shard 5/6","stream_store":"memory","redis_image":"","suite":"full","shard":"5/6","artifact":"memory-5-of-6"},{"name":"memory, shard 6/6","stream_store":"memory","redis_image":"","suite":"full","shard":"6/6","artifact":"memory-6-of-6"},{"name":"redis transport, shard 1/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"1/2","artifact":"redis-transport-1-of-2"},{"name":"redis transport, shard 2/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"2/2","artifact":"redis-transport-2-of-2"}]}'
|
||||
run: |
|
||||
set +e
|
||||
note() { echo "$1" >> "$GITHUB_STEP_SUMMARY"; }
|
||||
|
|
@ -112,7 +119,7 @@ jobs:
|
|||
INCLUDE="$FULL_INCLUDE"
|
||||
note "| redis transport | run |"
|
||||
fi
|
||||
if ! echo "$INCLUDE" | jq -e '.include | length >= 3' >/dev/null 2>&1; then
|
||||
if ! echo "$INCLUDE" | jq -e '.include | length >= 6' >/dev/null 2>&1; then
|
||||
note "_matrix assembly failed; running FULL_"
|
||||
exit 0
|
||||
fi
|
||||
|
|
@ -161,14 +168,16 @@ jobs:
|
|||
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association)))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
# Separate runners isolate each shard's user, MongoDB, Redis and fixture state.
|
||||
# Keep workers: 1 in Playwright; shared-user cleanup is not worker-safe.
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: >-
|
||||
${{
|
||||
(github.event_name == 'pull_request' && needs.codegraph_select.outputs.e2e_include != '' &&
|
||||
fromJSON(needs.codegraph_select.outputs.e2e_include)) ||
|
||||
(github.event_name == 'pull_request' &&
|
||||
fromJSON('{"include":[{"name":"memory, shard 1/3","stream_store":"memory","redis_image":"","suite":"full","shard":"1/3","artifact":"memory-1-of-3"},{"name":"memory, shard 2/3","stream_store":"memory","redis_image":"","suite":"full","shard":"2/3","artifact":"memory-2-of-3"},{"name":"memory, shard 3/3","stream_store":"memory","redis_image":"","suite":"full","shard":"3/3","artifact":"memory-3-of-3"},{"name":"redis transport","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"","artifact":"redis-transport"}]}')) ||
|
||||
((github.event_name == 'pull_request' || inputs.coverage == 'pr') &&
|
||||
fromJSON('{"include":[{"name":"memory, shard 1/6","stream_store":"memory","redis_image":"","suite":"full","shard":"1/6","artifact":"memory-1-of-6"},{"name":"memory, shard 2/6","stream_store":"memory","redis_image":"","suite":"full","shard":"2/6","artifact":"memory-2-of-6"},{"name":"memory, shard 3/6","stream_store":"memory","redis_image":"","suite":"full","shard":"3/6","artifact":"memory-3-of-6"},{"name":"memory, shard 4/6","stream_store":"memory","redis_image":"","suite":"full","shard":"4/6","artifact":"memory-4-of-6"},{"name":"memory, shard 5/6","stream_store":"memory","redis_image":"","suite":"full","shard":"5/6","artifact":"memory-5-of-6"},{"name":"memory, shard 6/6","stream_store":"memory","redis_image":"","suite":"full","shard":"6/6","artifact":"memory-6-of-6"},{"name":"redis transport, shard 1/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"1/2","artifact":"redis-transport-1-of-2"},{"name":"redis transport, shard 2/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"2/2","artifact":"redis-transport-2-of-2"}]}')) ||
|
||||
fromJSON('{"include":[{"name":"memory, shard 1/2","stream_store":"memory","redis_image":"","suite":"full","shard":"1/2","artifact":"memory-1-of-2"},{"name":"memory, shard 2/2","stream_store":"memory","redis_image":"","suite":"full","shard":"2/2","artifact":"memory-2-of-2"},{"name":"redis, shard 1/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"1/2","artifact":"redis-1-of-2"},{"name":"redis, shard 2/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"2/2","artifact":"redis-2-of-2"}]}')
|
||||
}}
|
||||
services:
|
||||
|
|
@ -211,6 +220,9 @@ jobs:
|
|||
if: steps.cache-node-modules.outputs.cache-hit != 'true'
|
||||
run: npm ci
|
||||
|
||||
- name: Test Playwright matrix selection
|
||||
run: node --test .github/scripts/playwright-matrix.test.mjs
|
||||
|
||||
- name: Restore data-provider build cache
|
||||
id: cache-data-provider
|
||||
uses: actions/cache@v5
|
||||
|
|
@ -356,7 +368,7 @@ jobs:
|
|||
|
||||
- name: Run Redis stream transport e2e
|
||||
if: matrix.suite == 'transport'
|
||||
run: npx playwright test --config=e2e/playwright.config.redis.ts
|
||||
run: npx playwright test --config=e2e/playwright.config.redis.ts --shard=${{ matrix.shard }}
|
||||
env:
|
||||
CI: 'true'
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue