mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-19 22:43:38 +00:00
🎲 ci: Fail Playwright Runs When a Test Passes Only on a Retry (#18051)
Playwright reports a test that fails and then passes on a retry as "flaky" and exits 0, so a regression that fails two attempts in three merges green. Set `failOnFlakyTests` in CI on the mock profile (inherited by the Redis transport, registered-email and other derived profiles) and on the base profile. Retry counts are unchanged: the later attempts still record the trace and video and show whether a failure repeats. A run failed this way still prints "N flaky" beside a passed count, so add a reporter that names each retried pass in the job log and the job summary and says that it is what failed the run.
This commit is contained in:
parent
97f7fcdbac
commit
c6b7a5ac12
8 changed files with 335 additions and 7 deletions
7
.github/workflows/playwright-mock.yml
vendored
7
.github/workflows/playwright-mock.yml
vendored
|
|
@ -245,8 +245,8 @@ jobs:
|
|||
if: steps.cache-node-modules.outputs.cache-hit != 'true'
|
||||
run: npm ci
|
||||
|
||||
- name: Test Playwright matrix selection and the vote gate
|
||||
run: node --test .github/scripts/playwright-matrix.test.mjs .github/scripts/playwright-shards.test.mjs .github/scripts/vote-gate.test.mjs
|
||||
- name: Test Playwright matrix selection, the vote gate and the retried-pass reporter
|
||||
run: node --test .github/scripts/playwright-matrix.test.mjs .github/scripts/playwright-shards.test.mjs .github/scripts/vote-gate.test.mjs e2e/reporters/retried.test.mjs
|
||||
|
||||
- name: Restore data-provider build cache
|
||||
id: cache-data-provider
|
||||
|
|
@ -440,7 +440,8 @@ jobs:
|
|||
E2E_TITLE_CONVO: 'true'
|
||||
run: >-
|
||||
npx playwright test --config=e2e/playwright.config.mock.ts
|
||||
conversation-management.spec.ts --grep 'first-turn|final-title polling' --reporter=line
|
||||
conversation-management.spec.ts --grep 'first-turn|final-title polling'
|
||||
--reporter=line,e2e/reporters/retried.mjs
|
||||
--output=e2e/specs/.test-results/title-ownership
|
||||
|
||||
- name: Upload title ownership diagnostics
|
||||
|
|
|
|||
|
|
@ -101,6 +101,11 @@ The nightly schedule and manual workflow dispatch run the complete mock suite in
|
|||
modes across ten shards per mode. Every shard keeps one worker so tests do not contend for its
|
||||
authenticated user and database.
|
||||
|
||||
CI retries a failed test, twice on the memory shards and three times on the Redis transport
|
||||
suite, so the report carries a trace of each failed attempt and shows whether the failure
|
||||
repeats. A test that passes only on a retry still fails the run (`failOnFlakyTests`), and the
|
||||
job log and the job summary name it (`e2e/reporters/retried.mjs`). Local runs do not retry.
|
||||
|
||||
CI splits the complete suite by recorded duration rather than test count: each shard lists the
|
||||
suite, and `.github/scripts/playwright-shards.mjs plan` hands out whole spec files longest-first
|
||||
to the least-loaded shard using `e2e/timings.mock.json`. A spec with no recorded time is
|
||||
|
|
|
|||
|
|
@ -53,7 +53,11 @@ export default defineConfig({
|
|||
outputDir: `specs/.test-results/email-change${profileSuffix}`,
|
||||
fullyParallel: false,
|
||||
workers: 1,
|
||||
reporter: [['html', { outputFolder: reportPath, open: 'never' }], ['list']],
|
||||
reporter: [
|
||||
['html', { outputFolder: reportPath, open: 'never' }],
|
||||
['list'],
|
||||
['./reporters/retried.mjs'],
|
||||
],
|
||||
use: {
|
||||
...mockConfig.use,
|
||||
storageState: path.resolve(rootPath, `e2e/.generated/storageState.email${profileSuffix}.json`),
|
||||
|
|
|
|||
|
|
@ -524,15 +524,21 @@ export default defineConfig({
|
|||
fullyParallel: true,
|
||||
forbidOnly: !!process.env.CI,
|
||||
retries: process.env.CI ? 2 : 0,
|
||||
/** In CI a test that fails and then passes on a retry fails the run. The retries stay for
|
||||
* what they show: a trace of each failed attempt, video of the first retry, and whether the
|
||||
* failure is deterministic. Derived profiles inherit this with the rest of the config. */
|
||||
failOnFlakyTests: !!process.env.CI,
|
||||
workers: 1,
|
||||
/** In CI the JSON results land beside the HTML report (reporters end in order, after the HTML
|
||||
* reporter has replaced its folder), so the report artifact also carries the per-file
|
||||
* timings `.github/scripts/playwright-shards.mjs record` rebuilds the shard plan from. */
|
||||
* timings `.github/scripts/playwright-shards.mjs record` rebuilds the shard plan from.
|
||||
* The last reporter names each retried pass in the job log and the job summary. */
|
||||
reporter: process.env.CI
|
||||
? [
|
||||
['html', { outputFolder: reportPath, open: 'never' }],
|
||||
['line'],
|
||||
['json', { outputFile: path.join(reportPath, 'results.json') }],
|
||||
['./reporters/retried.mjs'],
|
||||
]
|
||||
: [['html', { outputFolder: reportPath }], ['list']],
|
||||
use: {
|
||||
|
|
|
|||
|
|
@ -10,7 +10,9 @@ export default defineConfig({
|
|||
* pause/resume and rehydrate paths most of all, since they replay a whole job.
|
||||
* The suite is serial (`workers: 1`), so this is per-operation latency rather
|
||||
* than contention, and it is why this lane reports flaky runs the memory shards
|
||||
* never produce. Give it a larger default assertion budget and one more retry.
|
||||
* never produce. Give it a larger default assertion budget and one more retry. A
|
||||
* retried pass still fails CI (`failOnFlakyTests` in the mock profile); the extra
|
||||
* attempt only shows how often the failure repeats.
|
||||
*/
|
||||
retries: process.env.CI ? 3 : 0,
|
||||
expect: { ...mockConfig.expect, timeout: 20_000 },
|
||||
|
|
|
|||
|
|
@ -23,10 +23,12 @@ export default defineConfig({
|
|||
forbidOnly: !!process.env.CI,
|
||||
/* Retry on CI only */
|
||||
retries: process.env.CI ? 2 : 0,
|
||||
/* A test that passes only on a retry fails the run on CI; the last reporter names it. */
|
||||
failOnFlakyTests: !!process.env.CI,
|
||||
/* Opt out of parallel tests on CI. */
|
||||
workers: process.env.CI ? 1 : undefined,
|
||||
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
|
||||
reporter: [['html', { outputFolder: 'playwright-report' }]],
|
||||
reporter: [['html', { outputFolder: 'playwright-report' }], ['./reporters/retried.mjs']],
|
||||
testIgnore: ['**/mock/**'],
|
||||
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
|
||||
use: {
|
||||
|
|
|
|||
116
e2e/reporters/retried.mjs
Normal file
116
e2e/reporters/retried.mjs
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
/**
|
||||
* Playwright reporter that explains a run failed by a retried pass.
|
||||
*
|
||||
* With `failOnFlakyTests`, a test that fails and then passes on a retry fails the run. Playwright
|
||||
* still prints it under "N flaky" beside the passed count and exits 1, which reads as noise to
|
||||
* re-run. This names each such test in the job log and in the job summary
|
||||
* (`$GITHUB_STEP_SUMMARY`) and says that the retried pass is what failed. It reports nothing when
|
||||
* the option is off, as in a local run, or when no test needed a retry.
|
||||
*/
|
||||
import { appendFileSync } from 'node:fs';
|
||||
import { basename, relative, sep } from 'node:path';
|
||||
import { stripVTControlCharacters } from 'node:util';
|
||||
|
||||
const ERROR_LIMIT = 240;
|
||||
|
||||
const REASON =
|
||||
'Each test below failed on an earlier attempt and passed on a retry. In CI a retried pass ' +
|
||||
'fails the run (`failOnFlakyTests`): a test that needs a retry fails some of the time, so fix ' +
|
||||
'it like any other failure instead of re-running the job.';
|
||||
const ONLY_CAUSE = 'No test failed every attempt, so these retried passes are why the run failed.';
|
||||
const OTHER_CAUSES = 'The run failed for other reasons as well; the job log has them.';
|
||||
|
||||
/** Inline code that a backtick or a line break in the text cannot close early. */
|
||||
const code = (text) => `\`${text.replace(/\s+/g, ' ').replaceAll('`', "'")}\``;
|
||||
|
||||
/** The first line of an attempt's error, without colors, short enough for one list item. */
|
||||
function firstLine(message = '') {
|
||||
const line =
|
||||
stripVTControlCharacters(message)
|
||||
.split('\n')
|
||||
.find((text) => text.trim() !== '') ?? '';
|
||||
const text = line.trim();
|
||||
return text.length > ERROR_LIMIT ? `${text.slice(0, ERROR_LIMIT)}…` : text;
|
||||
}
|
||||
|
||||
function describe(test, root) {
|
||||
const [, project, , ...titles] = test.titlePath();
|
||||
const file = relative(root, test.location.file).split(sep).join('/');
|
||||
const location = `${file}:${test.location.line}`;
|
||||
return {
|
||||
name: [...(project ? [`[${project}]`] : []), location, ...titles].join(' › '),
|
||||
attempts: test.results.map((result) => result.status),
|
||||
error: firstLine(test.results.find((result) => result.error)?.error.message),
|
||||
};
|
||||
}
|
||||
|
||||
/** The tests that passed only on a retry, and how many failed every attempt. */
|
||||
export function collect(tests, root = process.cwd()) {
|
||||
const retried = [];
|
||||
let failed = 0;
|
||||
for (const test of tests) {
|
||||
const outcome = test.outcome();
|
||||
if (outcome === 'flaky') {
|
||||
retried.push(describe(test, root));
|
||||
} else if (outcome === 'unexpected') {
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
return { retried, failed };
|
||||
}
|
||||
|
||||
/** Markdown that also reads as plain text, so the job log and the job summary share it. */
|
||||
export function render({ config, retried, only }) {
|
||||
const count = `${retried.length} test${retried.length === 1 ? '' : 's'}`;
|
||||
const items = retried.map(({ name, attempts, error }) => {
|
||||
const cause = error ? ` First error: ${code(error)}` : '';
|
||||
return `- ${code(name)}: ${attempts.join(', ')}.${cause}`;
|
||||
});
|
||||
return [
|
||||
`### Playwright: ${count} passed only on a retry (${config})`,
|
||||
'',
|
||||
`${REASON} ${only ? ONLY_CAUSE : OTHER_CAUSES}`,
|
||||
'',
|
||||
...items,
|
||||
'',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
export default class RetriedReporter {
|
||||
#config;
|
||||
#suite;
|
||||
#errors = 0;
|
||||
|
||||
onBegin(config, suite) {
|
||||
this.#config = config;
|
||||
this.#suite = suite;
|
||||
}
|
||||
|
||||
onError() {
|
||||
this.#errors += 1;
|
||||
}
|
||||
|
||||
onEnd(result) {
|
||||
if (!this.#config?.failOnFlakyTests) {
|
||||
return;
|
||||
}
|
||||
const { retried, failed } = collect(this.#suite.allTests());
|
||||
if (retried.length === 0) {
|
||||
return;
|
||||
}
|
||||
const text = render({
|
||||
config: basename(this.#config.configFile ?? 'playwright'),
|
||||
retried,
|
||||
only: result.status === 'failed' && failed === 0 && this.#errors === 0,
|
||||
});
|
||||
console.log(`\n${text}`);
|
||||
if (process.env.GITHUB_STEP_SUMMARY) {
|
||||
appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${text}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Playwright adds its own terminal reporter to a run whose reporters all answer false. */
|
||||
printsToStdio() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
192
e2e/reporters/retried.test.mjs
Normal file
192
e2e/reporters/retried.test.mjs
Normal file
|
|
@ -0,0 +1,192 @@
|
|||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { join, resolve } from 'node:path';
|
||||
import { createRequire } from 'node:module';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { collect, render } from './retried.mjs';
|
||||
|
||||
const root = resolve(fileURLToPath(new URL('../..', import.meta.url)));
|
||||
const reporter = fileURLToPath(new URL('./retried.mjs', import.meta.url));
|
||||
const cli = createRequire(import.meta.url).resolve('@playwright/test/cli');
|
||||
|
||||
const SPEC = `
|
||||
import { test, expect } from '@playwright/test';
|
||||
|
||||
test('steady', () => {
|
||||
expect(1).toBe(1);
|
||||
});
|
||||
|
||||
test.describe('group', () => {
|
||||
test('recovers \`late\`', ({}, testInfo) => {
|
||||
test.skip(process.env.RETRIED_RECOVERS !== 'true');
|
||||
expect(testInfo.retry, 'first attempt fails').toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
test('broken', () => {
|
||||
test.skip(process.env.RETRIED_BROKEN !== 'true');
|
||||
expect(1).toBe(2);
|
||||
});
|
||||
`;
|
||||
|
||||
const CONFIG = `
|
||||
export default {
|
||||
testDir: '.',
|
||||
outputDir: 'results',
|
||||
workers: 1,
|
||||
retries: 1,
|
||||
failOnFlakyTests: process.env.RETRIED_STRICT === 'true',
|
||||
reporter: [[process.env.RETRIED_REPORTER]],
|
||||
};
|
||||
`;
|
||||
|
||||
/** Runs real Playwright over a three-test project and returns its exit, output and job summary. */
|
||||
function run(t, { strict, recovers = true, broken = false }) {
|
||||
const generated = join(root, 'e2e', '.generated');
|
||||
mkdirSync(generated, { recursive: true });
|
||||
const dir = mkdtempSync(join(generated, 'retried-'));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
writeFileSync(join(dir, 'probe.spec.mjs'), SPEC);
|
||||
writeFileSync(join(dir, 'playwright.config.mjs'), CONFIG);
|
||||
const summary = join(dir, 'summary.md');
|
||||
const child = spawnSync(
|
||||
process.execPath,
|
||||
[cli, 'test', '--config', join(dir, 'playwright.config.mjs')],
|
||||
{
|
||||
cwd: root,
|
||||
encoding: 'utf8',
|
||||
env: {
|
||||
...process.env,
|
||||
FORCE_COLOR: '0',
|
||||
GITHUB_STEP_SUMMARY: summary,
|
||||
RETRIED_REPORTER: reporter,
|
||||
RETRIED_STRICT: String(strict),
|
||||
RETRIED_RECOVERS: String(recovers),
|
||||
RETRIED_BROKEN: String(broken),
|
||||
},
|
||||
},
|
||||
);
|
||||
return {
|
||||
status: child.status,
|
||||
stdout: child.stdout,
|
||||
summary: existsSync(summary) ? readFileSync(summary, 'utf8') : '',
|
||||
};
|
||||
}
|
||||
|
||||
const NAME = /`e2e\/\.generated\/retried-[^/]+\/probe\.spec\.mjs:9 › group › recovers 'late'`/;
|
||||
|
||||
test('a retried pass fails a strict run and the job summary says why', (t) => {
|
||||
const { status, stdout, summary } = run(t, { strict: true });
|
||||
assert.equal(status, 1, stdout);
|
||||
assert.match(
|
||||
summary,
|
||||
/^### Playwright: 1 test passed only on a retry \(playwright\.config\.mjs\)$/m,
|
||||
);
|
||||
assert.match(summary, NAME);
|
||||
assert.match(summary, /: failed, passed\. First error: `Error: first attempt fails`$/m);
|
||||
assert.match(summary, /In CI a retried pass fails the run \(`failOnFlakyTests`\)/);
|
||||
assert.match(summary, /these retried passes are why the run failed/);
|
||||
assert.ok(stdout.includes(summary.trimEnd()), 'the job log carries the same explanation');
|
||||
assert.match(stdout, /^ {2}1 flaky$/m, "Playwright's own terminal reporter still runs");
|
||||
});
|
||||
|
||||
test('a retried pass beside a test that failed every attempt is not called the only cause', (t) => {
|
||||
const { status, summary } = run(t, { strict: true, broken: true });
|
||||
assert.equal(status, 1);
|
||||
assert.match(summary, NAME);
|
||||
assert.doesNotMatch(summary, /broken/);
|
||||
assert.match(summary, /The run failed for other reasons as well/);
|
||||
});
|
||||
|
||||
test('a lenient run passes a retried pass and reports nothing', (t) => {
|
||||
const { status, stdout, summary } = run(t, { strict: false });
|
||||
assert.equal(status, 0, stdout);
|
||||
assert.equal(summary, '');
|
||||
assert.doesNotMatch(stdout, /passed only on a retry/);
|
||||
});
|
||||
|
||||
test('a strict run with no retried pass reports nothing', (t) => {
|
||||
const clean = run(t, { strict: true, recovers: false });
|
||||
assert.equal(clean.status, 0, clean.stdout);
|
||||
assert.equal(clean.summary, '');
|
||||
const broken = run(t, { strict: true, recovers: false, broken: true });
|
||||
assert.equal(broken.status, 1);
|
||||
assert.equal(broken.summary, '');
|
||||
});
|
||||
|
||||
/** The fields of a Playwright `TestCase` the reporter reads. */
|
||||
function testCase({ outcome, titles = ['title'], project = 'chrome', line = 1, results = [] }) {
|
||||
return {
|
||||
outcome: () => outcome,
|
||||
titlePath: () => ['', project, 'file.spec.ts', ...titles],
|
||||
location: { file: join(root, 'e2e', 'specs', 'mock', 'file.spec.ts'), line, column: 1 },
|
||||
results,
|
||||
};
|
||||
}
|
||||
|
||||
const attempt = (status, message) => ({ status, ...(message ? { error: { message } } : {}) });
|
||||
|
||||
test('collect keeps retried passes in order and only counts the tests that failed outright', () => {
|
||||
const { retried, failed } = collect(
|
||||
[
|
||||
testCase({ outcome: 'expected', results: [attempt('passed')] }),
|
||||
testCase({
|
||||
outcome: 'flaky',
|
||||
line: 355,
|
||||
titles: ['parity', 'late open'],
|
||||
results: [
|
||||
attempt('timedOut', '\u001b[31mTest timeout of 30000ms exceeded.\u001b[39m'),
|
||||
attempt('failed', 'Error: second'),
|
||||
attempt('passed'),
|
||||
],
|
||||
}),
|
||||
testCase({ outcome: 'unexpected', results: [attempt('failed', 'Error: always')] }),
|
||||
testCase({ outcome: 'skipped' }),
|
||||
testCase({
|
||||
outcome: 'flaky',
|
||||
project: '',
|
||||
results: [
|
||||
attempt('failed', `\n\n Error: ${'x'.repeat(400)}\n at spec.ts:1`),
|
||||
attempt('passed'),
|
||||
],
|
||||
}),
|
||||
],
|
||||
root,
|
||||
);
|
||||
assert.equal(failed, 1);
|
||||
assert.deepEqual(retried[0], {
|
||||
name: '[chrome] › e2e/specs/mock/file.spec.ts:355 › parity › late open',
|
||||
attempts: ['timedOut', 'failed', 'passed'],
|
||||
error: 'Test timeout of 30000ms exceeded.',
|
||||
});
|
||||
assert.equal(retried[1].name, 'e2e/specs/mock/file.spec.ts:1 › title');
|
||||
assert.equal(retried[1].error, `Error: ${'x'.repeat(233)}…`);
|
||||
assert.equal(retried.length, 2);
|
||||
});
|
||||
|
||||
test('render counts in the plural and keeps a title or an error inside its code span', () => {
|
||||
const text = render({
|
||||
config: 'playwright.config.redis.ts',
|
||||
only: true,
|
||||
retried: [
|
||||
{
|
||||
name: 'a.spec.ts:1 › shows `code`\nacross lines',
|
||||
attempts: ['failed', 'passed'],
|
||||
error: 'a `b` | c',
|
||||
},
|
||||
{ name: 'b.spec.ts:2 › no error', attempts: ['interrupted', 'failed', 'passed'], error: '' },
|
||||
],
|
||||
});
|
||||
assert.match(
|
||||
text,
|
||||
/^### Playwright: 2 tests passed only on a retry \(playwright\.config\.redis\.ts\)$/m,
|
||||
);
|
||||
assert.ok(
|
||||
text.includes(
|
||||
"- `a.spec.ts:1 › shows 'code' across lines`: failed, passed. First error: `a 'b' | c`\n",
|
||||
),
|
||||
);
|
||||
assert.ok(text.includes('- `b.spec.ts:2 › no error`: interrupted, failed, passed.\n'));
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue