🎲 ci: Fail Playwright Runs When a Test Passes Only on a Retry (#18051)

Playwright reports a test that fails and then passes on a retry as "flaky" and exits 0, so a
regression that fails two attempts in three merges green. Set `failOnFlakyTests` in CI on the
mock profile (inherited by the Redis transport, registered-email and other derived profiles) and
on the base profile. Retry counts are unchanged: the later attempts still record the trace and
video and show whether a failure repeats.

A run failed this way still prints "N flaky" beside a passed count, so add a reporter that names
each retried pass in the job log and the job summary and says that it is what failed the run.
This commit is contained in:
Danny Avila 2026-11-11 10:31:06 -05:00 • committed by GitHub
parent 97f7fcdbac
commit c6b7a5ac12
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 335 additions and 7 deletions

View file

@ -245,8 +245,8 @@ jobs:
if: steps.cache-node-modules.outputs.cache-hit != 'true'
run: npm ci
- name: Test Playwright matrix selection and the vote gate
run: node --test .github/scripts/playwright-matrix.test.mjs .github/scripts/playwright-shards.test.mjs .github/scripts/vote-gate.test.mjs
- name: Test Playwright matrix selection, the vote gate and the retried-pass reporter
run: node --test .github/scripts/playwright-matrix.test.mjs .github/scripts/playwright-shards.test.mjs .github/scripts/vote-gate.test.mjs e2e/reporters/retried.test.mjs
- name: Restore data-provider build cache
id: cache-data-provider
@ -440,7 +440,8 @@ jobs:
E2E_TITLE_CONVO: 'true'
run: >-
npx playwright test --config=e2e/playwright.config.mock.ts
conversation-management.spec.ts --grep 'first-turn|final-title polling' --reporter=line
conversation-management.spec.ts --grep 'first-turn|final-title polling'
--reporter=line,e2e/reporters/retried.mjs
--output=e2e/specs/.test-results/title-ownership
- name: Upload title ownership diagnostics

View file

@ -101,6 +101,11 @@ The nightly schedule and manual workflow dispatch run the complete mock suite in
modes across ten shards per mode. Every shard keeps one worker so tests do not contend for its
authenticated user and database.
CI retries a failed test, twice on the memory shards and three times on the Redis transport
suite, so the report carries a trace of each failed attempt and shows whether the failure
repeats. A test that passes only on a retry still fails the run (`failOnFlakyTests`), and the
job log and the job summary name it (`e2e/reporters/retried.mjs`). Local runs do not retry.
CI splits the complete suite by recorded duration rather than test count: each shard lists the
suite, and `.github/scripts/playwright-shards.mjs plan` hands out whole spec files longest-first
to the least-loaded shard using `e2e/timings.mock.json`. A spec with no recorded time is

View file

@ -53,7 +53,11 @@ export default defineConfig({
outputDir: `specs/.test-results/email-change${profileSuffix}`,
fullyParallel: false,
workers: 1,
reporter: [['html', { outputFolder: reportPath, open: 'never' }], ['list']],
reporter: [
['html', { outputFolder: reportPath, open: 'never' }],
['list'],
['./reporters/retried.mjs'],
],
use: {
...mockConfig.use,
storageState: path.resolve(rootPath, `e2e/.generated/storageState.email${profileSuffix}.json`),

View file

@ -524,15 +524,21 @@ export default defineConfig({
fullyParallel: true,
forbidOnly: !!process.env.CI,
retries: process.env.CI ? 2 : 0,
/** In CI a test that fails and then passes on a retry fails the run. The retries stay for
* what they show: a trace of each failed attempt, video of the first retry, and whether the
* failure is deterministic. Derived profiles inherit this with the rest of the config. */
failOnFlakyTests: !!process.env.CI,
workers: 1,
/** In CI the JSON results land beside the HTML report (reporters end in order, after the HTML
* reporter has replaced its folder), so the report artifact also carries the per-file
* timings `.github/scripts/playwright-shards.mjs record` rebuilds the shard plan from. */
* timings `.github/scripts/playwright-shards.mjs record` rebuilds the shard plan from.
* The last reporter names each retried pass in the job log and the job summary. */
reporter: process.env.CI
? [
['html', { outputFolder: reportPath, open: 'never' }],
['line'],
['json', { outputFile: path.join(reportPath, 'results.json') }],
['./reporters/retried.mjs'],
]
: [['html', { outputFolder: reportPath }], ['list']],
use: {

View file

@ -10,7 +10,9 @@ export default defineConfig({
* pause/resume and rehydrate paths most of all, since they replay a whole job.
* The suite is serial (`workers: 1`), so this is per-operation latency rather
* than contention, and it is why this lane reports flaky runs the memory shards
* never produce. Give it a larger default assertion budget and one more retry.
* never produce. Give it a larger default assertion budget and one more retry. A
* retried pass still fails CI (`failOnFlakyTests` in the mock profile); the extra
* attempt only shows how often the failure repeats.
*/
retries: process.env.CI ? 3 : 0,
expect: { ...mockConfig.expect, timeout: 20_000 },

View file

@ -23,10 +23,12 @@ export default defineConfig({
forbidOnly: !!process.env.CI,
/* Retry on CI only */
retries: process.env.CI ? 2 : 0,
/* A test that passes only on a retry fails the run on CI; the last reporter names it. */
failOnFlakyTests: !!process.env.CI,
/* Opt out of parallel tests on CI. */
workers: process.env.CI ? 1 : undefined,
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
reporter: [['html', { outputFolder: 'playwright-report' }]],
reporter: [['html', { outputFolder: 'playwright-report' }], ['./reporters/retried.mjs']],
testIgnore: ['**/mock/**'],
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
use: {

116
e2e/reporters/retried.mjs Normal file
View file

@ -0,0 +1,116 @@
/**
* Playwright reporter that explains a run failed by a retried pass.
*
* With `failOnFlakyTests`, a test that fails and then passes on a retry fails the run. Playwright
* still prints it under "N flaky" beside the passed count and exits 1, which reads as noise to
* re-run. This names each such test in the job log and in the job summary
* (`$GITHUB_STEP_SUMMARY`) and says that the retried pass is what failed. It reports nothing when
* the option is off, as in a local run, or when no test needed a retry.
*/
import { appendFileSync } from 'node:fs';
import { basename, relative, sep } from 'node:path';
import { stripVTControlCharacters } from 'node:util';
const ERROR_LIMIT = 240;
const REASON =
'Each test below failed on an earlier attempt and passed on a retry. In CI a retried pass ' +
'fails the run (`failOnFlakyTests`): a test that needs a retry fails some of the time, so fix ' +
'it like any other failure instead of re-running the job.';
const ONLY_CAUSE = 'No test failed every attempt, so these retried passes are why the run failed.';
const OTHER_CAUSES = 'The run failed for other reasons as well; the job log has them.';
/** Inline code that a backtick or a line break in the text cannot close early. */
const code = (text) => `\`${text.replace(/\s+/g, ' ').replaceAll('`', "'")}\``;
/** The first line of an attempt's error, without colors, short enough for one list item. */
function firstLine(message = '') {
const line =
stripVTControlCharacters(message)
.split('\n')
.find((text) => text.trim() !== '') ?? '';
const text = line.trim();
return text.length > ERROR_LIMIT ? `${text.slice(0, ERROR_LIMIT)}…` : text;
}
function describe(test, root) {
const [, project, , ...titles] = test.titlePath();
const file = relative(root, test.location.file).split(sep).join('/');
const location = `${file}:${test.location.line}`;
return {
name: [...(project ? [`[${project}]`] : []), location, ...titles].join(' › '),
attempts: test.results.map((result) => result.status),
error: firstLine(test.results.find((result) => result.error)?.error.message),
};
}
/** The tests that passed only on a retry, and how many failed every attempt. */
export function collect(tests, root = process.cwd()) {
const retried = [];
let failed = 0;
for (const test of tests) {
const outcome = test.outcome();
if (outcome === 'flaky') {
retried.push(describe(test, root));
} else if (outcome === 'unexpected') {
failed += 1;
}
}
return { retried, failed };
}
/** Markdown that also reads as plain text, so the job log and the job summary share it. */
export function render({ config, retried, only }) {
const count = `${retried.length} test${retried.length === 1 ? '' : 's'}`;
const items = retried.map(({ name, attempts, error }) => {
const cause = error ? ` First error: ${code(error)}` : '';
return `- ${code(name)}: ${attempts.join(', ')}.${cause}`;
});
return [
`### Playwright: ${count} passed only on a retry (${config})`,
'',
`${REASON} ${only ? ONLY_CAUSE : OTHER_CAUSES}`,
'',
...items,
'',
].join('\n');
}
export default class RetriedReporter {
#config;
#suite;
#errors = 0;
onBegin(config, suite) {
this.#config = config;
this.#suite = suite;
}
onError() {
this.#errors += 1;
}
onEnd(result) {
if (!this.#config?.failOnFlakyTests) {
return;
}
const { retried, failed } = collect(this.#suite.allTests());
if (retried.length === 0) {
return;
}
const text = render({
config: basename(this.#config.configFile ?? 'playwright'),
retried,
only: result.status === 'failed' && failed === 0 && this.#errors === 0,
});
console.log(`\n${text}`);
if (process.env.GITHUB_STEP_SUMMARY) {
appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${text}\n`);
}
}
/** Playwright adds its own terminal reporter to a run whose reporters all answer false. */
printsToStdio() {
return false;
}
}

View file

@ -0,0 +1,192 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { fileURLToPath } from 'node:url';
import { join, resolve } from 'node:path';
import { createRequire } from 'node:module';
import { spawnSync } from 'node:child_process';
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { collect, render } from './retried.mjs';
const root = resolve(fileURLToPath(new URL('../..', import.meta.url)));
const reporter = fileURLToPath(new URL('./retried.mjs', import.meta.url));
const cli = createRequire(import.meta.url).resolve('@playwright/test/cli');
const SPEC = `
import { test, expect } from '@playwright/test';
test('steady', () => {
expect(1).toBe(1);
});
test.describe('group', () => {
test('recovers \`late\`', ({}, testInfo) => {
test.skip(process.env.RETRIED_RECOVERS !== 'true');
expect(testInfo.retry, 'first attempt fails').toBe(1);
});
});
test('broken', () => {
test.skip(process.env.RETRIED_BROKEN !== 'true');
expect(1).toBe(2);
});
`;
const CONFIG = `
export default {
testDir: '.',
outputDir: 'results',
workers: 1,
retries: 1,
failOnFlakyTests: process.env.RETRIED_STRICT === 'true',
reporter: [[process.env.RETRIED_REPORTER]],
};
`;
/** Runs real Playwright over a three-test project and returns its exit, output and job summary. */
function run(t, { strict, recovers = true, broken = false }) {
const generated = join(root, 'e2e', '.generated');
mkdirSync(generated, { recursive: true });
const dir = mkdtempSync(join(generated, 'retried-'));
t.after(() => rmSync(dir, { recursive: true, force: true }));
writeFileSync(join(dir, 'probe.spec.mjs'), SPEC);
writeFileSync(join(dir, 'playwright.config.mjs'), CONFIG);
const summary = join(dir, 'summary.md');
const child = spawnSync(
process.execPath,
[cli, 'test', '--config', join(dir, 'playwright.config.mjs')],
{
cwd: root,
encoding: 'utf8',
env: {
...process.env,
FORCE_COLOR: '0',
GITHUB_STEP_SUMMARY: summary,
RETRIED_REPORTER: reporter,
RETRIED_STRICT: String(strict),
RETRIED_RECOVERS: String(recovers),
RETRIED_BROKEN: String(broken),
},
},
);
return {
status: child.status,
stdout: child.stdout,
summary: existsSync(summary) ? readFileSync(summary, 'utf8') : '',
};
}
const NAME = /`e2e\/\.generated\/retried-[^/]+\/probe\.spec\.mjs:9 › group › recovers 'late'`/;
test('a retried pass fails a strict run and the job summary says why', (t) => {
const { status, stdout, summary } = run(t, { strict: true });
assert.equal(status, 1, stdout);
assert.match(
summary,
/^### Playwright: 1 test passed only on a retry \(playwright\.config\.mjs\)$/m,
);
assert.match(summary, NAME);
assert.match(summary, /: failed, passed\. First error: `Error: first attempt fails`$/m);
assert.match(summary, /In CI a retried pass fails the run \(`failOnFlakyTests`\)/);
assert.match(summary, /these retried passes are why the run failed/);
assert.ok(stdout.includes(summary.trimEnd()), 'the job log carries the same explanation');
assert.match(stdout, /^ {2}1 flaky$/m, "Playwright's own terminal reporter still runs");
});
test('a retried pass beside a test that failed every attempt is not called the only cause', (t) => {
const { status, summary } = run(t, { strict: true, broken: true });
assert.equal(status, 1);
assert.match(summary, NAME);
assert.doesNotMatch(summary, /broken/);
assert.match(summary, /The run failed for other reasons as well/);
});
test('a lenient run passes a retried pass and reports nothing', (t) => {
const { status, stdout, summary } = run(t, { strict: false });
assert.equal(status, 0, stdout);
assert.equal(summary, '');
assert.doesNotMatch(stdout, /passed only on a retry/);
});
test('a strict run with no retried pass reports nothing', (t) => {
const clean = run(t, { strict: true, recovers: false });
assert.equal(clean.status, 0, clean.stdout);
assert.equal(clean.summary, '');
const broken = run(t, { strict: true, recovers: false, broken: true });
assert.equal(broken.status, 1);
assert.equal(broken.summary, '');
});
/** The fields of a Playwright `TestCase` the reporter reads. */
function testCase({ outcome, titles = ['title'], project = 'chrome', line = 1, results = [] }) {
return {
outcome: () => outcome,
titlePath: () => ['', project, 'file.spec.ts', ...titles],
location: { file: join(root, 'e2e', 'specs', 'mock', 'file.spec.ts'), line, column: 1 },
results,
};
}
const attempt = (status, message) => ({ status, ...(message ? { error: { message } } : {}) });
test('collect keeps retried passes in order and only counts the tests that failed outright', () => {
const { retried, failed } = collect(
[
testCase({ outcome: 'expected', results: [attempt('passed')] }),
testCase({
outcome: 'flaky',
line: 355,
titles: ['parity', 'late open'],
results: [
attempt('timedOut', '\u001b[31mTest timeout of 30000ms exceeded.\u001b[39m'),
attempt('failed', 'Error: second'),
attempt('passed'),
],
}),
testCase({ outcome: 'unexpected', results: [attempt('failed', 'Error: always')] }),
testCase({ outcome: 'skipped' }),
testCase({
outcome: 'flaky',
project: '',
results: [
attempt('failed', `\n\n Error: ${'x'.repeat(400)}\n at spec.ts:1`),
attempt('passed'),
],
}),
],
root,
);
assert.equal(failed, 1);
assert.deepEqual(retried[0], {
name: '[chrome] › e2e/specs/mock/file.spec.ts:355 › parity › late open',
attempts: ['timedOut', 'failed', 'passed'],
error: 'Test timeout of 30000ms exceeded.',
});
assert.equal(retried[1].name, 'e2e/specs/mock/file.spec.ts:1 › title');
assert.equal(retried[1].error, `Error: ${'x'.repeat(233)}…`);
assert.equal(retried.length, 2);
});
test('render counts in the plural and keeps a title or an error inside its code span', () => {
const text = render({
config: 'playwright.config.redis.ts',
only: true,
retried: [
{
name: 'a.spec.ts:1 › shows `code`\nacross lines',
attempts: ['failed', 'passed'],
error: 'a `b` | c',
},
{ name: 'b.spec.ts:2 › no error', attempts: ['interrupted', 'failed', 'passed'], error: '' },
],
});
assert.match(
text,
/^### Playwright: 2 tests passed only on a retry \(playwright\.config\.redis\.ts\)$/m,
);
assert.ok(
text.includes(
"- `a.spec.ts:1 › shows 'code' across lines`: failed, passed. First error: `a 'b' | c`\n",
),
);
assert.ok(text.includes('- `b.spec.ts:2 › no error`: interrupted, failed, passed.\n'));
});