From c6b7a5ac12270dffa63193ad60e73fddb4efea01 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 11 Nov 2026 10:31:06 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=B2=20ci:=20Fail=20Playwright=20Runs?= =?UTF-8?q?=20When=20a=20Test=20Passes=20Only=20on=20a=20Retry=20(#18051)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Playwright reports a test that fails and then passes on a retry as "flaky" and exits 0, so a regression that fails two attempts in three merges green. Set `failOnFlakyTests` in CI on the mock profile (inherited by the Redis transport, registered-email and other derived profiles) and on the base profile. Retry counts are unchanged: the later attempts still record the trace and video and show whether a failure repeats. A run failed this way still prints "N flaky" beside a passed count, so add a reporter that names each retried pass in the job log and the job summary and says that it is what failed the run. --- .github/workflows/playwright-mock.yml | 7 +- e2e/README.md | 5 + e2e/playwright.config.email.ts | 6 +- e2e/playwright.config.mock.ts | 8 +- e2e/playwright.config.redis.ts | 4 +- e2e/playwright.config.ts | 4 +- e2e/reporters/retried.mjs | 116 ++++++++++++++++ e2e/reporters/retried.test.mjs | 192 ++++++++++++++++++++++++++ 8 files changed, 335 insertions(+), 7 deletions(-) create mode 100644 e2e/reporters/retried.mjs create mode 100644 e2e/reporters/retried.test.mjs diff --git a/.github/workflows/playwright-mock.yml b/.github/workflows/playwright-mock.yml index 13a3c64af3..56cb484d7f 100644 --- a/.github/workflows/playwright-mock.yml +++ b/.github/workflows/playwright-mock.yml @@ -245,8 +245,8 @@ jobs: if: steps.cache-node-modules.outputs.cache-hit != 'true' run: npm ci - - name: Test Playwright matrix selection and the vote gate - run: node --test .github/scripts/playwright-matrix.test.mjs .github/scripts/playwright-shards.test.mjs .github/scripts/vote-gate.test.mjs + - name: Test Playwright matrix selection, the vote gate and the retried-pass reporter + run: node --test .github/scripts/playwright-matrix.test.mjs .github/scripts/playwright-shards.test.mjs .github/scripts/vote-gate.test.mjs e2e/reporters/retried.test.mjs - name: Restore data-provider build cache id: cache-data-provider @@ -440,7 +440,8 @@ jobs: E2E_TITLE_CONVO: 'true' run: >- npx playwright test --config=e2e/playwright.config.mock.ts - conversation-management.spec.ts --grep 'first-turn|final-title polling' --reporter=line + conversation-management.spec.ts --grep 'first-turn|final-title polling' + --reporter=line,e2e/reporters/retried.mjs --output=e2e/specs/.test-results/title-ownership - name: Upload title ownership diagnostics diff --git a/e2e/README.md b/e2e/README.md index 832d64a002..4c59b0d7f2 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -101,6 +101,11 @@ The nightly schedule and manual workflow dispatch run the complete mock suite in modes across ten shards per mode. Every shard keeps one worker so tests do not contend for its authenticated user and database. +CI retries a failed test, twice on the memory shards and three times on the Redis transport +suite, so the report carries a trace of each failed attempt and shows whether the failure +repeats. A test that passes only on a retry still fails the run (`failOnFlakyTests`), and the +job log and the job summary name it (`e2e/reporters/retried.mjs`). Local runs do not retry. + CI splits the complete suite by recorded duration rather than test count: each shard lists the suite, and `.github/scripts/playwright-shards.mjs plan` hands out whole spec files longest-first to the least-loaded shard using `e2e/timings.mock.json`. A spec with no recorded time is diff --git a/e2e/playwright.config.email.ts b/e2e/playwright.config.email.ts index 1f0fbca8a3..3b744ff7f6 100644 --- a/e2e/playwright.config.email.ts +++ b/e2e/playwright.config.email.ts @@ -53,7 +53,11 @@ export default defineConfig({ outputDir: `specs/.test-results/email-change${profileSuffix}`, fullyParallel: false, workers: 1, - reporter: [['html', { outputFolder: reportPath, open: 'never' }], ['list']], + reporter: [ + ['html', { outputFolder: reportPath, open: 'never' }], + ['list'], + ['./reporters/retried.mjs'], + ], use: { ...mockConfig.use, storageState: path.resolve(rootPath, `e2e/.generated/storageState.email${profileSuffix}.json`), diff --git a/e2e/playwright.config.mock.ts b/e2e/playwright.config.mock.ts index f5945d6167..16d0f0af04 100644 --- a/e2e/playwright.config.mock.ts +++ b/e2e/playwright.config.mock.ts @@ -524,15 +524,21 @@ export default defineConfig({ fullyParallel: true, forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 0, + /** In CI a test that fails and then passes on a retry fails the run. The retries stay for + * what they show: a trace of each failed attempt, video of the first retry, and whether the + * failure is deterministic. Derived profiles inherit this with the rest of the config. */ + failOnFlakyTests: !!process.env.CI, workers: 1, /** In CI the JSON results land beside the HTML report (reporters end in order, after the HTML * reporter has replaced its folder), so the report artifact also carries the per-file - * timings `.github/scripts/playwright-shards.mjs record` rebuilds the shard plan from. */ + * timings `.github/scripts/playwright-shards.mjs record` rebuilds the shard plan from. + * The last reporter names each retried pass in the job log and the job summary. */ reporter: process.env.CI ? [ ['html', { outputFolder: reportPath, open: 'never' }], ['line'], ['json', { outputFile: path.join(reportPath, 'results.json') }], + ['./reporters/retried.mjs'], ] : [['html', { outputFolder: reportPath }], ['list']], use: { diff --git a/e2e/playwright.config.redis.ts b/e2e/playwright.config.redis.ts index 6b99afe50b..51f787b8b6 100644 --- a/e2e/playwright.config.redis.ts +++ b/e2e/playwright.config.redis.ts @@ -10,7 +10,9 @@ export default defineConfig({ * pause/resume and rehydrate paths most of all, since they replay a whole job. * The suite is serial (`workers: 1`), so this is per-operation latency rather * than contention, and it is why this lane reports flaky runs the memory shards - * never produce. Give it a larger default assertion budget and one more retry. + * never produce. Give it a larger default assertion budget and one more retry. A + * retried pass still fails CI (`failOnFlakyTests` in the mock profile); the extra + * attempt only shows how often the failure repeats. */ retries: process.env.CI ? 3 : 0, expect: { ...mockConfig.expect, timeout: 20_000 }, diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts index e62856c2d3..02b53d4acd 100644 --- a/e2e/playwright.config.ts +++ b/e2e/playwright.config.ts @@ -23,10 +23,12 @@ export default defineConfig({ forbidOnly: !!process.env.CI, /* Retry on CI only */ retries: process.env.CI ? 2 : 0, + /* A test that passes only on a retry fails the run on CI; the last reporter names it. */ + failOnFlakyTests: !!process.env.CI, /* Opt out of parallel tests on CI. */ workers: process.env.CI ? 1 : undefined, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: [['html', { outputFolder: 'playwright-report' }]], + reporter: [['html', { outputFolder: 'playwright-report' }], ['./reporters/retried.mjs']], testIgnore: ['**/mock/**'], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { diff --git a/e2e/reporters/retried.mjs b/e2e/reporters/retried.mjs new file mode 100644 index 0000000000..ab2a5a75b5 --- /dev/null +++ b/e2e/reporters/retried.mjs @@ -0,0 +1,116 @@ +/** + * Playwright reporter that explains a run failed by a retried pass. + * + * With `failOnFlakyTests`, a test that fails and then passes on a retry fails the run. Playwright + * still prints it under "N flaky" beside the passed count and exits 1, which reads as noise to + * re-run. This names each such test in the job log and in the job summary + * (`$GITHUB_STEP_SUMMARY`) and says that the retried pass is what failed. It reports nothing when + * the option is off, as in a local run, or when no test needed a retry. + */ +import { appendFileSync } from 'node:fs'; +import { basename, relative, sep } from 'node:path'; +import { stripVTControlCharacters } from 'node:util'; + +const ERROR_LIMIT = 240; + +const REASON = + 'Each test below failed on an earlier attempt and passed on a retry. In CI a retried pass ' + + 'fails the run (`failOnFlakyTests`): a test that needs a retry fails some of the time, so fix ' + + 'it like any other failure instead of re-running the job.'; +const ONLY_CAUSE = 'No test failed every attempt, so these retried passes are why the run failed.'; +const OTHER_CAUSES = 'The run failed for other reasons as well; the job log has them.'; + +/** Inline code that a backtick or a line break in the text cannot close early. */ +const code = (text) => `\`${text.replace(/\s+/g, ' ').replaceAll('`', "'")}\``; + +/** The first line of an attempt's error, without colors, short enough for one list item. */ +function firstLine(message = '') { + const line = + stripVTControlCharacters(message) + .split('\n') + .find((text) => text.trim() !== '') ?? ''; + const text = line.trim(); + return text.length > ERROR_LIMIT ? `${text.slice(0, ERROR_LIMIT)}…` : text; +} + +function describe(test, root) { + const [, project, , ...titles] = test.titlePath(); + const file = relative(root, test.location.file).split(sep).join('/'); + const location = `${file}:${test.location.line}`; + return { + name: [...(project ? [`[${project}]`] : []), location, ...titles].join(' › '), + attempts: test.results.map((result) => result.status), + error: firstLine(test.results.find((result) => result.error)?.error.message), + }; +} + +/** The tests that passed only on a retry, and how many failed every attempt. */ +export function collect(tests, root = process.cwd()) { + const retried = []; + let failed = 0; + for (const test of tests) { + const outcome = test.outcome(); + if (outcome === 'flaky') { + retried.push(describe(test, root)); + } else if (outcome === 'unexpected') { + failed += 1; + } + } + return { retried, failed }; +} + +/** Markdown that also reads as plain text, so the job log and the job summary share it. */ +export function render({ config, retried, only }) { + const count = `${retried.length} test${retried.length === 1 ? '' : 's'}`; + const items = retried.map(({ name, attempts, error }) => { + const cause = error ? ` First error: ${code(error)}` : ''; + return `- ${code(name)}: ${attempts.join(', ')}.${cause}`; + }); + return [ + `### Playwright: ${count} passed only on a retry (${config})`, + '', + `${REASON} ${only ? ONLY_CAUSE : OTHER_CAUSES}`, + '', + ...items, + '', + ].join('\n'); +} + +export default class RetriedReporter { + #config; + #suite; + #errors = 0; + + onBegin(config, suite) { + this.#config = config; + this.#suite = suite; + } + + onError() { + this.#errors += 1; + } + + onEnd(result) { + if (!this.#config?.failOnFlakyTests) { + return; + } + const { retried, failed } = collect(this.#suite.allTests()); + if (retried.length === 0) { + return; + } + const text = render({ + config: basename(this.#config.configFile ?? 'playwright'), + retried, + only: result.status === 'failed' && failed === 0 && this.#errors === 0, + }); + console.log(`\n${text}`); + if (process.env.GITHUB_STEP_SUMMARY) { + appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${text}\n`); + } + } + + /** Playwright adds its own terminal reporter to a run whose reporters all answer false. */ + printsToStdio() { + return false; + } +} diff --git a/e2e/reporters/retried.test.mjs b/e2e/reporters/retried.test.mjs new file mode 100644 index 0000000000..a8ecfd84bf --- /dev/null +++ b/e2e/reporters/retried.test.mjs @@ -0,0 +1,192 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { fileURLToPath } from 'node:url'; +import { join, resolve } from 'node:path'; +import { createRequire } from 'node:module'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { collect, render } from './retried.mjs'; + +const root = resolve(fileURLToPath(new URL('../..', import.meta.url))); +const reporter = fileURLToPath(new URL('./retried.mjs', import.meta.url)); +const cli = createRequire(import.meta.url).resolve('@playwright/test/cli'); + +const SPEC = ` +import { test, expect } from '@playwright/test'; + +test('steady', () => { + expect(1).toBe(1); +}); + +test.describe('group', () => { + test('recovers \`late\`', ({}, testInfo) => { + test.skip(process.env.RETRIED_RECOVERS !== 'true'); + expect(testInfo.retry, 'first attempt fails').toBe(1); + }); +}); + +test('broken', () => { + test.skip(process.env.RETRIED_BROKEN !== 'true'); + expect(1).toBe(2); +}); +`; + +const CONFIG = ` +export default { + testDir: '.', + outputDir: 'results', + workers: 1, + retries: 1, + failOnFlakyTests: process.env.RETRIED_STRICT === 'true', + reporter: [[process.env.RETRIED_REPORTER]], +}; +`; + +/** Runs real Playwright over a three-test project and returns its exit, output and job summary. */ +function run(t, { strict, recovers = true, broken = false }) { + const generated = join(root, 'e2e', '.generated'); + mkdirSync(generated, { recursive: true }); + const dir = mkdtempSync(join(generated, 'retried-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + writeFileSync(join(dir, 'probe.spec.mjs'), SPEC); + writeFileSync(join(dir, 'playwright.config.mjs'), CONFIG); + const summary = join(dir, 'summary.md'); + const child = spawnSync( + process.execPath, + [cli, 'test', '--config', join(dir, 'playwright.config.mjs')], + { + cwd: root, + encoding: 'utf8', + env: { + ...process.env, + FORCE_COLOR: '0', + GITHUB_STEP_SUMMARY: summary, + RETRIED_REPORTER: reporter, + RETRIED_STRICT: String(strict), + RETRIED_RECOVERS: String(recovers), + RETRIED_BROKEN: String(broken), + }, + }, + ); + return { + status: child.status, + stdout: child.stdout, + summary: existsSync(summary) ? readFileSync(summary, 'utf8') : '', + }; +} + +const NAME = /`e2e\/\.generated\/retried-[^/]+\/probe\.spec\.mjs:9 › group › recovers 'late'`/; + +test('a retried pass fails a strict run and the job summary says why', (t) => { + const { status, stdout, summary } = run(t, { strict: true }); + assert.equal(status, 1, stdout); + assert.match( + summary, + /^### Playwright: 1 test passed only on a retry \(playwright\.config\.mjs\)$/m, + ); + assert.match(summary, NAME); + assert.match(summary, /: failed, passed\. First error: `Error: first attempt fails`$/m); + assert.match(summary, /In CI a retried pass fails the run \(`failOnFlakyTests`\)/); + assert.match(summary, /these retried passes are why the run failed/); + assert.ok(stdout.includes(summary.trimEnd()), 'the job log carries the same explanation'); + assert.match(stdout, /^ {2}1 flaky$/m, "Playwright's own terminal reporter still runs"); +}); + +test('a retried pass beside a test that failed every attempt is not called the only cause', (t) => { + const { status, summary } = run(t, { strict: true, broken: true }); + assert.equal(status, 1); + assert.match(summary, NAME); + assert.doesNotMatch(summary, /broken/); + assert.match(summary, /The run failed for other reasons as well/); +}); + +test('a lenient run passes a retried pass and reports nothing', (t) => { + const { status, stdout, summary } = run(t, { strict: false }); + assert.equal(status, 0, stdout); + assert.equal(summary, ''); + assert.doesNotMatch(stdout, /passed only on a retry/); +}); + +test('a strict run with no retried pass reports nothing', (t) => { + const clean = run(t, { strict: true, recovers: false }); + assert.equal(clean.status, 0, clean.stdout); + assert.equal(clean.summary, ''); + const broken = run(t, { strict: true, recovers: false, broken: true }); + assert.equal(broken.status, 1); + assert.equal(broken.summary, ''); +}); + +/** The fields of a Playwright `TestCase` the reporter reads. */ +function testCase({ outcome, titles = ['title'], project = 'chrome', line = 1, results = [] }) { + return { + outcome: () => outcome, + titlePath: () => ['', project, 'file.spec.ts', ...titles], + location: { file: join(root, 'e2e', 'specs', 'mock', 'file.spec.ts'), line, column: 1 }, + results, + }; +} + +const attempt = (status, message) => ({ status, ...(message ? { error: { message } } : {}) }); + +test('collect keeps retried passes in order and only counts the tests that failed outright', () => { + const { retried, failed } = collect( + [ + testCase({ outcome: 'expected', results: [attempt('passed')] }), + testCase({ + outcome: 'flaky', + line: 355, + titles: ['parity', 'late open'], + results: [ + attempt('timedOut', '\u001b[31mTest timeout of 30000ms exceeded.\u001b[39m'), + attempt('failed', 'Error: second'), + attempt('passed'), + ], + }), + testCase({ outcome: 'unexpected', results: [attempt('failed', 'Error: always')] }), + testCase({ outcome: 'skipped' }), + testCase({ + outcome: 'flaky', + project: '', + results: [ + attempt('failed', `\n\n Error: ${'x'.repeat(400)}\n at spec.ts:1`), + attempt('passed'), + ], + }), + ], + root, + ); + assert.equal(failed, 1); + assert.deepEqual(retried[0], { + name: '[chrome] › e2e/specs/mock/file.spec.ts:355 › parity › late open', + attempts: ['timedOut', 'failed', 'passed'], + error: 'Test timeout of 30000ms exceeded.', + }); + assert.equal(retried[1].name, 'e2e/specs/mock/file.spec.ts:1 › title'); + assert.equal(retried[1].error, `Error: ${'x'.repeat(233)}…`); + assert.equal(retried.length, 2); +}); + +test('render counts in the plural and keeps a title or an error inside its code span', () => { + const text = render({ + config: 'playwright.config.redis.ts', + only: true, + retried: [ + { + name: 'a.spec.ts:1 › shows `code`\nacross lines', + attempts: ['failed', 'passed'], + error: 'a `b` | c', + }, + { name: 'b.spec.ts:2 › no error', attempts: ['interrupted', 'failed', 'passed'], error: '' }, + ], + }); + assert.match( + text, + /^### Playwright: 2 tests passed only on a retry \(playwright\.config\.redis\.ts\)$/m, + ); + assert.ok( + text.includes( + "- `a.spec.ts:1 › shows 'code' across lines`: failed, passed. First error: `a 'b' | c`\n", + ), + ); + assert.ok(text.includes('- `b.spec.ts:2 › no error`: interrupted, failed, passed.\n')); +});