From 0fdabfe283045174697bce4bebb2ab2e14f02be5 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 11 Nov 2026 17:54:25 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=AA=20test:=20Close=20Three=20Timing?= =?UTF-8?q?=20Gaps=20Behind=20Intermittent=20Mock=20E2E=20Failures=20(#180?= =?UTF-8?q?67)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 👋 test: Wait for the Header Selector Before Counting Agent Names on the Greeting Landing The greeting scenario read how many header controls named the agent with a one-shot `count()` and then expected the same number of mentions inside `main`. The header selector takes the agent's name after the welcome screen paints, so when the name landed between those two reads the first said 0 and the second saw the selector's own label, which sits inside `main`. When both reads ran before the name landed the check passed without the selector ever naming the agent. Wait for the selector to show the name, then expect that label to be the only mention inside `main`. * 🗂️ test: Keep the Sidebar Open When Chat History Is Already Showing On desktop `openChatHistory` clicked the Chat History rail button unconditionally. The rail button of the panel already showing collapses the sidebar, and Chat History is the panel every document starts on, so after the reload in the subagent activity scenario the helper closed the conversation list it was meant to show. The assertions that follow only passed while the list was still mounted during the collapse. Open the sidebar first on both layouts and press the rail button only when another panel is showing, the guard `openPanel` already uses. * 🚦 test: Hold the Mock Suite Until the Server Reports Ready Playwright starts the suite once the base URL answers, which the server does as soon as it listens. It keeps initializing after that and refuses chat starts and schedule writes with a 503 until `/readyz` turns 200, so a spec that runs first against a slow start got `SCHEDULES_NOT_READY`. The two-replica wrapper already waits on `/readyz` before exposing its proxy; the single-process server had no such gate. Poll `/readyz` in global setup before authenticating. --- e2e/setup/global-setup.ts | 2 + e2e/setup/readiness.ts | 39 +++++++++++++++++++ .../mock/scenarios/landing-greeting.spec.ts | 12 +++--- e2e/specs/mock/scenarios/sidebar.ts | 9 ++++- 4 files changed, 54 insertions(+), 8 deletions(-) create mode 100644 e2e/setup/readiness.ts diff --git a/e2e/setup/global-setup.ts b/e2e/setup/global-setup.ts index c417ebc340..8e19812217 100644 --- a/e2e/setup/global-setup.ts +++ b/e2e/setup/global-setup.ts @@ -1,8 +1,10 @@ import { FullConfig } from '@playwright/test'; +import { waitForReadiness } from './readiness'; import authenticate from './authenticate'; import { getE2EUser } from './user'; async function globalSetup(config: FullConfig) { + await waitForReadiness(config); await authenticate(config, getE2EUser()); } diff --git a/e2e/setup/readiness.ts b/e2e/setup/readiness.ts new file mode 100644 index 0000000000..1a9f290583 --- /dev/null +++ b/e2e/setup/readiness.ts @@ -0,0 +1,39 @@ +import { expect, request } from '@playwright/test'; +import type { FullConfig } from '@playwright/test'; + +/** The same budget the profiles give the server to start listening. */ +const READINESS_TIMEOUT_MS = 120_000; + +/** + * Holds the run until the app reports ready. Playwright starts the suite once the base URL + * answers, which the server does as soon as it listens; it then finishes initializing (MCP + * servers, migrations, the trigger and schedule engines) and refuses chat starts and schedule + * writes with a 503 until `/readyz` turns 200. The two-replica wrapper already waits for that + * before exposing its proxy, and this gives the single-process server the same ordering. + */ +export async function waitForReadiness(config: FullConfig): Promise { + const { baseURL } = config.projects[0].use; + if (typeof baseURL !== 'string') { + throw new Error('🤖: baseURL is not defined'); + } + const readyz = new URL('/readyz', baseURL).toString(); + const api = await request.newContext({ ignoreHTTPSErrors: true }); + try { + await expect + .poll( + () => + api.get(readyz, { timeout: 5_000 }).then( + (response) => response.status(), + () => 0, + ), + { + message: `${readyz} must report ready before the first spec runs`, + timeout: READINESS_TIMEOUT_MS, + intervals: [100, 250, 500], + }, + ) + .toBe(200); + } finally { + await api.dispose(); + } +} diff --git a/e2e/specs/mock/scenarios/landing-greeting.spec.ts b/e2e/specs/mock/scenarios/landing-greeting.spec.ts index b7d9c80732..4ca869e75a 100644 --- a/e2e/specs/mock/scenarios/landing-greeting.spec.ts +++ b/e2e/specs/mock/scenarios/landing-greeting.spec.ts @@ -125,12 +125,12 @@ test.describe('welcome screen heading and footer', () => { await expect(page.getByText(DISCLAIMER)).toHaveCount(0); /** No footer beneath it, so the composer keeps only its shadow clearance. */ await expect.poll(() => composerClearance(page)).toBe(16); - /** The header selector still names the agent; the welcome screen does not repeat it. */ - const inSelector = await page - .locator('button, [role="combobox"]') - .getByText(name, { exact: true }) - .count(); - await expect(main.getByText(name, { exact: true })).toHaveCount(inSelector); + /** The header selector follows the conversation on a short delay, so it names the agent a + * beat after the welcome screen paints. Once it does, that label is the only mention. */ + await expect(main.getByRole('button', { name: 'Select a model', exact: true })).toHaveText( + name, + ); + await expect(main.getByText(name, { exact: true })).toHaveCount(1); } finally { await cleanupAgent(page, agentId); } diff --git a/e2e/specs/mock/scenarios/sidebar.ts b/e2e/specs/mock/scenarios/sidebar.ts index 3f4b3b1067..85296d96e1 100644 --- a/e2e/specs/mock/scenarios/sidebar.ts +++ b/e2e/specs/mock/scenarios/sidebar.ts @@ -64,12 +64,17 @@ const NARROW_MAX_WIDTH = 768; * builder, say) may have replaced. */ export async function openChatHistory(page: Page) { + await openSidebar(page); const narrow = (page.viewportSize()?.width ?? Infinity) <= NARROW_MAX_WIDTH; if (!narrow) { - await page.getByRole('button', { name: 'Chat History' }).click(); + /* The rail button of the panel already showing collapses the sidebar, so it is pressed + only to switch panels. */ + const trigger = page.getByRole('button', { name: 'Chat History' }); + if ((await trigger.getAttribute('aria-pressed')) !== 'true') { + await trigger.click(); + } return; } - await openSidebar(page); const switcher = page.getByTestId('panel-switcher-button'); if (!(await switcher.textContent())?.includes('Chat History')) { await switcher.click();