mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-17 21:43:06 +00:00
📷 test: Capture Revision-Pinned PR Screenshot Evidence (#16191)
Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
parent
453b35dc57
commit
07e91ccdec
3 changed files with 205 additions and 0 deletions
|
|
@ -165,3 +165,26 @@ node e2e/setup/record.js --output=e2e/recordings/settings-draft.spec.ts
|
|||
7. Run the finished spec with `npm run e2e:mock -- <spec name>`.
|
||||
|
||||
Generated recordings are a draft, not the final test. The committed version should use the shared helpers in `e2e/specs/mock/helpers.ts` where possible, wait on network or visible UI state instead of fixed sleeps, and keep test data deterministic.
|
||||
|
||||
## PR screenshot pilot
|
||||
|
||||
`e2e/screenshots/playwright.config.ts` is an opt-in evidence capture lane, not a pixel-baseline suite. It runs the real app with the mock profile and a private ephemeral MongoDB, then captures welcome screens at desktop/mobile sizes in both themes. Desktop captures also include temporary chat and the settings dialog. No model request is made. Motion is reduced, so these stills do not prove transitions or streaming behavior.
|
||||
|
||||
Build and run each revision in its own clean worktree with its own locked install:
|
||||
|
||||
```sh
|
||||
npm ci
|
||||
npm run frontend
|
||||
npx playwright install chromium
|
||||
E2E_CAPTURE_SHA=$(git rev-parse HEAD) \
|
||||
E2E_CAPTURE_DIR="$PWD/e2e/.generated/evidence-before" \
|
||||
E2E_USE_MEMORY_MONGO=true \
|
||||
E2E_BASE_URL=http://127.0.0.1:3333 \
|
||||
npx playwright test --config=e2e/screenshots/playwright.config.ts
|
||||
```
|
||||
|
||||
For a historical revision without this lane, copy the two `e2e/screenshots/*.ts` files into its worktree, leaving its application code and lockfile unchanged. Run the same scenario source on both revisions. Use a new output directory for every attempt: existing images are never overwritten. Run revisions sequentially, or give every fixture service its own port using the mock profile's `E2E_*_PORT` settings. Do not reuse a running development server or a real user's database.
|
||||
|
||||
Each PNG has a JSON sidecar containing the revision, browser version, viewport, theme, scenario hash, lockfile hash, built HTML hash, and image hash. Visible images must decode, fonts must finish loading, greeting springs must settle, and consecutive captures must match. A passing run and human inspection are both required before treating the pair as reviewed evidence; sidecars from a failed run are not complete evidence. Compare matching filenames between revisions. Pin the browser and font environment as well as the app revisions.
|
||||
|
||||
Keep storage state, traces, logs, and session data private. Check that the PNGs contain only intended synthetic test data before uploading with the attachment-capable `gh` described in the PR template, then read back the PR body to verify real asset URLs. Mark any pending visual review explicitly. Do not commit the images. A new surface has no before state; label it accordingly rather than substituting another screen.
|
||||
|
|
|
|||
133
e2e/screenshots/pilot.spec.ts
Normal file
133
e2e/screenshots/pilot.spec.ts
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { version as playwrightVersion } from 'playwright/package.json';
|
||||
import type { Page, TestInfo } from '@playwright/test';
|
||||
|
||||
const sha256 = (data: Buffer) => createHash('sha256').update(data).digest('hex');
|
||||
const output = process.env.E2E_CAPTURE_DIR;
|
||||
if (!output) {
|
||||
throw new Error('E2E_CAPTURE_DIR is required');
|
||||
}
|
||||
const outputDir = path.resolve(output);
|
||||
mkdirSync(outputDir, { recursive: true, mode: 0o700 });
|
||||
|
||||
async function capture(page: Page, info: TestInfo, scenario: string) {
|
||||
const ready =
|
||||
scenario === 'settings'
|
||||
? page.getByRole('heading', { name: 'Settings', exact: true })
|
||||
: page.getByRole('textbox', { name: 'Message input' });
|
||||
await expect(ready).toBeVisible();
|
||||
await page.mouse.move(0, 0);
|
||||
await expect(page.getByRole('tooltip')).toHaveCount(0);
|
||||
await expect
|
||||
.poll(() =>
|
||||
page.locator('.split-parent span[style]').evaluateAll((letters) =>
|
||||
letters.every((letter) => {
|
||||
const style = getComputedStyle(letter);
|
||||
return (
|
||||
Number(style.opacity) === 1 &&
|
||||
(style.transform === 'none' || new DOMMatrixReadOnly(style.transform).isIdentity)
|
||||
);
|
||||
}),
|
||||
),
|
||||
)
|
||||
.toBe(true);
|
||||
await page.evaluate(async () => {
|
||||
await document.fonts.ready;
|
||||
await Promise.all(
|
||||
Array.from(document.images)
|
||||
.filter((image) => image.getBoundingClientRect().width > 0)
|
||||
.map((image) => image.decode()),
|
||||
);
|
||||
});
|
||||
const options = { animations: 'disabled', caret: 'hide', scale: 'css' } as const;
|
||||
let image: Buffer = Buffer.alloc(0);
|
||||
await expect(async () => {
|
||||
const previous = await page.screenshot(options);
|
||||
image = await page.screenshot(options);
|
||||
expect(image.equals(previous), 'consecutive captures must be stable').toBe(true);
|
||||
}).toPass({ timeout: 10_000 });
|
||||
const file = `${info.title}-${scenario}.png`;
|
||||
writeFileSync(path.join(outputDir, file), image, { flag: 'wx', mode: 0o600 });
|
||||
const manifest = {
|
||||
file,
|
||||
sha256: sha256(image),
|
||||
revision: process.env.E2E_CAPTURE_SHA,
|
||||
scenario,
|
||||
theme: await page.locator('html').getAttribute('class'),
|
||||
viewport: page.viewportSize(),
|
||||
deviceScaleFactor: 1,
|
||||
locale: 'en-US',
|
||||
timezoneId: 'UTC',
|
||||
reducedMotion: 'reduce',
|
||||
browser: page.context().browser()?.version(),
|
||||
playwright: playwrightVersion,
|
||||
lockfileSha256: sha256(readFileSync('package-lock.json')),
|
||||
htmlSha256: sha256(readFileSync('client/dist/index.html')),
|
||||
scenarioSha256: sha256(readFileSync(__filename)),
|
||||
provider: 'mock harness; no model request in this scenario',
|
||||
consecutiveFramesIdentical: true,
|
||||
};
|
||||
writeFileSync(path.join(outputDir, `${file}.json`), `${JSON.stringify(manifest, null, 2)}\n`, {
|
||||
flag: 'wx',
|
||||
mode: 0o600,
|
||||
});
|
||||
}
|
||||
|
||||
for (const viewport of [
|
||||
{ name: 'desktop', width: 1280, height: 900 },
|
||||
{ name: 'mobile', width: 390, height: 844 },
|
||||
]) {
|
||||
for (const theme of ['light', 'dark'] as const) {
|
||||
test.describe(`${viewport.name} ${theme}`, () => {
|
||||
test.use({
|
||||
viewport: { width: viewport.width, height: viewport.height },
|
||||
colorScheme: theme,
|
||||
});
|
||||
test(`${viewport.name}-${theme}`, async ({ page, baseURL }, info) => {
|
||||
if (!baseURL) {
|
||||
throw new Error('A local base URL is required');
|
||||
}
|
||||
const pageErrors: string[] = [];
|
||||
page.on('pageerror', (error) => pageErrors.push(error.message));
|
||||
await page.route('**/*', async (route) => {
|
||||
const url = new URL(route.request().url());
|
||||
if (url.origin !== new URL(baseURL).origin) {
|
||||
await route.abort('blockedbyclient');
|
||||
return;
|
||||
}
|
||||
await route.continue();
|
||||
});
|
||||
await page.addInitScript((selectedTheme) => {
|
||||
localStorage.setItem('color-theme', selectedTheme);
|
||||
localStorage.setItem('navVisible', 'true');
|
||||
localStorage.setItem('i18nextLng', 'en-US');
|
||||
}, theme);
|
||||
await page.goto('/c/new');
|
||||
expect(await page.evaluate(() => window.devicePixelRatio)).toBe(1);
|
||||
expect(
|
||||
await page.evaluate(() => matchMedia('(prefers-reduced-motion: reduce)').matches),
|
||||
).toBe(true);
|
||||
await expect(page.locator('html')).toHaveClass(new RegExp(`(^|\\s)${theme}(\\s|$)`));
|
||||
await expect(page.getByRole('textbox', { name: 'Message input' })).toBeEnabled();
|
||||
await capture(page, info, 'welcome');
|
||||
if (viewport.name === 'desktop') {
|
||||
const toggle = page.getByRole('button', { name: 'Temporary Chat', exact: true });
|
||||
await expect(toggle).toHaveAttribute('aria-pressed', 'false');
|
||||
await toggle.click();
|
||||
await expect(toggle).toHaveAttribute('aria-pressed', 'true');
|
||||
await expect(page.getByText('Temporary Chat', { exact: true }).first()).toBeVisible();
|
||||
await capture(page, info, 'temporary');
|
||||
await toggle.click();
|
||||
await page.getByTestId('nav-user').click();
|
||||
await page.getByRole('menuitem', { name: 'Settings', exact: true }).click();
|
||||
await expect(page.getByRole('heading', { name: 'Settings', exact: true })).toBeVisible();
|
||||
await capture(page, info, 'settings');
|
||||
}
|
||||
expect(pageErrors).toEqual([]);
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
49
e2e/screenshots/playwright.config.ts
Normal file
49
e2e/screenshots/playwright.config.ts
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
import path from 'node:path';
|
||||
import { defineConfig } from '@playwright/test';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import mockConfig from '../playwright.config.mock';
|
||||
|
||||
const revision = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim();
|
||||
if (process.env.E2E_CAPTURE_SHA !== revision) {
|
||||
throw new Error('E2E_CAPTURE_SHA must equal the checked-out commit');
|
||||
}
|
||||
execFileSync('git', ['diff', '--quiet', 'HEAD']);
|
||||
const baseURL = new URL(process.env.E2E_BASE_URL ?? 'http://localhost:3080');
|
||||
if (
|
||||
baseURL.protocol !== 'http:' ||
|
||||
!['127.0.0.1', 'localhost', '[::1]'].includes(baseURL.hostname)
|
||||
) {
|
||||
throw new Error('Screenshot trials must run against a loopback address');
|
||||
}
|
||||
if (!process.env.E2E_CAPTURE_DIR) {
|
||||
throw new Error('E2E_CAPTURE_DIR must name a fresh, private output directory');
|
||||
}
|
||||
if (process.env.E2E_USE_MEMORY_MONGO !== 'true') {
|
||||
throw new Error('Screenshot trials require their own ephemeral MongoDB');
|
||||
}
|
||||
if (process.env.E2E_MODEL_FIXTURES === 'record') {
|
||||
throw new Error('Screenshot trials must not use paid model recording');
|
||||
}
|
||||
|
||||
export default defineConfig({
|
||||
...mockConfig,
|
||||
testDir: '.',
|
||||
testMatch: 'pilot.spec.ts',
|
||||
outputDir: path.resolve(process.cwd(), 'e2e/specs/.test-results'),
|
||||
fullyParallel: false,
|
||||
workers: 1,
|
||||
retries: 0,
|
||||
timeout: 60_000,
|
||||
reporter: [['list']],
|
||||
use: {
|
||||
...mockConfig.use,
|
||||
locale: 'en-US',
|
||||
timezoneId: 'UTC',
|
||||
deviceScaleFactor: 1,
|
||||
contextOptions: { reducedMotion: 'reduce' },
|
||||
serviceWorkers: 'block',
|
||||
video: 'off',
|
||||
trace: 'off',
|
||||
screenshot: 'off',
|
||||
},
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue