From b45e4aeae5124b8060df85026ca45b04a4a6e9ac Mon Sep 17 00:00:00 2001 From: Teresa Blanco Date: Tue, 2 Jun 2026 22:36:39 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=AD=20feat:=20Add=20Credential-Free=20?= =?UTF-8?q?Playwright=20Smoke=20Suite=20with=20a=20Local=20Mock=20LLM=20(#?= =?UTF-8?q?13472)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🧪 feat: add e2e playwright tests * 🧪 feat: Add Playwright Recording Harness * test: fix mock playwright config * test: harden mock e2e environment * test: preserve mock dotenv secrets * test: harden mock isolation setup * ci: cache mock e2e builds * test: harden e2e cache and recorder checks * test: preserve data-provider exports in oauth route test * test: isolate mock auth logout state * test: allow isolated logout smoke setup * test: prepare logout smoke auth via api * test: isolate oauth route module mock --------- Co-authored-by: Danny Avila --- .github/workflows/playwright-mock.yml | 143 ++++++++ .gitignore | 1 + api/server/index.js | 5 +- api/server/routes/oauth.test.js | 11 +- e2e/README.md | 40 +++ e2e/config/librechat.e2e.yaml | 26 ++ e2e/playwright.config.mock.ts | 145 ++++++++ e2e/playwright.config.ts | 1 + e2e/recordings/.gitignore | 2 + e2e/setup/env.ts | 4 +- e2e/setup/global-teardown.mock.ts | 14 + e2e/setup/mock-llm-server.js | 128 +++++++ e2e/setup/record.js | 454 +++++++++++++++++++++++++ e2e/setup/users.mock.ts | 20 ++ e2e/specs/mock/app-load.spec.ts | 22 ++ e2e/specs/mock/auth.spec.ts | 82 +++++ e2e/specs/mock/chat.spec.ts | 37 ++ e2e/specs/mock/helpers.ts | 47 +++ e2e/specs/mock/isolation.spec.ts | 97 ++++++ e2e/specs/mock/model-switching.spec.ts | 23 ++ package.json | 8 +- 21 files changed, 1302 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/playwright-mock.yml create mode 100644 e2e/README.md create mode 100644 e2e/config/librechat.e2e.yaml create mode 100644 e2e/playwright.config.mock.ts create mode 100644 e2e/recordings/.gitignore create mode 100644 e2e/setup/global-teardown.mock.ts create mode 100644 e2e/setup/mock-llm-server.js create mode 100644 e2e/setup/record.js create mode 100644 e2e/setup/users.mock.ts create mode 100644 e2e/specs/mock/app-load.spec.ts create mode 100644 e2e/specs/mock/auth.spec.ts create mode 100644 e2e/specs/mock/chat.spec.ts create mode 100644 e2e/specs/mock/helpers.ts create mode 100644 e2e/specs/mock/isolation.spec.ts create mode 100644 e2e/specs/mock/model-switching.spec.ts diff --git a/.github/workflows/playwright-mock.yml b/.github/workflows/playwright-mock.yml new file mode 100644 index 0000000000..3d071a9056 --- /dev/null +++ b/.github/workflows/playwright-mock.yml @@ -0,0 +1,143 @@ +name: Playwright E2E (Mock LLM) + +on: + pull_request: + workflow_dispatch: + inputs: + reason: + description: 'Reason for manual trigger' + required: false + default: 'Manual e2e run' + +permissions: + contents: read + +concurrency: + group: playwright-mock-${{ github.ref }} + cancel-in-progress: true + +env: + NODE_OPTIONS: '--max-old-space-size=${{ secrets.NODE_MAX_OLD_SPACE_SIZE || 6144 }}' + +jobs: + e2e: + name: Tier-1 smoke (headless Chromium) + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - name: Use Node.js 24.16.0 + uses: actions/setup-node@v4 + with: + node-version: '24.16.0' + + - name: Restore node_modules cache + id: cache-node-modules + uses: actions/cache@v4 + with: + path: | + node_modules + client/node_modules + packages/client/node_modules + packages/data-provider/node_modules + packages/data-schemas/node_modules + packages/api/node_modules + api/node_modules + key: node-modules-e2e-${{ runner.os }}-24.16.0-${{ hashFiles('package-lock.json') }} + + - name: Install dependencies + if: steps.cache-node-modules.outputs.cache-hit != 'true' + run: npm ci + + - name: Restore data-provider build cache + id: cache-data-provider + uses: actions/cache@v4 + with: + path: packages/data-provider/dist + key: build-data-provider-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/rollup.config.js', 'packages/data-provider/package.json') }} + + - name: Build data-provider + if: steps.cache-data-provider.outputs.cache-hit != 'true' + run: npm run build:data-provider + + - name: Restore data-schemas build cache + id: cache-data-schemas + uses: actions/cache@v4 + with: + path: packages/data-schemas/dist + key: build-data-schemas-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/data-schemas/src/**', 'packages/data-schemas/tsconfig*.json', 'packages/data-schemas/rollup.config.js', 'packages/data-schemas/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/rollup.config.js', 'packages/data-provider/package.json') }} + + - name: Build data-schemas + if: steps.cache-data-schemas.outputs.cache-hit != 'true' + run: npm run build:data-schemas + + - name: Restore api build cache + id: cache-api + uses: actions/cache@v4 + with: + path: packages/api/dist + key: build-api-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/api/src/**', 'packages/api/tsconfig*.json', 'packages/api/server-rollup.config.js', 'packages/api/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/rollup.config.js', 'packages/data-provider/package.json', 'packages/data-schemas/src/**', 'packages/data-schemas/tsconfig*.json', 'packages/data-schemas/rollup.config.js', 'packages/data-schemas/package.json') }} + + - name: Build api + if: steps.cache-api.outputs.cache-hit != 'true' + run: npm run build:api + + - name: Restore client-package build cache + id: cache-client-package + uses: actions/cache@v4 + with: + path: packages/client/dist + key: build-client-package-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/client/src/**', 'packages/client/tsconfig*.json', 'packages/client/rollup.config.js', 'packages/client/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/rollup.config.js', 'packages/data-provider/package.json') }} + + - name: Build client-package + if: steps.cache-client-package.outputs.cache-hit != 'true' + run: npm run build:client-package + + - name: Restore client app build cache + id: cache-client-app + uses: actions/cache@v4 + with: + path: client/dist + key: build-client-app-e2e-${{ runner.os }}-${{ hashFiles('package-lock.json', 'client/src/**', 'client/public/**', 'client/scripts/post-build.cjs', 'client/index.html', 'client/package.json', 'client/vite.config.*', 'client/tsconfig*.json', 'client/tailwind.config.*', 'client/postcss.config.*', 'packages/client/src/**', 'packages/client/tsconfig*.json', 'packages/client/rollup.config.js', 'packages/client/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/rollup.config.js', 'packages/data-provider/package.json') }} + + - name: Build client app + if: steps.cache-client-app.outputs.cache-hit != 'true' + run: npm run build:client + + - name: Resolve Playwright version + id: pw + run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT" + + - name: Cache Playwright browsers + id: cache-playwright + uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: playwright-${{ runner.os }}-${{ steps.pw.outputs.version }} + + - name: Install Playwright Chromium + run: npx playwright install --with-deps chromium + + - name: Run mock-LLM Tier-1 e2e + run: npx playwright test --config=e2e/playwright.config.mock.ts + env: + CI: 'true' + + - name: Upload Playwright HTML report + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: playwright-report + path: e2e/playwright-report/** + retention-days: 7 + if-no-files-found: ignore + + - name: Upload traces & screenshots + if: failure() + uses: actions/upload-artifact@v4 + with: + name: playwright-test-results + path: e2e/specs/.test-results/** + retention-days: 7 + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index d775e70a26..4fd8ecb2db 100644 --- a/.gitignore +++ b/.gitignore @@ -88,6 +88,7 @@ archive .vscode/settings.json src/style - official.css /e2e/specs/.test-results/ +/e2e/.generated/ /e2e/playwright-report/ /playwright/.cache/ .DS_Store diff --git a/api/server/index.js b/api/server/index.js index 57b9e3e3a3..bd2d260e76 100644 --- a/api/server/index.js +++ b/api/server/index.js @@ -281,7 +281,10 @@ const startServer = async () => { // Configure stream services (auto-detects Redis from USE_REDIS env var) const streamServices = createStreamServices(); - GenerationJobManager.configure(streamServices); + GenerationJobManager.configure({ + ...streamServices, + cleanupOnComplete: !isEnabled(process.env.STREAM_KEEP_COMPLETED_JOBS), + }); GenerationJobManager.initialize(); const inspectFlags = process.execArgv.some((arg) => arg.startsWith('--inspect')); diff --git a/api/server/routes/oauth.test.js b/api/server/routes/oauth.test.js index 20661a9084..0937525c5f 100644 --- a/api/server/routes/oauth.test.js +++ b/api/server/routes/oauth.test.js @@ -63,6 +63,7 @@ jest.mock('@librechat/data-schemas', () => ({ })); jest.mock('librechat-data-provider', () => ({ + ...jest.requireActual('librechat-data-provider'), ErrorTypes: { AUTH_FAILED: 'auth_failed', }, @@ -99,8 +100,6 @@ jest.mock('~/server/services/Config', () => ({ getAppConfig: jest.fn(), })); -const oauthRouter = require('./oauth'); - afterAll(() => { if (originalDomainClient === undefined) { delete process.env.DOMAIN_CLIENT; @@ -109,6 +108,11 @@ afterAll(() => { process.env.DOMAIN_CLIENT = originalDomainClient; }); +function getOAuthRouter() { + jest.resetModules(); + return require('./oauth'); +} + function createApp(sessionMessages) { const app = express(); app.use((req, _res, next) => { @@ -117,7 +121,7 @@ function createApp(sessionMessages) { } next(); }); - app.use('/oauth', oauthRouter); + app.use('/oauth', getOAuthRouter()); app.use((err, _req, res, _next) => { res.status(500).json({ message: err.message }); }); @@ -136,6 +140,7 @@ describe('OAuth route failure logging', () => { mockGetOAuthFailureMessage.mockClear(); mockRedirectToAuthFailure.mockClear(); mockPassportAuthenticate.mockClear(); + mockOpenIDCallbackAuthenticatorOptions = undefined; mockPassportAuthenticate.mockImplementation(() => (_req, _res, next) => next()); mockOpenIDCallbackMiddleware.mockImplementation((_req, _res, next) => next()); }); diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 0000000000..822297103f --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,40 @@ +# LibreChat e2e + +The mock e2e profile is the safest default for generated tests. It starts LibreChat with `e2e/config/librechat.e2e.yaml`, points custom endpoints at the local mock LLM server, creates an authenticated e2e user, and avoids real provider credentials. + +## Recording Tests + +Use Playwright codegen when you want to turn an exploratory browser session into a draft test: + +```sh +npm run e2e:record +``` + +That command builds the app, starts the mock LLM and LibreChat test server when needed, writes `e2e/storageState.json`, and opens Playwright codegen at `/c/new`. The npm script uses `http://localhost:3333` so it does not collide with a normal dev server on `3080`. Raw recordings are written to `e2e/recordings/` and ignored by git. + +For a real local LibreChat config instead of the mock LLM profile: + +```sh +npm run e2e:record:local +``` + +Useful direct options: + +```sh +node e2e/setup/record.js --url=http://localhost:3080/c/new +node e2e/setup/record.js --profile=local --no-output +node e2e/setup/record.js --auth-only +node e2e/setup/record.js --output=e2e/recordings/settings-draft.spec.ts +``` + +## LLM-Assisted Loop + +1. Start `npm run e2e:record`. +2. Let the LLM use Computer Use to operate the headed Playwright browser. +3. Stop codegen after the workflow is captured. +4. Move the useful parts from `e2e/recordings/` into a committed spec under `e2e/specs/mock/`. +5. Replace brittle generated selectors with role, label, text, or `data-testid` locators. +6. Add assertions that prove the behavior, not just the clicked path. +7. Run the finished spec with `npm run e2e:mock -- `. + +Generated recordings are a draft, not the final test. The committed version should use the shared helpers in `e2e/specs/mock/helpers.ts` where possible, wait on network or visible UI state instead of fixed sleeps, and keep test data deterministic. diff --git a/e2e/config/librechat.e2e.yaml b/e2e/config/librechat.e2e.yaml new file mode 100644 index 0000000000..e7e9e62271 --- /dev/null +++ b/e2e/config/librechat.e2e.yaml @@ -0,0 +1,26 @@ +# Credential-free e2e config template. e2e/playwright.config.mock.ts writes +# an ignored runtime copy and rewrites the mock LLM port from MOCK_LLM_PORT. +version: 1.3.11 +cache: true + +endpoints: + custom: + - name: 'Mock Provider A' + apiKey: 'e2e-mock-key-a' + baseURL: 'http://127.0.0.1:8889/v1' + models: + default: + - 'mock-model-a' + fetch: false + titleConvo: false + modelDisplayLabel: 'Mock Provider A' + + - name: 'Mock Provider B' + apiKey: 'e2e-mock-key-b' + baseURL: 'http://127.0.0.1:8889/v1' + models: + default: + - 'mock-model-b' + fetch: false + titleConvo: false + modelDisplayLabel: 'Mock Provider B' diff --git a/e2e/playwright.config.mock.ts b/e2e/playwright.config.mock.ts new file mode 100644 index 0000000000..b0ebc82567 --- /dev/null +++ b/e2e/playwright.config.mock.ts @@ -0,0 +1,145 @@ +import { defineConfig, devices } from '@playwright/test'; +import fs from 'fs'; +import path from 'path'; +import { getLocalE2EEnv, getE2EBaseURL } from './setup/env'; + +const rootPath = path.resolve(__dirname, '..'); +const serverPath = path.resolve(rootPath, 'e2e/setup/start-server.js'); +const mockLlmPath = path.resolve(rootPath, 'e2e/setup/mock-llm-server.js'); +const configTemplatePath = path.resolve(rootPath, 'e2e/config/librechat.e2e.yaml'); +const configPath = path.resolve(rootPath, 'e2e/.generated/librechat.e2e.yaml'); +const reportPath = path.resolve(rootPath, 'e2e/playwright-report'); + +const baseURL = getE2EBaseURL(); +const mockLlmPort = getMockLlmPort(); +const defaultMockLlmBaseURL = 'http://127.0.0.1:8889/v1'; +const mockLlmBaseURL = `http://127.0.0.1:${mockLlmPort}/v1`; + +const vanillaOverrides = { + TENANT_ISOLATION_STRICT: 'false', + OPENAI_API_KEY: 'user_provided', + OPENID_CLIENT_ID: '', + OPENID_ISSUER: '', + OPENID_AUTO_REDIRECT: 'false', + ALLOW_SOCIAL_LOGIN: 'false', + ALLOW_SOCIAL_REGISTRATION: 'false', + STREAM_KEEP_COMPLETED_JOBS: 'true', +}; + +const baseEnv = { + ...getLocalE2EEnv(), + CONFIG_PATH: configPath, + MOCK_LLM_PORT: mockLlmPort, + ...vanillaOverrides, +}; + +const SECRET_KEY_PATTERN = /(API_KEY|SECRET|TOKEN|PASSWORD|CREDENTIALS|CLIENT_ID|_KEY)$/i; +const preservedCredentialEnvKeys = new Set([ + ...Object.keys(baseEnv), + 'E2E_USER_PASSWORD', + 'E2E_USER_B_PASSWORD', +]); + +function getMockLlmPort() { + const port = process.env.MOCK_LLM_PORT ?? '8889'; + if (!/^\d+$/.test(port)) { + throw new Error('MOCK_LLM_PORT must be a numeric port'); + } + return port; +} + +function writeRuntimeMockConfig() { + const template = fs.readFileSync(configTemplatePath, 'utf8'); + + if (!template.includes(defaultMockLlmBaseURL)) { + throw new Error(`Expected mock config template to include ${defaultMockLlmBaseURL}`); + } + + fs.mkdirSync(path.dirname(configPath), { recursive: true }); + fs.writeFileSync(configPath, template.replaceAll(defaultMockLlmBaseURL, mockLlmBaseURL)); +} + +function neutralizeCredentialEnv(env: NodeJS.ProcessEnv, keep: Set) { + for (const key of Object.keys(env)) { + if (!keep.has(key) && SECRET_KEY_PATTERN.test(key)) { + env[key] = ''; + } + } +} + +/** Blank any credential-like vars from a local `.env` so they never reach the test server. */ +function neutralizeDotenvSecrets(envFile: string, keep: Set) { + if (!fs.existsSync(envFile)) { + return; + } + const lines = fs.readFileSync(envFile, 'utf8').split('\n'); + for (const line of lines) { + const match = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=/); + if (!match) { + continue; + } + const key = match[1]; + if (keep.has(key)) { + continue; + } + if (SECRET_KEY_PATTERN.test(key)) { + process.env[key] = ''; + } + } +} + +writeRuntimeMockConfig(); +neutralizeCredentialEnv(process.env, preservedCredentialEnvKeys); +Object.assign(process.env, baseEnv); +neutralizeDotenvSecrets(path.resolve(rootPath, '.env'), preservedCredentialEnvKeys); + +export default defineConfig({ + globalSetup: require.resolve('./setup/global-setup'), + globalTeardown: require.resolve('./setup/global-teardown.mock'), + testDir: 'specs/mock/', + outputDir: 'specs/.test-results', + fullyParallel: false, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + workers: 1, + reporter: process.env.CI + ? [['html', { outputFolder: reportPath, open: 'never' }], ['line']] + : [['html', { outputFolder: reportPath }], ['list']], + use: { + baseURL, + video: 'on-first-retry', + trace: 'retain-on-failure', + ignoreHTTPSErrors: true, + headless: true, + storageState: path.resolve(process.cwd(), 'e2e/storageState.json'), + screenshot: 'only-on-failure', + }, + expect: { + timeout: 10000, + }, + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + ], + webServer: [ + { + command: `node ${mockLlmPath}`, + cwd: rootPath, + url: `http://127.0.0.1:${mockLlmPort}/health`, + stdout: 'pipe', + timeout: 30_000, + reuseExistingServer: false, + }, + { + command: `node ${serverPath}`, + cwd: rootPath, + url: baseURL, + stdout: 'pipe', + ignoreHTTPSErrors: true, + timeout: 120_000, + reuseExistingServer: false, + }, + ], +}); diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts index 009d77f24e..e62856c2d3 100644 --- a/e2e/playwright.config.ts +++ b/e2e/playwright.config.ts @@ -27,6 +27,7 @@ export default defineConfig({ workers: process.env.CI ? 1 : undefined, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ reporter: [['html', { outputFolder: 'playwright-report' }]], + testIgnore: ['**/mock/**'], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { baseURL, diff --git a/e2e/recordings/.gitignore b/e2e/recordings/.gitignore new file mode 100644 index 0000000000..d6b7ef32c8 --- /dev/null +++ b/e2e/recordings/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/e2e/setup/env.ts b/e2e/setup/env.ts index a17f882953..9a4b184a7a 100644 --- a/e2e/setup/env.ts +++ b/e2e/setup/env.ts @@ -93,9 +93,9 @@ export function getBaseE2EEnv(): Record { JWT_REFRESH_SECRET: process.env.JWT_REFRESH_SECRET ?? GENERATED_JWT_REFRESH_SECRET, EMAIL_HOST: '', SEARCH: 'false', - SESSION_EXPIRY: '60000', + SESSION_EXPIRY: process.env.SESSION_EXPIRY ?? '3600000', ALLOW_REGISTRATION: 'true', - REFRESH_TOKEN_EXPIRY: '300000', + REFRESH_TOKEN_EXPIRY: process.env.REFRESH_TOKEN_EXPIRY ?? '3600000', }; } diff --git a/e2e/setup/global-teardown.mock.ts b/e2e/setup/global-teardown.mock.ts new file mode 100644 index 0000000000..72dc93a221 --- /dev/null +++ b/e2e/setup/global-teardown.mock.ts @@ -0,0 +1,14 @@ +import cleanupUser from './cleanupUser'; +import { getPrimaryE2EUser, getSecondaryE2EUser } from './users.mock'; + +async function globalTeardown() { + for (const user of [getPrimaryE2EUser(), getSecondaryE2EUser()]) { + try { + await cleanupUser(user); + } catch (error) { + console.error('Error:', error); + } + } +} + +export default globalTeardown; diff --git a/e2e/setup/mock-llm-server.js b/e2e/setup/mock-llm-server.js new file mode 100644 index 0000000000..1aa9ad78a6 --- /dev/null +++ b/e2e/setup/mock-llm-server.js @@ -0,0 +1,128 @@ +/** + * OpenAI-compatible mock server for credential-free e2e tests. Answers + * `${baseURL}/chat/completions` with deterministic content. Run standalone + * (Playwright `webServer`) or import `startMockLlm()` for programmatic control. + */ +const http = require('http'); + +const DEFAULT_PORT = 8889; +const MOCK_REPLY = process.env.MOCK_LLM_REPLY || 'E2E mock reply: pong'; +const MODEL_FALLBACK = 'mock-model'; +const STREAM_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_CHUNK_DELAY_MS) || 60; + +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +function readJsonBody(req) { + return new Promise((resolve) => { + let raw = ''; + req.on('data', (chunk) => { + raw += chunk; + }); + req.on('end', () => { + try { + resolve(raw ? JSON.parse(raw) : {}); + } catch { + resolve({}); + } + }); + }); +} + +function toChunks(text) { + return text.match(/\S+\s*/g) || [text]; +} + +async function streamCompletion(res, model) { + res.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + }); + if (typeof res.flushHeaders === 'function') { + res.flushHeaders(); + } + + const id = 'chatcmpl-e2e-mock'; + const created = 1700000000; + const base = { id, object: 'chat.completion.chunk', created, model }; + + const send = (delta, finishReason = null) => { + const payload = { + ...base, + choices: [{ index: 0, delta, finish_reason: finishReason }], + }; + res.write(`data: ${JSON.stringify(payload)}\n\n`); + }; + + send({ role: 'assistant', content: '' }); + for (const chunk of toChunks(MOCK_REPLY)) { + await delay(STREAM_CHUNK_DELAY_MS); + send({ content: chunk }); + } + send({}, 'stop'); + res.write('data: [DONE]\n\n'); + res.end(); +} + +function jsonCompletion(res, model) { + const payload = { + id: 'chatcmpl-e2e-mock', + object: 'chat.completion', + created: 1700000000, + model, + choices: [ + { + index: 0, + message: { role: 'assistant', content: MOCK_REPLY }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(payload)); +} + +async function handleRequest(req, res) { + if (req.method === 'GET' && (req.url === '/health' || req.url === '/')) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ status: 'ok' })); + return; + } + + if (req.method === 'POST' && req.url && req.url.endsWith('/chat/completions')) { + const body = await readJsonBody(req); + const model = body.model || MODEL_FALLBACK; + if (body.stream) { + await streamCompletion(res, model); + } else { + jsonCompletion(res, model); + } + return; + } + + res.writeHead(404, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'not found' })); +} + +function startMockLlm(port = Number(process.env.MOCK_LLM_PORT) || DEFAULT_PORT) { + const server = http.createServer((req, res) => { + handleRequest(req, res).catch(() => { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'mock server error' })); + }); + }); + + return new Promise((resolve) => { + server.listen(port, '127.0.0.1', () => { + console.log(`[e2e] Mock LLM server listening on http://127.0.0.1:${port}`); + resolve(server); + }); + }); +} + +if (require.main === module) { + startMockLlm(); +} + +module.exports = { startMockLlm, MOCK_REPLY }; diff --git a/e2e/setup/record.js b/e2e/setup/record.js new file mode 100644 index 0000000000..6bbb1d0c98 --- /dev/null +++ b/e2e/setup/record.js @@ -0,0 +1,454 @@ +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const { spawn } = require('child_process'); + +const rootPath = path.resolve(__dirname, '../..'); +const baseURL = process.env.E2E_BASE_URL || 'http://localhost:3080'; +const storageStatePath = path.resolve(rootPath, 'e2e/storageState.json'); +const configTemplatePath = path.resolve(rootPath, 'e2e/config/librechat.e2e.yaml'); +const configPath = path.resolve(rootPath, 'e2e/.generated/librechat.e2e.yaml'); +const defaultMockLlmBaseURL = 'http://127.0.0.1:8889/v1'; +const mockLlmPort = getMockLlmPort(); +const mockLlmBaseURL = `http://127.0.0.1:${mockLlmPort}/v1`; +const defaultUser = { + email: 'testuser@example.com', + name: 'Test User', + password: 'securepassword123', +}; + +const rateLimitOverrides = { + LOGIN_VIOLATION_SCORE: '0', + REGISTRATION_VIOLATION_SCORE: '0', + CONCURRENT_VIOLATION_SCORE: '0', + MESSAGE_VIOLATION_SCORE: '0', + NON_BROWSER_VIOLATION_SCORE: '0', + FORK_VIOLATION_SCORE: '0', + IMPORT_VIOLATION_SCORE: '0', + TTS_VIOLATION_SCORE: '0', + STT_VIOLATION_SCORE: '0', + FILE_UPLOAD_VIOLATION_SCORE: '0', + RESET_PASSWORD_VIOLATION_SCORE: '0', + VERIFY_EMAIL_VIOLATION_SCORE: '0', + TOOL_CALL_VIOLATION_SCORE: '0', + CONVO_ACCESS_VIOLATION_SCORE: '0', + ILLEGAL_MODEL_REQ_SCORE: '0', + LOGIN_MAX: '20', + LOGIN_WINDOW: '1', + REGISTER_MAX: '20', + REGISTER_WINDOW: '1', + LIMIT_CONCURRENT_MESSAGES: 'false', + CONCURRENT_MESSAGE_MAX: '20', + LIMIT_MESSAGE_IP: 'false', + MESSAGE_IP_MAX: '100', + MESSAGE_IP_WINDOW: '1', + LIMIT_MESSAGE_USER: 'false', + MESSAGE_USER_MAX: '100', + MESSAGE_USER_WINDOW: '1', +}; + +const mockOverrides = { + CONFIG_PATH: configPath, + MOCK_LLM_PORT: mockLlmPort, + OPENAI_API_KEY: 'user_provided', + TENANT_ISOLATION_STRICT: 'false', + OPENID_CLIENT_ID: '', + OPENID_ISSUER: '', + OPENID_AUTO_REDIRECT: 'false', + ALLOW_SOCIAL_LOGIN: 'false', + ALLOW_SOCIAL_REGISTRATION: 'false', + STREAM_KEEP_COMPLETED_JOBS: 'true', +}; + +const secretKeyPattern = /(API_KEY|SECRET|TOKEN|PASSWORD|CREDENTIALS|CLIENT_ID|_KEY)$/i; +const preservedCredentialEnvKeys = new Set([ + ...Object.keys(rateLimitOverrides), + ...Object.keys(mockOverrides).filter((key) => key !== 'OPENAI_API_KEY'), + 'CREDS_KEY', + 'CREDS_IV', + 'E2E_USER_PASSWORD', + 'E2E_USER_B_PASSWORD', + 'JWT_SECRET', + 'JWT_REFRESH_SECRET', + 'REFRESH_TOKEN_EXPIRY', + 'SESSION_EXPIRY', +]); +const npxBin = process.platform === 'win32' ? 'npx.cmd' : 'npx'; + +function getMockLlmPort() { + const port = process.env.MOCK_LLM_PORT || '8889'; + if (!/^\d+$/.test(port)) { + throw new Error('MOCK_LLM_PORT must be a numeric port'); + } + return port; +} + +function appURL(pathname = '') { + const normalizedBaseURL = baseURL.endsWith('/') ? baseURL : `${baseURL}/`; + return new URL(pathname.replace(/^\/+/, ''), normalizedBaseURL).toString(); +} + +function getServerAddress() { + const url = new URL(baseURL); + const host = url.hostname.replace(/^\[(.*)\]$/, '$1'); + const port = url.port || (url.protocol === 'https:' ? '443' : '80'); + return { host, port }; +} + +function randomHex(bytes) { + return crypto.randomBytes(bytes).toString('hex'); +} + +function getUser(env) { + return { + email: env.E2E_USER_EMAIL || defaultUser.email, + name: env.E2E_USER_NAME || defaultUser.name, + password: env.E2E_USER_PASSWORD || defaultUser.password, + }; +} + +function getBaseEnv() { + const { host, port } = getServerAddress(); + return { + ...process.env, + NODE_ENV: 'CI', + HOST: process.env.E2E_HOST || host, + PORT: process.env.E2E_PORT || port, + MONGO_URI: process.env.MONGO_URI || 'mongodb://127.0.0.1:27017/LibreChat-e2e', + DOMAIN_CLIENT: process.env.E2E_DOMAIN_CLIENT || baseURL, + DOMAIN_SERVER: process.env.E2E_DOMAIN_SERVER || baseURL, + E2E_RUNTIME_ENV_PATH: + process.env.E2E_RUNTIME_ENV_PATH || + path.resolve(rootPath, 'e2e/specs/.test-results/runtime-env.json'), + E2E_USE_MEMORY_MONGO: process.env.E2E_USE_MEMORY_MONGO || 'auto', + NO_INDEX: process.env.NO_INDEX || 'true', + OPENAI_API_KEY: process.env.OPENAI_API_KEY || 'user_provided', + CREDS_KEY: process.env.CREDS_KEY || randomHex(32), + CREDS_IV: process.env.CREDS_IV || randomHex(16), + JWT_SECRET: process.env.JWT_SECRET || randomHex(32), + JWT_REFRESH_SECRET: process.env.JWT_REFRESH_SECRET || randomHex(32), + EMAIL_HOST: '', + SEARCH: 'false', + SESSION_EXPIRY: process.env.SESSION_EXPIRY || '3600000', + ALLOW_REGISTRATION: 'true', + REFRESH_TOKEN_EXPIRY: process.env.REFRESH_TOKEN_EXPIRY || '3600000', + TITLE_CONVO: 'false', + ...rateLimitOverrides, + }; +} + +function neutralizeCredentialEnv(env) { + for (const key of Object.keys(env)) { + if (!preservedCredentialEnvKeys.has(key) && secretKeyPattern.test(key)) { + env[key] = ''; + } + } +} + +function neutralizeDotenvSecrets(envFile, env) { + if (!fs.existsSync(envFile)) { + return; + } + const lines = fs.readFileSync(envFile, 'utf8').split('\n'); + for (const line of lines) { + const match = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=/); + if (!match) { + continue; + } + const key = match[1]; + if (!preservedCredentialEnvKeys.has(key) && secretKeyPattern.test(key)) { + env[key] = ''; + } + } +} + +function getEnv(profile) { + const env = getBaseEnv(); + if (profile === 'mock') { + neutralizeCredentialEnv(env); + neutralizeDotenvSecrets(path.resolve(rootPath, '.env'), env); + Object.assign(env, mockOverrides); + } + return env; +} + +function formatDate(date) { + return date + .toISOString() + .replace(/\.\d{3}Z$/, '') + .replace(/[:T]/g, '-'); +} + +function writeRuntimeMockConfig() { + const template = fs.readFileSync(configTemplatePath, 'utf8'); + + if (!template.includes(defaultMockLlmBaseURL)) { + throw new Error(`Expected mock config template to include ${defaultMockLlmBaseURL}`); + } + + fs.mkdirSync(path.dirname(configPath), { recursive: true }); + fs.writeFileSync(configPath, template.replaceAll(defaultMockLlmBaseURL, mockLlmBaseURL)); +} + +function parseArgs(argv) { + const options = { + profile: 'mock', + output: path.resolve(rootPath, `e2e/recordings/recording-${formatDate(new Date())}.spec.ts`), + storage: storageStatePath, + url: appURL('c/new'), + authOnly: false, + saveOutput: true, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const next = argv[index + 1]; + const readValue = () => { + if (arg.includes('=')) { + return arg.slice(arg.indexOf('=') + 1); + } + index += 1; + return next; + }; + + if (arg === '--help' || arg === '-h') { + options.help = true; + } else if (arg.startsWith('--profile')) { + options.profile = readValue(); + } else if (arg.startsWith('--url')) { + const value = readValue(); + options.url = /^https?:\/\//i.test(value) ? value : appURL(value); + } else if (arg.startsWith('--output')) { + options.output = path.resolve(rootPath, readValue()); + } else if (arg.startsWith('--storage')) { + options.storage = path.resolve(rootPath, readValue()); + } else if (arg === '--no-output') { + options.saveOutput = false; + } else if (arg === '--auth-only') { + options.authOnly = true; + } + } + + return options; +} + +function printHelp() { + console.log(` +Usage: node e2e/setup/record.js [options] + +Options: + --profile mock|local Server profile to record against. Defaults to mock. + --url URL opened by Playwright codegen. Defaults to /c/new. + --output Raw recording output path under the repo. + --storage Auth storage state path. Defaults to e2e/storageState.json. + --no-output Let codegen show generated code without writing a file. + --auth-only Start servers, write storage state, then exit. + +Examples: + node e2e/setup/record.js + node e2e/setup/record.js --profile=local --url=http://localhost:3080/c/new +`); +} + +async function waitForURL(url, timeoutMs) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 1000); + try { + const response = await fetch(url, { signal: controller.signal }); + if (response.ok) { + return true; + } + } catch { + await new Promise((resolve) => setTimeout(resolve, 500)); + } finally { + clearTimeout(timeout); + } + } + return false; +} + +function spawnProcess(name, command, args, env) { + const child = spawn(command, args, { + cwd: rootPath, + env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + child.stdout.on('data', (chunk) => process.stdout.write(`[${name}] ${chunk}`)); + child.stderr.on('data', (chunk) => process.stderr.write(`[${name}] ${chunk}`)); + child.on('exit', (code) => { + if (code && code !== 0) { + console.error(`[${name}] exited with code ${code}`); + } + }); + + return child; +} + +async function stopProcess(child) { + if (!child || child.exitCode != null) { + return; + } + + child.kill('SIGTERM'); + await new Promise((resolve) => { + const timeout = setTimeout(() => { + child.kill('SIGKILL'); + resolve(); + }, 5000); + child.once('exit', () => { + clearTimeout(timeout); + resolve(); + }); + }); +} + +async function register(page, user, timeout) { + await page.getByRole('link', { name: 'Sign up' }).click({ timeout }); + await page.getByLabel('Full name').fill(user.name); + await page.getByLabel('Email').fill(user.email); + await page.getByTestId('password').fill(user.password); + await page.getByTestId('confirm_password').fill(user.password); + await page.getByLabel('Submit registration').click(); +} + +async function login(page, user) { + await page.getByLabel('Email').fill(user.email); + await page.getByLabel('Password').fill(user.password); + await page.getByTestId('login-button').click(); +} + +async function writeStorageState(env, storagePath) { + const { chromium } = require('@playwright/test'); + const user = getUser(env); + const timeout = Number(env.E2E_AUTH_TIMEOUT || 15000); + const conversationURL = appURL('c/new'); + const loginURL = appURL('login'); + const browser = await chromium.launch({ headless: true }); + + try { + const page = await browser.newPage(); + await page.context().addInitScript(() => { + localStorage.setItem('navVisible', 'true'); + }); + + await page.goto(baseURL, { timeout }); + try { + await register(page, user, timeout); + await page.waitForURL(conversationURL, { timeout }); + } catch { + await page.goto(loginURL, { timeout }); + await login(page, user); + await page.waitForURL(conversationURL, { timeout }); + } + + fs.mkdirSync(path.dirname(storagePath), { recursive: true }); + await page.context().storageState({ path: storagePath }); + console.log(`[record] Saved authenticated storage state to ${storagePath}`); + } finally { + await browser.close(); + } +} + +function runCodegen(options, env) { + const args = [ + 'playwright', + 'codegen', + '--target=playwright-test', + '--test-id-attribute=data-testid', + '--load-storage', + options.storage, + ]; + + if (options.saveOutput) { + fs.mkdirSync(path.dirname(options.output), { recursive: true }); + args.push('--output', options.output); + } + + args.push(options.url); + console.log(`[record] Opening Playwright codegen at ${options.url}`); + if (options.saveOutput) { + console.log(`[record] Raw recording will be written to ${options.output}`); + } + + return new Promise((resolve, reject) => { + const child = spawn(npxBin, args, { + cwd: rootPath, + env, + stdio: 'inherit', + }); + child.on('exit', (code) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(`Playwright codegen exited with code ${code}`)); + } + }); + }); +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + if (options.help) { + printHelp(); + return; + } + + if (!['mock', 'local'].includes(options.profile)) { + throw new Error('--profile must be "mock" or "local"'); + } + + const env = getEnv(options.profile); + const children = []; + + try { + if (options.profile === 'mock') { + writeRuntimeMockConfig(); + + const mockURL = `http://127.0.0.1:${mockLlmPort}/health`; + if (!(await waitForURL(mockURL, 1000))) { + children.push( + spawnProcess( + 'mock-llm', + 'node', + [path.resolve(rootPath, 'e2e/setup/mock-llm-server.js')], + env, + ), + ); + if (!(await waitForURL(mockURL, 30000))) { + throw new Error(`Mock LLM server did not become ready at ${mockURL}`); + } + } + } + + if (await waitForURL(baseURL, 1000)) { + if (options.profile === 'mock') { + console.warn('[record] Reusing an existing app server; make sure it uses e2e mock config.'); + } + } else { + children.push( + spawnProcess('app', 'node', [path.resolve(rootPath, 'e2e/setup/start-server.js')], env), + ); + if (!(await waitForURL(baseURL, 120000))) { + throw new Error(`LibreChat server did not become ready at ${baseURL}`); + } + } + + await writeStorageState(env, options.storage); + if (options.authOnly) { + return; + } + await runCodegen(options, env); + } finally { + for (const child of children.reverse()) { + await stopProcess(child); + } + } +} + +main().catch((error) => { + console.error('[record] Failed:', error); + process.exit(1); +}); diff --git a/e2e/setup/users.mock.ts b/e2e/setup/users.mock.ts new file mode 100644 index 0000000000..8e0a4ec3d4 --- /dev/null +++ b/e2e/setup/users.mock.ts @@ -0,0 +1,20 @@ +import type { User } from '../types'; +import { getE2EUser } from './user'; + +const DEFAULT_SECONDARY_USER: User = { + email: 'testuser-b@example.com', + name: 'Test User B', + password: 'securepassword456', +}; + +export function getPrimaryE2EUser(): User { + return getE2EUser(); +} + +export function getSecondaryE2EUser(): User { + return { + email: process.env.E2E_USER_B_EMAIL ?? DEFAULT_SECONDARY_USER.email, + name: process.env.E2E_USER_B_NAME ?? DEFAULT_SECONDARY_USER.name, + password: process.env.E2E_USER_B_PASSWORD ?? DEFAULT_SECONDARY_USER.password, + }; +} diff --git a/e2e/specs/mock/app-load.spec.ts b/e2e/specs/mock/app-load.spec.ts new file mode 100644 index 0000000000..aadb3ec4db --- /dev/null +++ b/e2e/specs/mock/app-load.spec.ts @@ -0,0 +1,22 @@ +import { expect, test } from '@playwright/test'; +import { NEW_CHAT_PATH } from './helpers'; + +test.describe('app loads cleanly', () => { + test('authenticated user lands on a rendered chat view without runtime errors', async ({ + page, + }) => { + const pageErrors: string[] = []; + page.on('pageerror', (error) => pageErrors.push(error.message)); + + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + + await expect(page).toHaveURL(/\/c\/new$/); + await expect(page.getByRole('main')).toBeVisible(); + await expect(page.getByRole('textbox', { name: 'Message input' })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Select a model' }).first()).toBeVisible(); + await expect(page.getByTestId('nav-user')).toBeVisible(); + + await expect(page.getByText(/something went wrong/i)).toHaveCount(0); + expect(pageErrors, `Unexpected runtime errors: ${pageErrors.join(', ')}`).toHaveLength(0); + }); +}); diff --git a/e2e/specs/mock/auth.spec.ts b/e2e/specs/mock/auth.spec.ts new file mode 100644 index 0000000000..d965d4270f --- /dev/null +++ b/e2e/specs/mock/auth.spec.ts @@ -0,0 +1,82 @@ +import { expect, test } from '@playwright/test'; +import type { APIRequestContext } from '@playwright/test'; +import type { User } from '../../types'; +import { getSecondaryE2EUser } from '../../setup/users.mock'; +import cleanupUser from '../../setup/cleanupUser'; +import { NEW_CHAT_PATH } from './helpers'; + +async function getIsolatedStorageState(request: APIRequestContext, user: User) { + await cleanupUser(user); + + const registerResponse = await request.post('/api/auth/register', { + data: { + email: user.email, + name: user.name, + password: user.password, + confirm_password: user.password, + }, + }); + expect(registerResponse.ok()).toBeTruthy(); + + const loginResponse = await request.post('/api/auth/login', { + data: { + email: user.email, + password: user.password, + }, + }); + expect(loginResponse.ok()).toBeTruthy(); + + return request.storageState(); +} + +test.describe('auth session', () => { + test('session persists across a full page reload', async ({ page }) => { + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await expect(page).not.toHaveURL(/\/login/); + await expect(page.getByTestId('nav-user')).toBeVisible(); + + await page.reload({ timeout: 10000 }); + + await expect(page).not.toHaveURL(/\/login/); + await expect(page.getByTestId('nav-user')).toBeVisible(); + await expect(page.getByRole('textbox', { name: 'Message input' })).toBeVisible(); + }); + + test('logout ends the session and protects authenticated routes', async ({ + request, + browser, + baseURL, + }) => { + test.setTimeout(90000); + if (typeof baseURL !== 'string') { + throw new Error('baseURL must be configured for mock auth tests'); + } + + const user = getSecondaryE2EUser(); + const context = await browser.newContext({ + storageState: await getIsolatedStorageState(request, user), + baseURL, + }); + await context.addInitScript(() => { + localStorage.setItem('navVisible', 'true'); + }); + const page = await context.newPage(); + + try { + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await expect(page).not.toHaveURL(/\/login/); + + await page.getByTestId('nav-user').click(); + await page.getByRole('menuitem', { name: 'Log out' }).click(); + + await page.waitForURL(/\/login/, { timeout: 10000 }); + await expect(page.getByLabel('Email')).toBeVisible(); + + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await expect(page).toHaveURL(/\/login/); + } finally { + await context.close().catch(() => undefined); + await cleanupUser(user); + } + }); +}); diff --git a/e2e/specs/mock/chat.spec.ts b/e2e/specs/mock/chat.spec.ts new file mode 100644 index 0000000000..60a45d5310 --- /dev/null +++ b/e2e/specs/mock/chat.spec.ts @@ -0,0 +1,37 @@ +import { expect, test } from '@playwright/test'; +import { + MOCK_ENDPOINTS, + NEW_CHAT_PATH, + mockReply, + selectMockEndpoint, + sendMessage, +} from './helpers'; + +test.describe('core chat loop', () => { + test('streams a response, saves the conversation, and persists across reload', async ({ + page, + }) => { + test.setTimeout(60000); + const userMessage = 'ping from e2e'; + + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await selectMockEndpoint(page, MOCK_ENDPOINTS[0]); + + const response = await sendMessage(page, userMessage); + expect(response.ok()).toBeTruthy(); + + await expect(page.getByText(userMessage)).toBeVisible(); + await expect(mockReply(page)).toBeVisible(); + + await expect(page).toHaveURL(/\/c\/[0-9a-fA-F-]{36}$/); + const conversationUrl = page.url(); + + await expect(page.getByTestId('convo-item').first()).toBeVisible(); + + await page.reload({ timeout: 10000 }); + await expect(page).toHaveURL(conversationUrl); + await expect(page.getByText(userMessage)).toBeVisible(); + await expect(mockReply(page)).toBeVisible(); + await expect(page.getByTestId('convo-item').first()).toBeVisible(); + }); +}); diff --git a/e2e/specs/mock/helpers.ts b/e2e/specs/mock/helpers.ts new file mode 100644 index 0000000000..f60837bba3 --- /dev/null +++ b/e2e/specs/mock/helpers.ts @@ -0,0 +1,47 @@ +import { expect } from '@playwright/test'; +import type { Page, Response } from '@playwright/test'; + +/** Substring of the reply emitted by the mock LLM server. */ +export const MOCK_REPLY_TEXT = 'E2E mock reply'; + +/** Custom endpoints defined in e2e/config/librechat.e2e.yaml. */ +export const MOCK_ENDPOINTS = [ + { label: 'Mock Provider A', model: 'mock-model-a' }, + { label: 'Mock Provider B', model: 'mock-model-b' }, +] as const; + +export type MockEndpoint = (typeof MOCK_ENDPOINTS)[number]; + +export const NEW_CHAT_PATH = '/c/new'; + +export function isAgentsStream(response: Response) { + return response.url().includes('/api/agents') && response.status() === 200; +} + +const modelSelectorTrigger = (page: Page) => + page.getByRole('button', { name: 'Select a model' }).first(); + +/** Open the model selector, choose an endpoint, then its model (committed on the model click). */ +export async function selectMockEndpoint(page: Page, endpoint: MockEndpoint) { + await modelSelectorTrigger(page).click(); + await page.getByRole('option', { name: endpoint.label }).click(); + await page.getByRole('option', { name: endpoint.model, exact: true }).click(); + await expect(modelSelectorTrigger(page)).not.toHaveText('Select a model'); +} + +/** The mock reply as rendered in the conversation, scoped to the messages view. */ +export function mockReply(page: Page) { + return page.getByTestId('messages-view').getByText(new RegExp(MOCK_REPLY_TEXT, 'i')); +} + +/** Type a message, send it, and wait for the streamed `/api/agents` response. */ +export async function sendMessage(page: Page, text: string): Promise { + const input = page.getByRole('textbox', { name: 'Message input' }); + await input.click(); + await input.fill(text); + const [response] = await Promise.all([ + page.waitForResponse(isAgentsStream, { timeout: 30000 }), + input.press('Enter'), + ]); + return response; +} diff --git a/e2e/specs/mock/isolation.spec.ts b/e2e/specs/mock/isolation.spec.ts new file mode 100644 index 0000000000..a9805813b4 --- /dev/null +++ b/e2e/specs/mock/isolation.spec.ts @@ -0,0 +1,97 @@ +import { expect, test } from '@playwright/test'; +import type { Browser, Page } from '@playwright/test'; +import type { User } from '../../types'; +import { MOCK_ENDPOINTS, NEW_CHAT_PATH, selectMockEndpoint, sendMessage } from './helpers'; +import { getSecondaryE2EUser } from '../../setup/users.mock'; +import cleanupUser from '../../setup/cleanupUser'; + +const A_PRIVATE_MARKER = 'A-private-conversation-marker'; + +async function register(page: Page, user: User) { + await page.getByRole('link', { name: 'Sign up' }).click(); + await page.getByLabel('Full name').fill(user.name); + await page.getByLabel('Email').fill(user.email); + await page.getByTestId('password').fill(user.password); + await page.getByTestId('confirm_password').fill(user.password); + await page.getByLabel('Submit registration').click(); +} + +async function registrationErrorIsVisible(page: Page) { + return page + .getByTestId('registration-error') + .isVisible({ timeout: 500 }) + .catch(() => false); +} + +async function registerSecondaryUser(page: Page, user: User) { + await page.goto('/', { timeout: 10000 }); + await page.waitForURL(/\/login/, { timeout: 10000 }); + await register(page, user); + + try { + await page.waitForURL(/\/c\/new/, { timeout: 10000 }); + } catch (error) { + if (!(await registrationErrorIsVisible(page))) { + throw error; + } + + await cleanupUser(user); + await page.goto('/', { timeout: 10000 }); + await page.waitForURL(/\/login/, { timeout: 10000 }); + await register(page, user); + await page.waitForURL(/\/c\/new/, { timeout: 10000 }); + } +} + +/** Register the secondary user in a throwaway context, then log in within `page`. */ +async function ensureSecondaryUser(browser: Browser, page: Page, user: User, baseURL: string) { + const setupContext = await browser.newContext({ storageState: undefined, baseURL }); + const setupPage = await setupContext.newPage(); + try { + await registerSecondaryUser(setupPage, user); + } finally { + await setupContext.close(); + } + + await page.goto('/login', { timeout: 10000 }); + await page.getByLabel('Email').fill(user.email); + await page.getByLabel('Password').fill(user.password); + await page.getByTestId('login-button').click(); + await page.waitForURL(/\/c\/new/, { timeout: 10000 }); +} + +test.describe('user isolation', () => { + test('user B cannot see user A conversations', async ({ page, browser, baseURL }) => { + test.setTimeout(90000); + if (typeof baseURL !== 'string') { + throw new Error('baseURL must be configured for mock isolation tests'); + } + + // User A (authenticated via storageState) creates a private conversation. + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await selectMockEndpoint(page, MOCK_ENDPOINTS[0]); + await sendMessage(page, A_PRIVATE_MARKER); + await expect(page.getByText(A_PRIVATE_MARKER)).toBeVisible(); + await expect(page).toHaveURL(/\/c\/[0-9a-fA-F-]{36}$/); + const conversationAUrl = page.url(); + + // User B in a fresh, unauthenticated context. + const contextB = await browser.newContext({ storageState: undefined, baseURL }); + const pageB = await contextB.newPage(); + try { + await ensureSecondaryUser(browser, pageB, getSecondaryE2EUser(), baseURL); + + // (a) Sidebar list does not expose A's conversation. + await pageB.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await expect(pageB.getByRole('textbox', { name: 'Message input' })).toBeVisible(); + await expect(pageB.getByText(A_PRIVATE_MARKER)).toHaveCount(0); + + // (b) Direct navigation to A's conversation does not reveal its content. + await pageB.goto(conversationAUrl, { timeout: 10000 }); + await expect(pageB.getByRole('textbox', { name: 'Message input' })).toBeVisible(); + await expect(pageB.getByText(A_PRIVATE_MARKER)).toHaveCount(0); + } finally { + await contextB.close(); + } + }); +}); diff --git a/e2e/specs/mock/model-switching.spec.ts b/e2e/specs/mock/model-switching.spec.ts new file mode 100644 index 0000000000..7c9a62de90 --- /dev/null +++ b/e2e/specs/mock/model-switching.spec.ts @@ -0,0 +1,23 @@ +import { expect, test } from '@playwright/test'; +import { + MOCK_ENDPOINTS, + NEW_CHAT_PATH, + mockReply, + selectMockEndpoint, + sendMessage, +} from './helpers'; + +test.describe('endpoint switching', () => { + for (const endpoint of MOCK_ENDPOINTS) { + test(`"${endpoint.label}" returns a streamed response`, async ({ page }) => { + test.setTimeout(60000); + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + + await selectMockEndpoint(page, endpoint); + + const response = await sendMessage(page, `hello ${endpoint.model}`); + expect(response.ok()).toBeTruthy(); + await expect(mockReply(page)).toBeVisible(); + }); + } +}); diff --git a/package.json b/package.json index 25091fdd98..4f57b42699 100644 --- a/package.json +++ b/package.json @@ -58,9 +58,13 @@ "e2e:headed": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.local.ts --headed", "e2e:a11y": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.a11y.ts --headed", "e2e:ci": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.ts", + "e2e:mock": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.mock.ts", + "e2e:mock:ci": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.mock.ts", "e2e:debug": "npm run e2e:prepare && cross-env PWDEBUG=1 playwright test --config=e2e/playwright.config.local.ts", - "e2e:codegen": "npx playwright codegen --load-storage=e2e/storageState.json http://localhost:3080/c/new", - "e2e:login": "npx playwright codegen --save-storage=e2e/auth.json http://localhost:3080/login", + "e2e:record": "npm run e2e:prepare && cross-env E2E_BASE_URL=http://localhost:3333 node e2e/setup/record.js", + "e2e:record:local": "npm run e2e:prepare && node e2e/setup/record.js --profile=local", + "e2e:codegen": "npx playwright codegen --target=playwright-test --test-id-attribute=data-testid --load-storage=e2e/storageState.json http://localhost:3080/c/new", + "e2e:login": "npx playwright codegen --save-storage=e2e/storageState.json http://localhost:3080/login", "e2e:github": "act -W .github/workflows/playwright.yml --secret-file my.secrets", "test:client": "cd client && npm run test:ci", "test:api": "cd api && npm run test:ci",