mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-08-03 22:32:42 +00:00
🎭 feat: Add Credential-Free Playwright Smoke Suite with a Local Mock LLM (#13472)
* 🧪 feat: add e2e playwright tests * 🧪 feat: Add Playwright Recording Harness * test: fix mock playwright config * test: harden mock e2e environment * test: preserve mock dotenv secrets * test: harden mock isolation setup * ci: cache mock e2e builds * test: harden e2e cache and recorder checks * test: preserve data-provider exports in oauth route test * test: isolate mock auth logout state * test: allow isolated logout smoke setup * test: prepare logout smoke auth via api * test: isolate oauth route module mock --------- Co-authored-by: Danny Avila <danny@librechat.ai>
This commit is contained in:
Teresa Blanco
GitHub
parent
317b8dfbd5
commit
b45e4aeae5
21 changed files with 1302 additions and 8 deletions
143
.github/workflows/playwright-mock.yml
vendored
Normal file
143
.github/workflows/playwright-mock.yml
vendored
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
name: Playwright E2E (Mock LLM)
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
reason:
|
||||
description: 'Reason for manual trigger'
|
||||
required: false
|
||||
default: 'Manual e2e run'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: playwright-mock-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
NODE_OPTIONS: '--max-old-space-size=${{ secrets.NODE_MAX_OLD_SPACE_SIZE || 6144 }}'
|
||||
|
||||
jobs:
|
||||
e2e:
|
||||
name: Tier-1 smoke (headless Chromium)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Use Node.js 24.16.0
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '24.16.0'
|
||||
|
||||
- name: Restore node_modules cache
|
||||
id: cache-node-modules
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
node_modules
|
||||
client/node_modules
|
||||
packages/client/node_modules
|
||||
packages/data-provider/node_modules
|
||||
packages/data-schemas/node_modules
|
||||
packages/api/node_modules
|
||||
api/node_modules
|
||||
key: node-modules-e2e-${{ runner.os }}-24.16.0-${{ hashFiles('package-lock.json') }}
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.cache-node-modules.outputs.cache-hit != 'true'
|
||||
run: npm ci
|
||||
|
||||
- name: Restore data-provider build cache
|
||||
id: cache-data-provider
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: packages/data-provider/dist
|
||||
key: build-data-provider-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/rollup.config.js', 'packages/data-provider/package.json') }}
|
||||
|
||||
- name: Build data-provider
|
||||
if: steps.cache-data-provider.outputs.cache-hit != 'true'
|
||||
run: npm run build:data-provider
|
||||
|
||||
- name: Restore data-schemas build cache
|
||||
id: cache-data-schemas
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: packages/data-schemas/dist
|
||||
key: build-data-schemas-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/data-schemas/src/**', 'packages/data-schemas/tsconfig*.json', 'packages/data-schemas/rollup.config.js', 'packages/data-schemas/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/rollup.config.js', 'packages/data-provider/package.json') }}
|
||||
|
||||
- name: Build data-schemas
|
||||
if: steps.cache-data-schemas.outputs.cache-hit != 'true'
|
||||
run: npm run build:data-schemas
|
||||
|
||||
- name: Restore api build cache
|
||||
id: cache-api
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: packages/api/dist
|
||||
key: build-api-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/api/src/**', 'packages/api/tsconfig*.json', 'packages/api/server-rollup.config.js', 'packages/api/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/rollup.config.js', 'packages/data-provider/package.json', 'packages/data-schemas/src/**', 'packages/data-schemas/tsconfig*.json', 'packages/data-schemas/rollup.config.js', 'packages/data-schemas/package.json') }}
|
||||
|
||||
- name: Build api
|
||||
if: steps.cache-api.outputs.cache-hit != 'true'
|
||||
run: npm run build:api
|
||||
|
||||
- name: Restore client-package build cache
|
||||
id: cache-client-package
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: packages/client/dist
|
||||
key: build-client-package-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/client/src/**', 'packages/client/tsconfig*.json', 'packages/client/rollup.config.js', 'packages/client/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/rollup.config.js', 'packages/data-provider/package.json') }}
|
||||
|
||||
- name: Build client-package
|
||||
if: steps.cache-client-package.outputs.cache-hit != 'true'
|
||||
run: npm run build:client-package
|
||||
|
||||
- name: Restore client app build cache
|
||||
id: cache-client-app
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: client/dist
|
||||
key: build-client-app-e2e-${{ runner.os }}-${{ hashFiles('package-lock.json', 'client/src/**', 'client/public/**', 'client/scripts/post-build.cjs', 'client/index.html', 'client/package.json', 'client/vite.config.*', 'client/tsconfig*.json', 'client/tailwind.config.*', 'client/postcss.config.*', 'packages/client/src/**', 'packages/client/tsconfig*.json', 'packages/client/rollup.config.js', 'packages/client/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/rollup.config.js', 'packages/data-provider/package.json') }}
|
||||
|
||||
- name: Build client app
|
||||
if: steps.cache-client-app.outputs.cache-hit != 'true'
|
||||
run: npm run build:client
|
||||
|
||||
- name: Resolve Playwright version
|
||||
id: pw
|
||||
run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache Playwright browsers
|
||||
id: cache-playwright
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
key: playwright-${{ runner.os }}-${{ steps.pw.outputs.version }}
|
||||
|
||||
- name: Install Playwright Chromium
|
||||
run: npx playwright install --with-deps chromium
|
||||
|
||||
- name: Run mock-LLM Tier-1 e2e
|
||||
run: npx playwright test --config=e2e/playwright.config.mock.ts
|
||||
env:
|
||||
CI: 'true'
|
||||
|
||||
- name: Upload Playwright HTML report
|
||||
if: ${{ !cancelled() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: playwright-report
|
||||
path: e2e/playwright-report/**
|
||||
retention-days: 7
|
||||
if-no-files-found: ignore
|
||||
|
||||
- name: Upload traces & screenshots
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: playwright-test-results
|
||||
path: e2e/specs/.test-results/**
|
||||
retention-days: 7
|
||||
if-no-files-found: ignore
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -88,6 +88,7 @@ archive
|
|||
.vscode/settings.json
|
||||
src/style - official.css
|
||||
/e2e/specs/.test-results/
|
||||
/e2e/.generated/
|
||||
/e2e/playwright-report/
|
||||
/playwright/.cache/
|
||||
.DS_Store
|
||||
|
|
|
|||
|
|
@ -281,7 +281,10 @@ const startServer = async () => {
|
|||
|
||||
// Configure stream services (auto-detects Redis from USE_REDIS env var)
|
||||
const streamServices = createStreamServices();
|
||||
GenerationJobManager.configure(streamServices);
|
||||
GenerationJobManager.configure({
|
||||
...streamServices,
|
||||
cleanupOnComplete: !isEnabled(process.env.STREAM_KEEP_COMPLETED_JOBS),
|
||||
});
|
||||
GenerationJobManager.initialize();
|
||||
|
||||
const inspectFlags = process.execArgv.some((arg) => arg.startsWith('--inspect'));
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ jest.mock('@librechat/data-schemas', () => ({
|
|||
}));
|
||||
|
||||
jest.mock('librechat-data-provider', () => ({
|
||||
...jest.requireActual('librechat-data-provider'),
|
||||
ErrorTypes: {
|
||||
AUTH_FAILED: 'auth_failed',
|
||||
},
|
||||
|
|
@ -99,8 +100,6 @@ jest.mock('~/server/services/Config', () => ({
|
|||
getAppConfig: jest.fn(),
|
||||
}));
|
||||
|
||||
const oauthRouter = require('./oauth');
|
||||
|
||||
afterAll(() => {
|
||||
if (originalDomainClient === undefined) {
|
||||
delete process.env.DOMAIN_CLIENT;
|
||||
|
|
@ -109,6 +108,11 @@ afterAll(() => {
|
|||
process.env.DOMAIN_CLIENT = originalDomainClient;
|
||||
});
|
||||
|
||||
function getOAuthRouter() {
|
||||
jest.resetModules();
|
||||
return require('./oauth');
|
||||
}
|
||||
|
||||
function createApp(sessionMessages) {
|
||||
const app = express();
|
||||
app.use((req, _res, next) => {
|
||||
|
|
@ -117,7 +121,7 @@ function createApp(sessionMessages) {
|
|||
}
|
||||
next();
|
||||
});
|
||||
app.use('/oauth', oauthRouter);
|
||||
app.use('/oauth', getOAuthRouter());
|
||||
app.use((err, _req, res, _next) => {
|
||||
res.status(500).json({ message: err.message });
|
||||
});
|
||||
|
|
@ -136,6 +140,7 @@ describe('OAuth route failure logging', () => {
|
|||
mockGetOAuthFailureMessage.mockClear();
|
||||
mockRedirectToAuthFailure.mockClear();
|
||||
mockPassportAuthenticate.mockClear();
|
||||
mockOpenIDCallbackAuthenticatorOptions = undefined;
|
||||
mockPassportAuthenticate.mockImplementation(() => (_req, _res, next) => next());
|
||||
mockOpenIDCallbackMiddleware.mockImplementation((_req, _res, next) => next());
|
||||
});
|
||||
|
|
|
|||
40
e2e/README.md
Normal file
40
e2e/README.md
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
# LibreChat e2e
|
||||
|
||||
The mock e2e profile is the safest default for generated tests. It starts LibreChat with `e2e/config/librechat.e2e.yaml`, points custom endpoints at the local mock LLM server, creates an authenticated e2e user, and avoids real provider credentials.
|
||||
|
||||
## Recording Tests
|
||||
|
||||
Use Playwright codegen when you want to turn an exploratory browser session into a draft test:
|
||||
|
||||
```sh
|
||||
npm run e2e:record
|
||||
```
|
||||
|
||||
That command builds the app, starts the mock LLM and LibreChat test server when needed, writes `e2e/storageState.json`, and opens Playwright codegen at `/c/new`. The npm script uses `http://localhost:3333` so it does not collide with a normal dev server on `3080`. Raw recordings are written to `e2e/recordings/` and ignored by git.
|
||||
|
||||
For a real local LibreChat config instead of the mock LLM profile:
|
||||
|
||||
```sh
|
||||
npm run e2e:record:local
|
||||
```
|
||||
|
||||
Useful direct options:
|
||||
|
||||
```sh
|
||||
node e2e/setup/record.js --url=http://localhost:3080/c/new
|
||||
node e2e/setup/record.js --profile=local --no-output
|
||||
node e2e/setup/record.js --auth-only
|
||||
node e2e/setup/record.js --output=e2e/recordings/settings-draft.spec.ts
|
||||
```
|
||||
|
||||
## LLM-Assisted Loop
|
||||
|
||||
1. Start `npm run e2e:record`.
|
||||
2. Let the LLM use Computer Use to operate the headed Playwright browser.
|
||||
3. Stop codegen after the workflow is captured.
|
||||
4. Move the useful parts from `e2e/recordings/` into a committed spec under `e2e/specs/mock/`.
|
||||
5. Replace brittle generated selectors with role, label, text, or `data-testid` locators.
|
||||
6. Add assertions that prove the behavior, not just the clicked path.
|
||||
7. Run the finished spec with `npm run e2e:mock -- <spec name>`.
|
||||
|
||||
Generated recordings are a draft, not the final test. The committed version should use the shared helpers in `e2e/specs/mock/helpers.ts` where possible, wait on network or visible UI state instead of fixed sleeps, and keep test data deterministic.
|
||||
26
e2e/config/librechat.e2e.yaml
Normal file
26
e2e/config/librechat.e2e.yaml
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
# Credential-free e2e config template. e2e/playwright.config.mock.ts writes
|
||||
# an ignored runtime copy and rewrites the mock LLM port from MOCK_LLM_PORT.
|
||||
version: 1.3.11
|
||||
cache: true
|
||||
|
||||
endpoints:
|
||||
custom:
|
||||
- name: 'Mock Provider A'
|
||||
apiKey: 'e2e-mock-key-a'
|
||||
baseURL: 'http://127.0.0.1:8889/v1'
|
||||
models:
|
||||
default:
|
||||
- 'mock-model-a'
|
||||
fetch: false
|
||||
titleConvo: false
|
||||
modelDisplayLabel: 'Mock Provider A'
|
||||
|
||||
- name: 'Mock Provider B'
|
||||
apiKey: 'e2e-mock-key-b'
|
||||
baseURL: 'http://127.0.0.1:8889/v1'
|
||||
models:
|
||||
default:
|
||||
- 'mock-model-b'
|
||||
fetch: false
|
||||
titleConvo: false
|
||||
modelDisplayLabel: 'Mock Provider B'
|
||||
145
e2e/playwright.config.mock.ts
Normal file
145
e2e/playwright.config.mock.ts
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
import { defineConfig, devices } from '@playwright/test';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { getLocalE2EEnv, getE2EBaseURL } from './setup/env';
|
||||
|
||||
const rootPath = path.resolve(__dirname, '..');
|
||||
const serverPath = path.resolve(rootPath, 'e2e/setup/start-server.js');
|
||||
const mockLlmPath = path.resolve(rootPath, 'e2e/setup/mock-llm-server.js');
|
||||
const configTemplatePath = path.resolve(rootPath, 'e2e/config/librechat.e2e.yaml');
|
||||
const configPath = path.resolve(rootPath, 'e2e/.generated/librechat.e2e.yaml');
|
||||
const reportPath = path.resolve(rootPath, 'e2e/playwright-report');
|
||||
|
||||
const baseURL = getE2EBaseURL();
|
||||
const mockLlmPort = getMockLlmPort();
|
||||
const defaultMockLlmBaseURL = 'http://127.0.0.1:8889/v1';
|
||||
const mockLlmBaseURL = `http://127.0.0.1:${mockLlmPort}/v1`;
|
||||
|
||||
const vanillaOverrides = {
|
||||
TENANT_ISOLATION_STRICT: 'false',
|
||||
OPENAI_API_KEY: 'user_provided',
|
||||
OPENID_CLIENT_ID: '',
|
||||
OPENID_ISSUER: '',
|
||||
OPENID_AUTO_REDIRECT: 'false',
|
||||
ALLOW_SOCIAL_LOGIN: 'false',
|
||||
ALLOW_SOCIAL_REGISTRATION: 'false',
|
||||
STREAM_KEEP_COMPLETED_JOBS: 'true',
|
||||
};
|
||||
|
||||
const baseEnv = {
|
||||
...getLocalE2EEnv(),
|
||||
CONFIG_PATH: configPath,
|
||||
MOCK_LLM_PORT: mockLlmPort,
|
||||
...vanillaOverrides,
|
||||
};
|
||||
|
||||
const SECRET_KEY_PATTERN = /(API_KEY|SECRET|TOKEN|PASSWORD|CREDENTIALS|CLIENT_ID|_KEY)$/i;
|
||||
const preservedCredentialEnvKeys = new Set([
|
||||
...Object.keys(baseEnv),
|
||||
'E2E_USER_PASSWORD',
|
||||
'E2E_USER_B_PASSWORD',
|
||||
]);
|
||||
|
||||
function getMockLlmPort() {
|
||||
const port = process.env.MOCK_LLM_PORT ?? '8889';
|
||||
if (!/^\d+$/.test(port)) {
|
||||
throw new Error('MOCK_LLM_PORT must be a numeric port');
|
||||
}
|
||||
return port;
|
||||
}
|
||||
|
||||
function writeRuntimeMockConfig() {
|
||||
const template = fs.readFileSync(configTemplatePath, 'utf8');
|
||||
|
||||
if (!template.includes(defaultMockLlmBaseURL)) {
|
||||
throw new Error(`Expected mock config template to include ${defaultMockLlmBaseURL}`);
|
||||
}
|
||||
|
||||
fs.mkdirSync(path.dirname(configPath), { recursive: true });
|
||||
fs.writeFileSync(configPath, template.replaceAll(defaultMockLlmBaseURL, mockLlmBaseURL));
|
||||
}
|
||||
|
||||
function neutralizeCredentialEnv(env: NodeJS.ProcessEnv, keep: Set<string>) {
|
||||
for (const key of Object.keys(env)) {
|
||||
if (!keep.has(key) && SECRET_KEY_PATTERN.test(key)) {
|
||||
env[key] = '';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Blank any credential-like vars from a local `.env` so they never reach the test server. */
|
||||
function neutralizeDotenvSecrets(envFile: string, keep: Set<string>) {
|
||||
if (!fs.existsSync(envFile)) {
|
||||
return;
|
||||
}
|
||||
const lines = fs.readFileSync(envFile, 'utf8').split('\n');
|
||||
for (const line of lines) {
|
||||
const match = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=/);
|
||||
if (!match) {
|
||||
continue;
|
||||
}
|
||||
const key = match[1];
|
||||
if (keep.has(key)) {
|
||||
continue;
|
||||
}
|
||||
if (SECRET_KEY_PATTERN.test(key)) {
|
||||
process.env[key] = '';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
writeRuntimeMockConfig();
|
||||
neutralizeCredentialEnv(process.env, preservedCredentialEnvKeys);
|
||||
Object.assign(process.env, baseEnv);
|
||||
neutralizeDotenvSecrets(path.resolve(rootPath, '.env'), preservedCredentialEnvKeys);
|
||||
|
||||
export default defineConfig({
|
||||
globalSetup: require.resolve('./setup/global-setup'),
|
||||
globalTeardown: require.resolve('./setup/global-teardown.mock'),
|
||||
testDir: 'specs/mock/',
|
||||
outputDir: 'specs/.test-results',
|
||||
fullyParallel: false,
|
||||
forbidOnly: !!process.env.CI,
|
||||
retries: process.env.CI ? 2 : 0,
|
||||
workers: 1,
|
||||
reporter: process.env.CI
|
||||
? [['html', { outputFolder: reportPath, open: 'never' }], ['line']]
|
||||
: [['html', { outputFolder: reportPath }], ['list']],
|
||||
use: {
|
||||
baseURL,
|
||||
video: 'on-first-retry',
|
||||
trace: 'retain-on-failure',
|
||||
ignoreHTTPSErrors: true,
|
||||
headless: true,
|
||||
storageState: path.resolve(process.cwd(), 'e2e/storageState.json'),
|
||||
screenshot: 'only-on-failure',
|
||||
},
|
||||
expect: {
|
||||
timeout: 10000,
|
||||
},
|
||||
projects: [
|
||||
{
|
||||
name: 'chromium',
|
||||
use: { ...devices['Desktop Chrome'] },
|
||||
},
|
||||
],
|
||||
webServer: [
|
||||
{
|
||||
command: `node ${mockLlmPath}`,
|
||||
cwd: rootPath,
|
||||
url: `http://127.0.0.1:${mockLlmPort}/health`,
|
||||
stdout: 'pipe',
|
||||
timeout: 30_000,
|
||||
reuseExistingServer: false,
|
||||
},
|
||||
{
|
||||
command: `node ${serverPath}`,
|
||||
cwd: rootPath,
|
||||
url: baseURL,
|
||||
stdout: 'pipe',
|
||||
ignoreHTTPSErrors: true,
|
||||
timeout: 120_000,
|
||||
reuseExistingServer: false,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
|
@ -27,6 +27,7 @@ export default defineConfig({
|
|||
workers: process.env.CI ? 1 : undefined,
|
||||
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
|
||||
reporter: [['html', { outputFolder: 'playwright-report' }]],
|
||||
testIgnore: ['**/mock/**'],
|
||||
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
|
||||
use: {
|
||||
baseURL,
|
||||
|
|
|
|||
2
e2e/recordings/.gitignore
vendored
Normal file
2
e2e/recordings/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
*
|
||||
!.gitignore
|
||||
|
|
@ -93,9 +93,9 @@ export function getBaseE2EEnv(): Record<string, string> {
|
|||
JWT_REFRESH_SECRET: process.env.JWT_REFRESH_SECRET ?? GENERATED_JWT_REFRESH_SECRET,
|
||||
EMAIL_HOST: '',
|
||||
SEARCH: 'false',
|
||||
SESSION_EXPIRY: '60000',
|
||||
SESSION_EXPIRY: process.env.SESSION_EXPIRY ?? '3600000',
|
||||
ALLOW_REGISTRATION: 'true',
|
||||
REFRESH_TOKEN_EXPIRY: '300000',
|
||||
REFRESH_TOKEN_EXPIRY: process.env.REFRESH_TOKEN_EXPIRY ?? '3600000',
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
14
e2e/setup/global-teardown.mock.ts
Normal file
14
e2e/setup/global-teardown.mock.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
import cleanupUser from './cleanupUser';
|
||||
import { getPrimaryE2EUser, getSecondaryE2EUser } from './users.mock';
|
||||
|
||||
async function globalTeardown() {
|
||||
for (const user of [getPrimaryE2EUser(), getSecondaryE2EUser()]) {
|
||||
try {
|
||||
await cleanupUser(user);
|
||||
} catch (error) {
|
||||
console.error('Error:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default globalTeardown;
|
||||
128
e2e/setup/mock-llm-server.js
Normal file
128
e2e/setup/mock-llm-server.js
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
/**
|
||||
* OpenAI-compatible mock server for credential-free e2e tests. Answers
|
||||
* `${baseURL}/chat/completions` with deterministic content. Run standalone
|
||||
* (Playwright `webServer`) or import `startMockLlm()` for programmatic control.
|
||||
*/
|
||||
const http = require('http');
|
||||
|
||||
const DEFAULT_PORT = 8889;
|
||||
const MOCK_REPLY = process.env.MOCK_LLM_REPLY || 'E2E mock reply: pong';
|
||||
const MODEL_FALLBACK = 'mock-model';
|
||||
const STREAM_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_CHUNK_DELAY_MS) || 60;
|
||||
|
||||
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
function readJsonBody(req) {
|
||||
return new Promise((resolve) => {
|
||||
let raw = '';
|
||||
req.on('data', (chunk) => {
|
||||
raw += chunk;
|
||||
});
|
||||
req.on('end', () => {
|
||||
try {
|
||||
resolve(raw ? JSON.parse(raw) : {});
|
||||
} catch {
|
||||
resolve({});
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function toChunks(text) {
|
||||
return text.match(/\S+\s*/g) || [text];
|
||||
}
|
||||
|
||||
async function streamCompletion(res, model) {
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
Connection: 'keep-alive',
|
||||
});
|
||||
if (typeof res.flushHeaders === 'function') {
|
||||
res.flushHeaders();
|
||||
}
|
||||
|
||||
const id = 'chatcmpl-e2e-mock';
|
||||
const created = 1700000000;
|
||||
const base = { id, object: 'chat.completion.chunk', created, model };
|
||||
|
||||
const send = (delta, finishReason = null) => {
|
||||
const payload = {
|
||||
...base,
|
||||
choices: [{ index: 0, delta, finish_reason: finishReason }],
|
||||
};
|
||||
res.write(`data: ${JSON.stringify(payload)}\n\n`);
|
||||
};
|
||||
|
||||
send({ role: 'assistant', content: '' });
|
||||
for (const chunk of toChunks(MOCK_REPLY)) {
|
||||
await delay(STREAM_CHUNK_DELAY_MS);
|
||||
send({ content: chunk });
|
||||
}
|
||||
send({}, 'stop');
|
||||
res.write('data: [DONE]\n\n');
|
||||
res.end();
|
||||
}
|
||||
|
||||
function jsonCompletion(res, model) {
|
||||
const payload = {
|
||||
id: 'chatcmpl-e2e-mock',
|
||||
object: 'chat.completion',
|
||||
created: 1700000000,
|
||||
model,
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
message: { role: 'assistant', content: MOCK_REPLY },
|
||||
finish_reason: 'stop',
|
||||
},
|
||||
],
|
||||
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
|
||||
};
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify(payload));
|
||||
}
|
||||
|
||||
async function handleRequest(req, res) {
|
||||
if (req.method === 'GET' && (req.url === '/health' || req.url === '/')) {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ status: 'ok' }));
|
||||
return;
|
||||
}
|
||||
|
||||
if (req.method === 'POST' && req.url && req.url.endsWith('/chat/completions')) {
|
||||
const body = await readJsonBody(req);
|
||||
const model = body.model || MODEL_FALLBACK;
|
||||
if (body.stream) {
|
||||
await streamCompletion(res, model);
|
||||
} else {
|
||||
jsonCompletion(res, model);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
res.writeHead(404, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'not found' }));
|
||||
}
|
||||
|
||||
function startMockLlm(port = Number(process.env.MOCK_LLM_PORT) || DEFAULT_PORT) {
|
||||
const server = http.createServer((req, res) => {
|
||||
handleRequest(req, res).catch(() => {
|
||||
res.writeHead(500, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'mock server error' }));
|
||||
});
|
||||
});
|
||||
|
||||
return new Promise((resolve) => {
|
||||
server.listen(port, '127.0.0.1', () => {
|
||||
console.log(`[e2e] Mock LLM server listening on http://127.0.0.1:${port}`);
|
||||
resolve(server);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
startMockLlm();
|
||||
}
|
||||
|
||||
module.exports = { startMockLlm, MOCK_REPLY };
|
||||
454
e2e/setup/record.js
Normal file
454
e2e/setup/record.js
Normal file
|
|
@ -0,0 +1,454 @@
|
|||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const { spawn } = require('child_process');
|
||||
|
||||
const rootPath = path.resolve(__dirname, '../..');
|
||||
const baseURL = process.env.E2E_BASE_URL || 'http://localhost:3080';
|
||||
const storageStatePath = path.resolve(rootPath, 'e2e/storageState.json');
|
||||
const configTemplatePath = path.resolve(rootPath, 'e2e/config/librechat.e2e.yaml');
|
||||
const configPath = path.resolve(rootPath, 'e2e/.generated/librechat.e2e.yaml');
|
||||
const defaultMockLlmBaseURL = 'http://127.0.0.1:8889/v1';
|
||||
const mockLlmPort = getMockLlmPort();
|
||||
const mockLlmBaseURL = `http://127.0.0.1:${mockLlmPort}/v1`;
|
||||
const defaultUser = {
|
||||
email: 'testuser@example.com',
|
||||
name: 'Test User',
|
||||
password: 'securepassword123',
|
||||
};
|
||||
|
||||
const rateLimitOverrides = {
|
||||
LOGIN_VIOLATION_SCORE: '0',
|
||||
REGISTRATION_VIOLATION_SCORE: '0',
|
||||
CONCURRENT_VIOLATION_SCORE: '0',
|
||||
MESSAGE_VIOLATION_SCORE: '0',
|
||||
NON_BROWSER_VIOLATION_SCORE: '0',
|
||||
FORK_VIOLATION_SCORE: '0',
|
||||
IMPORT_VIOLATION_SCORE: '0',
|
||||
TTS_VIOLATION_SCORE: '0',
|
||||
STT_VIOLATION_SCORE: '0',
|
||||
FILE_UPLOAD_VIOLATION_SCORE: '0',
|
||||
RESET_PASSWORD_VIOLATION_SCORE: '0',
|
||||
VERIFY_EMAIL_VIOLATION_SCORE: '0',
|
||||
TOOL_CALL_VIOLATION_SCORE: '0',
|
||||
CONVO_ACCESS_VIOLATION_SCORE: '0',
|
||||
ILLEGAL_MODEL_REQ_SCORE: '0',
|
||||
LOGIN_MAX: '20',
|
||||
LOGIN_WINDOW: '1',
|
||||
REGISTER_MAX: '20',
|
||||
REGISTER_WINDOW: '1',
|
||||
LIMIT_CONCURRENT_MESSAGES: 'false',
|
||||
CONCURRENT_MESSAGE_MAX: '20',
|
||||
LIMIT_MESSAGE_IP: 'false',
|
||||
MESSAGE_IP_MAX: '100',
|
||||
MESSAGE_IP_WINDOW: '1',
|
||||
LIMIT_MESSAGE_USER: 'false',
|
||||
MESSAGE_USER_MAX: '100',
|
||||
MESSAGE_USER_WINDOW: '1',
|
||||
};
|
||||
|
||||
const mockOverrides = {
|
||||
CONFIG_PATH: configPath,
|
||||
MOCK_LLM_PORT: mockLlmPort,
|
||||
OPENAI_API_KEY: 'user_provided',
|
||||
TENANT_ISOLATION_STRICT: 'false',
|
||||
OPENID_CLIENT_ID: '',
|
||||
OPENID_ISSUER: '',
|
||||
OPENID_AUTO_REDIRECT: 'false',
|
||||
ALLOW_SOCIAL_LOGIN: 'false',
|
||||
ALLOW_SOCIAL_REGISTRATION: 'false',
|
||||
STREAM_KEEP_COMPLETED_JOBS: 'true',
|
||||
};
|
||||
|
||||
const secretKeyPattern = /(API_KEY|SECRET|TOKEN|PASSWORD|CREDENTIALS|CLIENT_ID|_KEY)$/i;
|
||||
const preservedCredentialEnvKeys = new Set([
|
||||
...Object.keys(rateLimitOverrides),
|
||||
...Object.keys(mockOverrides).filter((key) => key !== 'OPENAI_API_KEY'),
|
||||
'CREDS_KEY',
|
||||
'CREDS_IV',
|
||||
'E2E_USER_PASSWORD',
|
||||
'E2E_USER_B_PASSWORD',
|
||||
'JWT_SECRET',
|
||||
'JWT_REFRESH_SECRET',
|
||||
'REFRESH_TOKEN_EXPIRY',
|
||||
'SESSION_EXPIRY',
|
||||
]);
|
||||
const npxBin = process.platform === 'win32' ? 'npx.cmd' : 'npx';
|
||||
|
||||
function getMockLlmPort() {
|
||||
const port = process.env.MOCK_LLM_PORT || '8889';
|
||||
if (!/^\d+$/.test(port)) {
|
||||
throw new Error('MOCK_LLM_PORT must be a numeric port');
|
||||
}
|
||||
return port;
|
||||
}
|
||||
|
||||
function appURL(pathname = '') {
|
||||
const normalizedBaseURL = baseURL.endsWith('/') ? baseURL : `${baseURL}/`;
|
||||
return new URL(pathname.replace(/^\/+/, ''), normalizedBaseURL).toString();
|
||||
}
|
||||
|
||||
function getServerAddress() {
|
||||
const url = new URL(baseURL);
|
||||
const host = url.hostname.replace(/^\[(.*)\]$/, '$1');
|
||||
const port = url.port || (url.protocol === 'https:' ? '443' : '80');
|
||||
return { host, port };
|
||||
}
|
||||
|
||||
function randomHex(bytes) {
|
||||
return crypto.randomBytes(bytes).toString('hex');
|
||||
}
|
||||
|
||||
function getUser(env) {
|
||||
return {
|
||||
email: env.E2E_USER_EMAIL || defaultUser.email,
|
||||
name: env.E2E_USER_NAME || defaultUser.name,
|
||||
password: env.E2E_USER_PASSWORD || defaultUser.password,
|
||||
};
|
||||
}
|
||||
|
||||
function getBaseEnv() {
|
||||
const { host, port } = getServerAddress();
|
||||
return {
|
||||
...process.env,
|
||||
NODE_ENV: 'CI',
|
||||
HOST: process.env.E2E_HOST || host,
|
||||
PORT: process.env.E2E_PORT || port,
|
||||
MONGO_URI: process.env.MONGO_URI || 'mongodb://127.0.0.1:27017/LibreChat-e2e',
|
||||
DOMAIN_CLIENT: process.env.E2E_DOMAIN_CLIENT || baseURL,
|
||||
DOMAIN_SERVER: process.env.E2E_DOMAIN_SERVER || baseURL,
|
||||
E2E_RUNTIME_ENV_PATH:
|
||||
process.env.E2E_RUNTIME_ENV_PATH ||
|
||||
path.resolve(rootPath, 'e2e/specs/.test-results/runtime-env.json'),
|
||||
E2E_USE_MEMORY_MONGO: process.env.E2E_USE_MEMORY_MONGO || 'auto',
|
||||
NO_INDEX: process.env.NO_INDEX || 'true',
|
||||
OPENAI_API_KEY: process.env.OPENAI_API_KEY || 'user_provided',
|
||||
CREDS_KEY: process.env.CREDS_KEY || randomHex(32),
|
||||
CREDS_IV: process.env.CREDS_IV || randomHex(16),
|
||||
JWT_SECRET: process.env.JWT_SECRET || randomHex(32),
|
||||
JWT_REFRESH_SECRET: process.env.JWT_REFRESH_SECRET || randomHex(32),
|
||||
EMAIL_HOST: '',
|
||||
SEARCH: 'false',
|
||||
SESSION_EXPIRY: process.env.SESSION_EXPIRY || '3600000',
|
||||
ALLOW_REGISTRATION: 'true',
|
||||
REFRESH_TOKEN_EXPIRY: process.env.REFRESH_TOKEN_EXPIRY || '3600000',
|
||||
TITLE_CONVO: 'false',
|
||||
...rateLimitOverrides,
|
||||
};
|
||||
}
|
||||
|
||||
function neutralizeCredentialEnv(env) {
|
||||
for (const key of Object.keys(env)) {
|
||||
if (!preservedCredentialEnvKeys.has(key) && secretKeyPattern.test(key)) {
|
||||
env[key] = '';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function neutralizeDotenvSecrets(envFile, env) {
|
||||
if (!fs.existsSync(envFile)) {
|
||||
return;
|
||||
}
|
||||
const lines = fs.readFileSync(envFile, 'utf8').split('\n');
|
||||
for (const line of lines) {
|
||||
const match = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=/);
|
||||
if (!match) {
|
||||
continue;
|
||||
}
|
||||
const key = match[1];
|
||||
if (!preservedCredentialEnvKeys.has(key) && secretKeyPattern.test(key)) {
|
||||
env[key] = '';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getEnv(profile) {
|
||||
const env = getBaseEnv();
|
||||
if (profile === 'mock') {
|
||||
neutralizeCredentialEnv(env);
|
||||
neutralizeDotenvSecrets(path.resolve(rootPath, '.env'), env);
|
||||
Object.assign(env, mockOverrides);
|
||||
}
|
||||
return env;
|
||||
}
|
||||
|
||||
function formatDate(date) {
|
||||
return date
|
||||
.toISOString()
|
||||
.replace(/\.\d{3}Z$/, '')
|
||||
.replace(/[:T]/g, '-');
|
||||
}
|
||||
|
||||
function writeRuntimeMockConfig() {
|
||||
const template = fs.readFileSync(configTemplatePath, 'utf8');
|
||||
|
||||
if (!template.includes(defaultMockLlmBaseURL)) {
|
||||
throw new Error(`Expected mock config template to include ${defaultMockLlmBaseURL}`);
|
||||
}
|
||||
|
||||
fs.mkdirSync(path.dirname(configPath), { recursive: true });
|
||||
fs.writeFileSync(configPath, template.replaceAll(defaultMockLlmBaseURL, mockLlmBaseURL));
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = {
|
||||
profile: 'mock',
|
||||
output: path.resolve(rootPath, `e2e/recordings/recording-${formatDate(new Date())}.spec.ts`),
|
||||
storage: storageStatePath,
|
||||
url: appURL('c/new'),
|
||||
authOnly: false,
|
||||
saveOutput: true,
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
const next = argv[index + 1];
|
||||
const readValue = () => {
|
||||
if (arg.includes('=')) {
|
||||
return arg.slice(arg.indexOf('=') + 1);
|
||||
}
|
||||
index += 1;
|
||||
return next;
|
||||
};
|
||||
|
||||
if (arg === '--help' || arg === '-h') {
|
||||
options.help = true;
|
||||
} else if (arg.startsWith('--profile')) {
|
||||
options.profile = readValue();
|
||||
} else if (arg.startsWith('--url')) {
|
||||
const value = readValue();
|
||||
options.url = /^https?:\/\//i.test(value) ? value : appURL(value);
|
||||
} else if (arg.startsWith('--output')) {
|
||||
options.output = path.resolve(rootPath, readValue());
|
||||
} else if (arg.startsWith('--storage')) {
|
||||
options.storage = path.resolve(rootPath, readValue());
|
||||
} else if (arg === '--no-output') {
|
||||
options.saveOutput = false;
|
||||
} else if (arg === '--auth-only') {
|
||||
options.authOnly = true;
|
||||
}
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
Usage: node e2e/setup/record.js [options]
|
||||
|
||||
Options:
|
||||
--profile mock|local Server profile to record against. Defaults to mock.
|
||||
--url <url> URL opened by Playwright codegen. Defaults to /c/new.
|
||||
--output <path> Raw recording output path under the repo.
|
||||
--storage <path> Auth storage state path. Defaults to e2e/storageState.json.
|
||||
--no-output Let codegen show generated code without writing a file.
|
||||
--auth-only Start servers, write storage state, then exit.
|
||||
|
||||
Examples:
|
||||
node e2e/setup/record.js
|
||||
node e2e/setup/record.js --profile=local --url=http://localhost:3080/c/new
|
||||
`);
|
||||
}
|
||||
|
||||
async function waitForURL(url, timeoutMs) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 1000);
|
||||
try {
|
||||
const response = await fetch(url, { signal: controller.signal });
|
||||
if (response.ok) {
|
||||
return true;
|
||||
}
|
||||
} catch {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function spawnProcess(name, command, args, env) {
|
||||
const child = spawn(command, args, {
|
||||
cwd: rootPath,
|
||||
env,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
|
||||
child.stdout.on('data', (chunk) => process.stdout.write(`[${name}] ${chunk}`));
|
||||
child.stderr.on('data', (chunk) => process.stderr.write(`[${name}] ${chunk}`));
|
||||
child.on('exit', (code) => {
|
||||
if (code && code !== 0) {
|
||||
console.error(`[${name}] exited with code ${code}`);
|
||||
}
|
||||
});
|
||||
|
||||
return child;
|
||||
}
|
||||
|
||||
async function stopProcess(child) {
|
||||
if (!child || child.exitCode != null) {
|
||||
return;
|
||||
}
|
||||
|
||||
child.kill('SIGTERM');
|
||||
await new Promise((resolve) => {
|
||||
const timeout = setTimeout(() => {
|
||||
child.kill('SIGKILL');
|
||||
resolve();
|
||||
}, 5000);
|
||||
child.once('exit', () => {
|
||||
clearTimeout(timeout);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function register(page, user, timeout) {
|
||||
await page.getByRole('link', { name: 'Sign up' }).click({ timeout });
|
||||
await page.getByLabel('Full name').fill(user.name);
|
||||
await page.getByLabel('Email').fill(user.email);
|
||||
await page.getByTestId('password').fill(user.password);
|
||||
await page.getByTestId('confirm_password').fill(user.password);
|
||||
await page.getByLabel('Submit registration').click();
|
||||
}
|
||||
|
||||
async function login(page, user) {
|
||||
await page.getByLabel('Email').fill(user.email);
|
||||
await page.getByLabel('Password').fill(user.password);
|
||||
await page.getByTestId('login-button').click();
|
||||
}
|
||||
|
||||
async function writeStorageState(env, storagePath) {
|
||||
const { chromium } = require('@playwright/test');
|
||||
const user = getUser(env);
|
||||
const timeout = Number(env.E2E_AUTH_TIMEOUT || 15000);
|
||||
const conversationURL = appURL('c/new');
|
||||
const loginURL = appURL('login');
|
||||
const browser = await chromium.launch({ headless: true });
|
||||
|
||||
try {
|
||||
const page = await browser.newPage();
|
||||
await page.context().addInitScript(() => {
|
||||
localStorage.setItem('navVisible', 'true');
|
||||
});
|
||||
|
||||
await page.goto(baseURL, { timeout });
|
||||
try {
|
||||
await register(page, user, timeout);
|
||||
await page.waitForURL(conversationURL, { timeout });
|
||||
} catch {
|
||||
await page.goto(loginURL, { timeout });
|
||||
await login(page, user);
|
||||
await page.waitForURL(conversationURL, { timeout });
|
||||
}
|
||||
|
||||
fs.mkdirSync(path.dirname(storagePath), { recursive: true });
|
||||
await page.context().storageState({ path: storagePath });
|
||||
console.log(`[record] Saved authenticated storage state to ${storagePath}`);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
function runCodegen(options, env) {
|
||||
const args = [
|
||||
'playwright',
|
||||
'codegen',
|
||||
'--target=playwright-test',
|
||||
'--test-id-attribute=data-testid',
|
||||
'--load-storage',
|
||||
options.storage,
|
||||
];
|
||||
|
||||
if (options.saveOutput) {
|
||||
fs.mkdirSync(path.dirname(options.output), { recursive: true });
|
||||
args.push('--output', options.output);
|
||||
}
|
||||
|
||||
args.push(options.url);
|
||||
console.log(`[record] Opening Playwright codegen at ${options.url}`);
|
||||
if (options.saveOutput) {
|
||||
console.log(`[record] Raw recording will be written to ${options.output}`);
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(npxBin, args, {
|
||||
cwd: rootPath,
|
||||
env,
|
||||
stdio: 'inherit',
|
||||
});
|
||||
child.on('exit', (code) => {
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
} else {
|
||||
reject(new Error(`Playwright codegen exited with code ${code}`));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!['mock', 'local'].includes(options.profile)) {
|
||||
throw new Error('--profile must be "mock" or "local"');
|
||||
}
|
||||
|
||||
const env = getEnv(options.profile);
|
||||
const children = [];
|
||||
|
||||
try {
|
||||
if (options.profile === 'mock') {
|
||||
writeRuntimeMockConfig();
|
||||
|
||||
const mockURL = `http://127.0.0.1:${mockLlmPort}/health`;
|
||||
if (!(await waitForURL(mockURL, 1000))) {
|
||||
children.push(
|
||||
spawnProcess(
|
||||
'mock-llm',
|
||||
'node',
|
||||
[path.resolve(rootPath, 'e2e/setup/mock-llm-server.js')],
|
||||
env,
|
||||
),
|
||||
);
|
||||
if (!(await waitForURL(mockURL, 30000))) {
|
||||
throw new Error(`Mock LLM server did not become ready at ${mockURL}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (await waitForURL(baseURL, 1000)) {
|
||||
if (options.profile === 'mock') {
|
||||
console.warn('[record] Reusing an existing app server; make sure it uses e2e mock config.');
|
||||
}
|
||||
} else {
|
||||
children.push(
|
||||
spawnProcess('app', 'node', [path.resolve(rootPath, 'e2e/setup/start-server.js')], env),
|
||||
);
|
||||
if (!(await waitForURL(baseURL, 120000))) {
|
||||
throw new Error(`LibreChat server did not become ready at ${baseURL}`);
|
||||
}
|
||||
}
|
||||
|
||||
await writeStorageState(env, options.storage);
|
||||
if (options.authOnly) {
|
||||
return;
|
||||
}
|
||||
await runCodegen(options, env);
|
||||
} finally {
|
||||
for (const child of children.reverse()) {
|
||||
await stopProcess(child);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error('[record] Failed:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
20
e2e/setup/users.mock.ts
Normal file
20
e2e/setup/users.mock.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
import type { User } from '../types';
|
||||
import { getE2EUser } from './user';
|
||||
|
||||
const DEFAULT_SECONDARY_USER: User = {
|
||||
email: 'testuser-b@example.com',
|
||||
name: 'Test User B',
|
||||
password: 'securepassword456',
|
||||
};
|
||||
|
||||
export function getPrimaryE2EUser(): User {
|
||||
return getE2EUser();
|
||||
}
|
||||
|
||||
export function getSecondaryE2EUser(): User {
|
||||
return {
|
||||
email: process.env.E2E_USER_B_EMAIL ?? DEFAULT_SECONDARY_USER.email,
|
||||
name: process.env.E2E_USER_B_NAME ?? DEFAULT_SECONDARY_USER.name,
|
||||
password: process.env.E2E_USER_B_PASSWORD ?? DEFAULT_SECONDARY_USER.password,
|
||||
};
|
||||
}
|
||||
22
e2e/specs/mock/app-load.spec.ts
Normal file
22
e2e/specs/mock/app-load.spec.ts
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import { NEW_CHAT_PATH } from './helpers';
|
||||
|
||||
test.describe('app loads cleanly', () => {
|
||||
test('authenticated user lands on a rendered chat view without runtime errors', async ({
|
||||
page,
|
||||
}) => {
|
||||
const pageErrors: string[] = [];
|
||||
page.on('pageerror', (error) => pageErrors.push(error.message));
|
||||
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
|
||||
await expect(page).toHaveURL(/\/c\/new$/);
|
||||
await expect(page.getByRole('main')).toBeVisible();
|
||||
await expect(page.getByRole('textbox', { name: 'Message input' })).toBeVisible();
|
||||
await expect(page.getByRole('button', { name: 'Select a model' }).first()).toBeVisible();
|
||||
await expect(page.getByTestId('nav-user')).toBeVisible();
|
||||
|
||||
await expect(page.getByText(/something went wrong/i)).toHaveCount(0);
|
||||
expect(pageErrors, `Unexpected runtime errors: ${pageErrors.join(', ')}`).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
82
e2e/specs/mock/auth.spec.ts
Normal file
82
e2e/specs/mock/auth.spec.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import type { APIRequestContext } from '@playwright/test';
|
||||
import type { User } from '../../types';
|
||||
import { getSecondaryE2EUser } from '../../setup/users.mock';
|
||||
import cleanupUser from '../../setup/cleanupUser';
|
||||
import { NEW_CHAT_PATH } from './helpers';
|
||||
|
||||
async function getIsolatedStorageState(request: APIRequestContext, user: User) {
|
||||
await cleanupUser(user);
|
||||
|
||||
const registerResponse = await request.post('/api/auth/register', {
|
||||
data: {
|
||||
email: user.email,
|
||||
name: user.name,
|
||||
password: user.password,
|
||||
confirm_password: user.password,
|
||||
},
|
||||
});
|
||||
expect(registerResponse.ok()).toBeTruthy();
|
||||
|
||||
const loginResponse = await request.post('/api/auth/login', {
|
||||
data: {
|
||||
email: user.email,
|
||||
password: user.password,
|
||||
},
|
||||
});
|
||||
expect(loginResponse.ok()).toBeTruthy();
|
||||
|
||||
return request.storageState();
|
||||
}
|
||||
|
||||
test.describe('auth session', () => {
|
||||
test('session persists across a full page reload', async ({ page }) => {
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
await expect(page).not.toHaveURL(/\/login/);
|
||||
await expect(page.getByTestId('nav-user')).toBeVisible();
|
||||
|
||||
await page.reload({ timeout: 10000 });
|
||||
|
||||
await expect(page).not.toHaveURL(/\/login/);
|
||||
await expect(page.getByTestId('nav-user')).toBeVisible();
|
||||
await expect(page.getByRole('textbox', { name: 'Message input' })).toBeVisible();
|
||||
});
|
||||
|
||||
test('logout ends the session and protects authenticated routes', async ({
|
||||
request,
|
||||
browser,
|
||||
baseURL,
|
||||
}) => {
|
||||
test.setTimeout(90000);
|
||||
if (typeof baseURL !== 'string') {
|
||||
throw new Error('baseURL must be configured for mock auth tests');
|
||||
}
|
||||
|
||||
const user = getSecondaryE2EUser();
|
||||
const context = await browser.newContext({
|
||||
storageState: await getIsolatedStorageState(request, user),
|
||||
baseURL,
|
||||
});
|
||||
await context.addInitScript(() => {
|
||||
localStorage.setItem('navVisible', 'true');
|
||||
});
|
||||
const page = await context.newPage();
|
||||
|
||||
try {
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
await expect(page).not.toHaveURL(/\/login/);
|
||||
|
||||
await page.getByTestId('nav-user').click();
|
||||
await page.getByRole('menuitem', { name: 'Log out' }).click();
|
||||
|
||||
await page.waitForURL(/\/login/, { timeout: 10000 });
|
||||
await expect(page.getByLabel('Email')).toBeVisible();
|
||||
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
await expect(page).toHaveURL(/\/login/);
|
||||
} finally {
|
||||
await context.close().catch(() => undefined);
|
||||
await cleanupUser(user);
|
||||
}
|
||||
});
|
||||
});
|
||||
37
e2e/specs/mock/chat.spec.ts
Normal file
37
e2e/specs/mock/chat.spec.ts
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import {
|
||||
MOCK_ENDPOINTS,
|
||||
NEW_CHAT_PATH,
|
||||
mockReply,
|
||||
selectMockEndpoint,
|
||||
sendMessage,
|
||||
} from './helpers';
|
||||
|
||||
test.describe('core chat loop', () => {
|
||||
test('streams a response, saves the conversation, and persists across reload', async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(60000);
|
||||
const userMessage = 'ping from e2e';
|
||||
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
|
||||
|
||||
const response = await sendMessage(page, userMessage);
|
||||
expect(response.ok()).toBeTruthy();
|
||||
|
||||
await expect(page.getByText(userMessage)).toBeVisible();
|
||||
await expect(mockReply(page)).toBeVisible();
|
||||
|
||||
await expect(page).toHaveURL(/\/c\/[0-9a-fA-F-]{36}$/);
|
||||
const conversationUrl = page.url();
|
||||
|
||||
await expect(page.getByTestId('convo-item').first()).toBeVisible();
|
||||
|
||||
await page.reload({ timeout: 10000 });
|
||||
await expect(page).toHaveURL(conversationUrl);
|
||||
await expect(page.getByText(userMessage)).toBeVisible();
|
||||
await expect(mockReply(page)).toBeVisible();
|
||||
await expect(page.getByTestId('convo-item').first()).toBeVisible();
|
||||
});
|
||||
});
|
||||
47
e2e/specs/mock/helpers.ts
Normal file
47
e2e/specs/mock/helpers.ts
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
import { expect } from '@playwright/test';
|
||||
import type { Page, Response } from '@playwright/test';
|
||||
|
||||
/** Substring of the reply emitted by the mock LLM server. */
|
||||
export const MOCK_REPLY_TEXT = 'E2E mock reply';
|
||||
|
||||
/** Custom endpoints defined in e2e/config/librechat.e2e.yaml. */
|
||||
export const MOCK_ENDPOINTS = [
|
||||
{ label: 'Mock Provider A', model: 'mock-model-a' },
|
||||
{ label: 'Mock Provider B', model: 'mock-model-b' },
|
||||
] as const;
|
||||
|
||||
export type MockEndpoint = (typeof MOCK_ENDPOINTS)[number];
|
||||
|
||||
export const NEW_CHAT_PATH = '/c/new';
|
||||
|
||||
export function isAgentsStream(response: Response) {
|
||||
return response.url().includes('/api/agents') && response.status() === 200;
|
||||
}
|
||||
|
||||
const modelSelectorTrigger = (page: Page) =>
|
||||
page.getByRole('button', { name: 'Select a model' }).first();
|
||||
|
||||
/** Open the model selector, choose an endpoint, then its model (committed on the model click). */
|
||||
export async function selectMockEndpoint(page: Page, endpoint: MockEndpoint) {
|
||||
await modelSelectorTrigger(page).click();
|
||||
await page.getByRole('option', { name: endpoint.label }).click();
|
||||
await page.getByRole('option', { name: endpoint.model, exact: true }).click();
|
||||
await expect(modelSelectorTrigger(page)).not.toHaveText('Select a model');
|
||||
}
|
||||
|
||||
/** The mock reply as rendered in the conversation, scoped to the messages view. */
|
||||
export function mockReply(page: Page) {
|
||||
return page.getByTestId('messages-view').getByText(new RegExp(MOCK_REPLY_TEXT, 'i'));
|
||||
}
|
||||
|
||||
/** Type a message, send it, and wait for the streamed `/api/agents` response. */
|
||||
export async function sendMessage(page: Page, text: string): Promise<Response> {
|
||||
const input = page.getByRole('textbox', { name: 'Message input' });
|
||||
await input.click();
|
||||
await input.fill(text);
|
||||
const [response] = await Promise.all([
|
||||
page.waitForResponse(isAgentsStream, { timeout: 30000 }),
|
||||
input.press('Enter'),
|
||||
]);
|
||||
return response;
|
||||
}
|
||||
97
e2e/specs/mock/isolation.spec.ts
Normal file
97
e2e/specs/mock/isolation.spec.ts
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import type { Browser, Page } from '@playwright/test';
|
||||
import type { User } from '../../types';
|
||||
import { MOCK_ENDPOINTS, NEW_CHAT_PATH, selectMockEndpoint, sendMessage } from './helpers';
|
||||
import { getSecondaryE2EUser } from '../../setup/users.mock';
|
||||
import cleanupUser from '../../setup/cleanupUser';
|
||||
|
||||
const A_PRIVATE_MARKER = 'A-private-conversation-marker';
|
||||
|
||||
async function register(page: Page, user: User) {
|
||||
await page.getByRole('link', { name: 'Sign up' }).click();
|
||||
await page.getByLabel('Full name').fill(user.name);
|
||||
await page.getByLabel('Email').fill(user.email);
|
||||
await page.getByTestId('password').fill(user.password);
|
||||
await page.getByTestId('confirm_password').fill(user.password);
|
||||
await page.getByLabel('Submit registration').click();
|
||||
}
|
||||
|
||||
async function registrationErrorIsVisible(page: Page) {
|
||||
return page
|
||||
.getByTestId('registration-error')
|
||||
.isVisible({ timeout: 500 })
|
||||
.catch(() => false);
|
||||
}
|
||||
|
||||
async function registerSecondaryUser(page: Page, user: User) {
|
||||
await page.goto('/', { timeout: 10000 });
|
||||
await page.waitForURL(/\/login/, { timeout: 10000 });
|
||||
await register(page, user);
|
||||
|
||||
try {
|
||||
await page.waitForURL(/\/c\/new/, { timeout: 10000 });
|
||||
} catch (error) {
|
||||
if (!(await registrationErrorIsVisible(page))) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
await cleanupUser(user);
|
||||
await page.goto('/', { timeout: 10000 });
|
||||
await page.waitForURL(/\/login/, { timeout: 10000 });
|
||||
await register(page, user);
|
||||
await page.waitForURL(/\/c\/new/, { timeout: 10000 });
|
||||
}
|
||||
}
|
||||
|
||||
/** Register the secondary user in a throwaway context, then log in within `page`. */
|
||||
async function ensureSecondaryUser(browser: Browser, page: Page, user: User, baseURL: string) {
|
||||
const setupContext = await browser.newContext({ storageState: undefined, baseURL });
|
||||
const setupPage = await setupContext.newPage();
|
||||
try {
|
||||
await registerSecondaryUser(setupPage, user);
|
||||
} finally {
|
||||
await setupContext.close();
|
||||
}
|
||||
|
||||
await page.goto('/login', { timeout: 10000 });
|
||||
await page.getByLabel('Email').fill(user.email);
|
||||
await page.getByLabel('Password').fill(user.password);
|
||||
await page.getByTestId('login-button').click();
|
||||
await page.waitForURL(/\/c\/new/, { timeout: 10000 });
|
||||
}
|
||||
|
||||
test.describe('user isolation', () => {
|
||||
test('user B cannot see user A conversations', async ({ page, browser, baseURL }) => {
|
||||
test.setTimeout(90000);
|
||||
if (typeof baseURL !== 'string') {
|
||||
throw new Error('baseURL must be configured for mock isolation tests');
|
||||
}
|
||||
|
||||
// User A (authenticated via storageState) creates a private conversation.
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
|
||||
await sendMessage(page, A_PRIVATE_MARKER);
|
||||
await expect(page.getByText(A_PRIVATE_MARKER)).toBeVisible();
|
||||
await expect(page).toHaveURL(/\/c\/[0-9a-fA-F-]{36}$/);
|
||||
const conversationAUrl = page.url();
|
||||
|
||||
// User B in a fresh, unauthenticated context.
|
||||
const contextB = await browser.newContext({ storageState: undefined, baseURL });
|
||||
const pageB = await contextB.newPage();
|
||||
try {
|
||||
await ensureSecondaryUser(browser, pageB, getSecondaryE2EUser(), baseURL);
|
||||
|
||||
// (a) Sidebar list does not expose A's conversation.
|
||||
await pageB.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
await expect(pageB.getByRole('textbox', { name: 'Message input' })).toBeVisible();
|
||||
await expect(pageB.getByText(A_PRIVATE_MARKER)).toHaveCount(0);
|
||||
|
||||
// (b) Direct navigation to A's conversation does not reveal its content.
|
||||
await pageB.goto(conversationAUrl, { timeout: 10000 });
|
||||
await expect(pageB.getByRole('textbox', { name: 'Message input' })).toBeVisible();
|
||||
await expect(pageB.getByText(A_PRIVATE_MARKER)).toHaveCount(0);
|
||||
} finally {
|
||||
await contextB.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
23
e2e/specs/mock/model-switching.spec.ts
Normal file
23
e2e/specs/mock/model-switching.spec.ts
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import {
|
||||
MOCK_ENDPOINTS,
|
||||
NEW_CHAT_PATH,
|
||||
mockReply,
|
||||
selectMockEndpoint,
|
||||
sendMessage,
|
||||
} from './helpers';
|
||||
|
||||
test.describe('endpoint switching', () => {
|
||||
for (const endpoint of MOCK_ENDPOINTS) {
|
||||
test(`"${endpoint.label}" returns a streamed response`, async ({ page }) => {
|
||||
test.setTimeout(60000);
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
|
||||
await selectMockEndpoint(page, endpoint);
|
||||
|
||||
const response = await sendMessage(page, `hello ${endpoint.model}`);
|
||||
expect(response.ok()).toBeTruthy();
|
||||
await expect(mockReply(page)).toBeVisible();
|
||||
});
|
||||
}
|
||||
});
|
||||
|
|
@ -58,9 +58,13 @@
|
|||
"e2e:headed": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.local.ts --headed",
|
||||
"e2e:a11y": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.a11y.ts --headed",
|
||||
"e2e:ci": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.ts",
|
||||
"e2e:mock": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.mock.ts",
|
||||
"e2e:mock:ci": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.mock.ts",
|
||||
"e2e:debug": "npm run e2e:prepare && cross-env PWDEBUG=1 playwright test --config=e2e/playwright.config.local.ts",
|
||||
"e2e:codegen": "npx playwright codegen --load-storage=e2e/storageState.json http://localhost:3080/c/new",
|
||||
"e2e:login": "npx playwright codegen --save-storage=e2e/auth.json http://localhost:3080/login",
|
||||
"e2e:record": "npm run e2e:prepare && cross-env E2E_BASE_URL=http://localhost:3333 node e2e/setup/record.js",
|
||||
"e2e:record:local": "npm run e2e:prepare && node e2e/setup/record.js --profile=local",
|
||||
"e2e:codegen": "npx playwright codegen --target=playwright-test --test-id-attribute=data-testid --load-storage=e2e/storageState.json http://localhost:3080/c/new",
|
||||
"e2e:login": "npx playwright codegen --save-storage=e2e/storageState.json http://localhost:3080/login",
|
||||
"e2e:github": "act -W .github/workflows/playwright.yml --secret-file my.secrets",
|
||||
"test:client": "cd client && npm run test:ci",
|
||||
"test:api": "cd api && npm run test:ci",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue