From 52b2ebf9485953c70acb06f04d9ac6e9c6177825 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 31 Jul 2026 12:10:43 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=AA=20test:=20Run=20mock=20E2E=20again?= =?UTF-8?q?st=20Redis=20in=20shards=20(#14551)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🧪 test: Run mock E2E against Redis in shards * 🧪 test: Isolate local Redis E2E data --- .github/workflows/playwright-mock.yml | 41 +++++++++++++++++++++--- e2e/README.md | 16 ++++++++++ e2e/playwright.config.mock.ts | 2 +- e2e/setup/env.ts | 27 ++++++++++++++++ e2e/setup/fake-model.js | 15 +++++++++ e2e/setup/start-server.js | 46 ++++++++++++++++++++++++++- e2e/specs/mock/streaming.spec.ts | 41 ++++++++++++++++++++++++ package.json | 1 + 8 files changed, 183 insertions(+), 6 deletions(-) create mode 100644 e2e/specs/mock/streaming.spec.ts diff --git a/.github/workflows/playwright-mock.yml b/.github/workflows/playwright-mock.yml index 682497fb97..e9b524642d 100644 --- a/.github/workflows/playwright-mock.yml +++ b/.github/workflows/playwright-mock.yml @@ -26,7 +26,8 @@ env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1' jobs: - e2e: + e2e_shards: + name: e2e (${{ matrix.stream_store }}, shard ${{ matrix.shard }}/4) if: >- github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && @@ -34,8 +35,25 @@ jobs: contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association)) runs-on: ubuntu-latest timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + stream_store: [memory, redis] + shard: [1, 2, 3, 4] + services: + redis: + image: redis:7-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 3s + --health-retries 10 env: E2E_CHROMIUM_CHANNEL: chrome + E2E_STREAM_STORE: ${{ matrix.stream_store }} + REDIS_URI: redis://127.0.0.1:6379 steps: - uses: actions/checkout@v4 @@ -124,7 +142,7 @@ jobs: npx playwright install-deps chrome - name: Run mock-LLM Tier-1 e2e - run: npx playwright test --config=e2e/playwright.config.mock.ts + run: npx playwright test --config=e2e/playwright.config.mock.ts --shard=${{ matrix.shard }}/4 env: CI: 'true' @@ -132,7 +150,7 @@ jobs: if: ${{ !cancelled() }} uses: actions/upload-artifact@v4 with: - name: playwright-report + name: playwright-report-${{ matrix.stream_store }}-${{ matrix.shard }} path: e2e/playwright-report/** retention-days: 7 if-no-files-found: ignore @@ -141,7 +159,22 @@ jobs: if: failure() uses: actions/upload-artifact@v4 with: - name: playwright-test-results + name: playwright-test-results-${{ matrix.stream_store }}-${{ matrix.shard }} path: e2e/specs/.test-results/** retention-days: 7 if-no-files-found: ignore + + e2e: + name: e2e + if: >- + always() && + (github.event_name == 'workflow_dispatch' || + (github.event_name == 'pull_request' && + github.event.pull_request != null && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association))) + needs: e2e_shards + runs-on: ubuntu-latest + steps: + - name: Verify every Playwright shard passed + if: needs.e2e_shards.result != 'success' + run: exit 1 diff --git a/e2e/README.md b/e2e/README.md index 86ec912447..d0dae2c45a 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -2,6 +2,22 @@ The mock e2e profile is the safest default for generated tests. It starts LibreChat with `e2e/config/librechat.e2e.yaml`, injects an in-process fake LLM (via `LIBRECHAT_TEST_RUN_HOOK`), creates an authenticated e2e user, and avoids real provider credentials. +## Stream Stores and Shards + +The mock profile uses the in-memory generation stream store by default. To exercise the same browser scenarios through a real Redis job store and pub/sub transport, start Redis on port 6379 and run: + +```sh +npm run e2e:mock:redis +``` + +Memory mode explicitly disables Redis. Redis mode defaults to database 15 with a `LibreChatE2E` key prefix, and fails closed: the test server pings Redis and verifies that the generation job manager did not silently fall back to memory. Override `REDIS_URI` or `E2E_REDIS_KEY_PREFIX` when needed. + +CI runs the complete mock suite in both stream modes. Each mode is split across four Playwright shards, while each shard keeps one worker so tests do not contend for the shard's authenticated user and database: + +```sh +npx playwright test --config=e2e/playwright.config.mock.ts --shard=1/4 +``` + ## Recording Tests Use Playwright codegen when you want to turn an exploratory browser session into a draft test: diff --git a/e2e/playwright.config.mock.ts b/e2e/playwright.config.mock.ts index 758f1e6726..9d40af1eed 100644 --- a/e2e/playwright.config.mock.ts +++ b/e2e/playwright.config.mock.ts @@ -118,7 +118,7 @@ export default defineConfig({ globalTeardown: require.resolve('./setup/global-teardown.mock'), testDir: 'specs/mock/', outputDir: 'specs/.test-results', - fullyParallel: false, + fullyParallel: true, forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 0, workers: 1, diff --git a/e2e/setup/env.ts b/e2e/setup/env.ts index 9a4b184a7a..da4fb3fb52 100644 --- a/e2e/setup/env.ts +++ b/e2e/setup/env.ts @@ -8,6 +8,8 @@ const GENERATED_CREDS_KEY = crypto.randomBytes(32).toString('hex'); const GENERATED_CREDS_IV = crypto.randomBytes(16).toString('hex'); const GENERATED_JWT_SECRET = crypto.randomBytes(32).toString('hex'); const GENERATED_JWT_REFRESH_SECRET = crypto.randomBytes(32).toString('hex'); +const DEFAULT_REDIS_URI = 'redis://127.0.0.1:6379/15'; +const DEFAULT_REDIS_KEY_PREFIX = 'LibreChatE2E'; const PASSTHROUGH_ENV_KEYS = [ 'APPDATA', 'CI', @@ -71,6 +73,30 @@ function getPassthroughEnv(): Record { return env; } +function getStreamStoreEnv(): Record { + const streamStore = process.env.E2E_STREAM_STORE ?? 'memory'; + if (streamStore === 'memory') { + return { + E2E_REQUIRE_REDIS_STREAMS: 'false', + USE_REDIS: 'false', + USE_REDIS_STREAMS: 'false', + REDIS_KEY_PREFIX: '', + REDIS_KEY_PREFIX_VAR: '', + }; + } + if (streamStore === 'redis') { + return { + E2E_REQUIRE_REDIS_STREAMS: 'true', + USE_REDIS: 'true', + USE_REDIS_STREAMS: 'true', + REDIS_URI: process.env.REDIS_URI ?? DEFAULT_REDIS_URI, + REDIS_KEY_PREFIX: process.env.E2E_REDIS_KEY_PREFIX ?? DEFAULT_REDIS_KEY_PREFIX, + REDIS_KEY_PREFIX_VAR: '', + }; + } + throw new Error(`Unsupported E2E_STREAM_STORE "${streamStore}"`); +} + export function getBaseE2EEnv(): Record { const baseURL = getE2EBaseURL(); const { host, port } = getE2EServerAddress(baseURL); @@ -96,6 +122,7 @@ export function getBaseE2EEnv(): Record { SESSION_EXPIRY: process.env.SESSION_EXPIRY ?? '3600000', ALLOW_REGISTRATION: 'true', REFRESH_TOKEN_EXPIRY: process.env.REFRESH_TOKEN_EXPIRY ?? '3600000', + ...getStreamStoreEnv(), }; } diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.js index b7e5993c63..d0cb5890f7 100644 --- a/e2e/setup/fake-model.js +++ b/e2e/setup/fake-model.js @@ -26,6 +26,7 @@ const ASSERT_AGENT_CONTEXT_MARKER = 'E2E_ASSERT_AGENT_CONTEXT:'; const ASSERT_QUOTE_MARKER = 'E2E_ASSERT_QUOTE:'; const REPLY_MARKER = 'E2E_REPLY:'; const COUNTED_REPLY_MARKER = 'E2E_COUNTED_REPLY:'; +const ORDERED_REPLY_MARKER = 'E2E_ORDERED_REPLY:'; const SLOW_REPLY_MARKER = 'E2E_SLOW_REPLY:'; const SLOW_COUNTED_REPLY_MARKER = 'E2E_SLOW_COUNTED_REPLY:'; const STEER_TOOL_REPLY_MARKER = 'E2E_STEER_TOOL_REPLY:'; @@ -57,6 +58,8 @@ const STEER_LATE_FINAL_TEXT = 'E2E steer late reply done'; const ACTIVITY_FINAL_TEXT = 'E2E activity reply done'; const STEER_TOOL_NAME_PREFIX = 'remember_fact'; const SLOW_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_SLOW_CHUNK_DELAY_MS) || 35; +const ORDERED_CHUNK_DELAY_MS = 2; +const ORDERED_REPLY_PIECES = 64; const SLOW_REPLY_CHUNKS = 160; const RESUME_ICON_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_RESUME_ICON_CHUNK_DELAY_MS) || 60; const RESUME_ICON_REPLY_CHUNKS = 240; @@ -402,6 +405,18 @@ function replyResponses(text) { }; } + const orderedName = getMarkerValue(text, ORDERED_REPLY_MARKER); + if (orderedName) { + const pieces = Array.from( + { length: ORDERED_REPLY_PIECES }, + (_, index) => `piece-${String(index).padStart(3, '0')}`, + ).join(' '); + return { + responses: [`E2E ordered reply ${orderedName} ${pieces}`], + sleep: ORDERED_CHUNK_DELAY_MS, + }; + } + const slowName = getMarkerValue(text, SLOW_REPLY_MARKER); if (slowName) { const chunks = Array.from( diff --git a/e2e/setup/start-server.js b/e2e/setup/start-server.js index b81f01f35a..cb4949a461 100644 --- a/e2e/setup/start-server.js +++ b/e2e/setup/start-server.js @@ -5,6 +5,8 @@ require('dotenv').config(); const DEFAULT_MONGO_URI = 'mongodb://127.0.0.1:27017/LibreChat-e2e'; const DEFAULT_RUNTIME_ENV_PATH = path.resolve(__dirname, '../specs/.test-results/runtime-env.json'); +const REDIS_STREAM_STARTUP_TIMEOUT_MS = 15_000; +const REDIS_PING_TIMEOUT_MS = 10_000; let mongoServer; function decodeMongoValue(value) { @@ -156,6 +158,46 @@ async function shutdown() { } } +async function requireRedisStreams() { + if (process.env.E2E_REQUIRE_REDIS_STREAMS !== 'true') { + return; + } + const { ioredisClient } = require('@librechat/api'); + if (!ioredisClient) { + throw new Error('[e2e] Redis stream mode was required but no Redis client was configured'); + } + let timeout; + try { + await Promise.race([ + ioredisClient.ping(), + new Promise((_, reject) => { + timeout = setTimeout( + () => reject(new Error(`[e2e] Redis did not respond within ${REDIS_PING_TIMEOUT_MS}ms`)), + REDIS_PING_TIMEOUT_MS, + ); + }), + ]); + } finally { + clearTimeout(timeout); + } +} + +async function verifyRedisStreams() { + if (process.env.E2E_REQUIRE_REDIS_STREAMS !== 'true') { + return; + } + const { GenerationJobManager } = require('@librechat/api'); + const deadline = Date.now() + REDIS_STREAM_STARTUP_TIMEOUT_MS; + while (Date.now() < deadline) { + if (GenerationJobManager.isRedis) { + console.log('[e2e] Verified Redis-backed generation streams'); + return; + } + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error('[e2e] Redis stream mode was required but the server fell back to memory'); +} + process.once('SIGINT', async () => { await shutdown(); process.exit(130); @@ -168,8 +210,10 @@ process.once('SIGTERM', async () => { function startServer() { return maybeStartMemoryMongo() - .then(() => { + .then(requireRedisStreams) + .then(async () => { require(path.resolve(__dirname, '../../api/server/index.js')); + await verifyRedisStreams(); }) .catch((error) => { console.error('[e2e] Failed to start test server:', error); diff --git a/e2e/specs/mock/streaming.spec.ts b/e2e/specs/mock/streaming.spec.ts new file mode 100644 index 0000000000..f8f6c7d90f --- /dev/null +++ b/e2e/specs/mock/streaming.spec.ts @@ -0,0 +1,41 @@ +import { expect, test } from '@playwright/test'; +import { + MOCK_ENDPOINTS, + NEW_CHAT_PATH, + messagesView, + selectMockEndpoint, + sendMessage, +} from './helpers'; + +const ORDERED_PIECE_COUNT = 64; + +const orderedPieces = () => + Array.from( + { length: ORDERED_PIECE_COUNT }, + (_, index) => `piece-${String(index).padStart(3, '0')}`, + ); + +test.describe('stream transport fidelity', () => { + test('renders and persists every LLM chunk exactly once and in order', async ({ page }) => { + const label = `ordered-${Date.now()}`; + const expected = `E2E ordered reply ${label} ${orderedPieces().join(' ')}`; + + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await selectMockEndpoint(page, MOCK_ENDPOINTS[0]); + + const response = await sendMessage(page, `E2E_ORDERED_REPLY:${label}`); + expect(response.ok()).toBeTruthy(); + + const assistantContent = messagesView(page) + .locator('.message-render') + .last() + .locator('.message-content'); + await expect(assistantContent).toContainText('piece-010', { timeout: 30000 }); + await expect(assistantContent).toHaveText(expected, { timeout: 30000 }); + + await page.reload({ timeout: 10000 }); + await expect( + messagesView(page).locator('.message-render').last().locator('.message-content'), + ).toHaveText(expected, { timeout: 30000 }); + }); +}); diff --git a/package.json b/package.json index b10616129d..8b836380e7 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "e2e:a11y": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.a11y.ts --headed", "e2e:ci": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.ts", "e2e:mock": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.mock.ts", + "e2e:mock:redis": "npm run e2e:prepare && cross-env E2E_STREAM_STORE=redis playwright test --config=e2e/playwright.config.mock.ts", "e2e:benchmark:agents": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.benchmark.ts agent-startup.latency.spec.ts", "e2e:mock:enforce": "npm run e2e:prepare && cross-env E2E_MODEL_SPECS_ENFORCE=true playwright test --config=e2e/playwright.config.mock.ts enforced-model-specs.spec.ts", "e2e:mock:ci": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.mock.ts",