🧪 test: Run mock E2E against Redis in shards (#14551)

* 🧪 test: Run mock E2E against Redis in shards

* 🧪 test: Isolate local Redis E2E data

.github/workflows/playwright-mock.yml

@@ -26,7 +26,8 @@ env:
   PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1'
 
 jobs:
-  e2e:
+  e2e_shards:
+    name: e2e (${{ matrix.stream_store }}, shard ${{ matrix.shard }}/4)
     if: >-
       github.event_name == 'workflow_dispatch' ||
       (github.event_name == 'pull_request' &&
@@ -34,8 +35,25 @@ jobs:
       contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association))
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        stream_store: [memory, redis]
+        shard: [1, 2, 3, 4]
+    services:
+      redis:
+        image: redis:7-alpine
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 5s
+          --health-timeout 3s
+          --health-retries 10
     env:
       E2E_CHROMIUM_CHANNEL: chrome
+      E2E_STREAM_STORE: ${{ matrix.stream_store }}
+      REDIS_URI: redis://127.0.0.1:6379
     steps:
       - uses: actions/checkout@v4
 
@@ -124,7 +142,7 @@ jobs:
           npx playwright install-deps chrome
 
       - name: Run mock-LLM Tier-1 e2e
-        run: npx playwright test --config=e2e/playwright.config.mock.ts
+        run: npx playwright test --config=e2e/playwright.config.mock.ts --shard=${{ matrix.shard }}/4
         env:
           CI: 'true'
 
@@ -132,7 +150,7 @@ jobs:
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: playwright-report
+          name: playwright-report-${{ matrix.stream_store }}-${{ matrix.shard }}
           path: e2e/playwright-report/**
           retention-days: 7
           if-no-files-found: ignore
@@ -141,7 +159,22 @@ jobs:
         if: failure()
         uses: actions/upload-artifact@v4
         with:
-          name: playwright-test-results
+          name: playwright-test-results-${{ matrix.stream_store }}-${{ matrix.shard }}
           path: e2e/specs/.test-results/**
           retention-days: 7
           if-no-files-found: ignore
+
+  e2e:
+    name: e2e
+    if: >-
+      always() &&
+      (github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'pull_request' &&
+      github.event.pull_request != null &&
+      contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association)))
+    needs: e2e_shards
+    runs-on: ubuntu-latest
+    steps:
+      - name: Verify every Playwright shard passed
+        if: needs.e2e_shards.result != 'success'
+        run: exit 1

e2e/README.md

@@ -2,6 +2,22 @@
 
 The mock e2e profile is the safest default for generated tests. It starts LibreChat with `e2e/config/librechat.e2e.yaml`, injects an in-process fake LLM (via `LIBRECHAT_TEST_RUN_HOOK`), creates an authenticated e2e user, and avoids real provider credentials.
 
+## Stream Stores and Shards
+
+The mock profile uses the in-memory generation stream store by default. To exercise the same browser scenarios through a real Redis job store and pub/sub transport, start Redis on port 6379 and run:
+
+```sh
+npm run e2e:mock:redis
+```
+
+Memory mode explicitly disables Redis. Redis mode defaults to database 15 with a `LibreChatE2E` key prefix, and fails closed: the test server pings Redis and verifies that the generation job manager did not silently fall back to memory. Override `REDIS_URI` or `E2E_REDIS_KEY_PREFIX` when needed.
+
+CI runs the complete mock suite in both stream modes. Each mode is split across four Playwright shards, while each shard keeps one worker so tests do not contend for the shard's authenticated user and database:
+
+```sh
+npx playwright test --config=e2e/playwright.config.mock.ts --shard=1/4
+```
+
 ## Recording Tests
 
 Use Playwright codegen when you want to turn an exploratory browser session into a draft test:

e2e/playwright.config.mock.ts

@@ -118,7 +118,7 @@ export default defineConfig({
   globalTeardown: require.resolve('./setup/global-teardown.mock'),
   testDir: 'specs/mock/',
   outputDir: 'specs/.test-results',
-  fullyParallel: false,
+  fullyParallel: true,
   forbidOnly: !!process.env.CI,
   retries: process.env.CI ? 2 : 0,
   workers: 1,

e2e/setup/env.ts

@@ -8,6 +8,8 @@ const GENERATED_CREDS_KEY = crypto.randomBytes(32).toString('hex');
 const GENERATED_CREDS_IV = crypto.randomBytes(16).toString('hex');
 const GENERATED_JWT_SECRET = crypto.randomBytes(32).toString('hex');
 const GENERATED_JWT_REFRESH_SECRET = crypto.randomBytes(32).toString('hex');
+const DEFAULT_REDIS_URI = 'redis://127.0.0.1:6379/15';
+const DEFAULT_REDIS_KEY_PREFIX = 'LibreChatE2E';
 const PASSTHROUGH_ENV_KEYS = [
   'APPDATA',
   'CI',
@@ -71,6 +73,30 @@ function getPassthroughEnv(): Record<string, string> {
   return env;
 }
 
+function getStreamStoreEnv(): Record<string, string> {
+  const streamStore = process.env.E2E_STREAM_STORE ?? 'memory';
+  if (streamStore === 'memory') {
+    return {
+      E2E_REQUIRE_REDIS_STREAMS: 'false',
+      USE_REDIS: 'false',
+      USE_REDIS_STREAMS: 'false',
+      REDIS_KEY_PREFIX: '',
+      REDIS_KEY_PREFIX_VAR: '',
+    };
+  }
+  if (streamStore === 'redis') {
+    return {
+      E2E_REQUIRE_REDIS_STREAMS: 'true',
+      USE_REDIS: 'true',
+      USE_REDIS_STREAMS: 'true',
+      REDIS_URI: process.env.REDIS_URI ?? DEFAULT_REDIS_URI,
+      REDIS_KEY_PREFIX: process.env.E2E_REDIS_KEY_PREFIX ?? DEFAULT_REDIS_KEY_PREFIX,
+      REDIS_KEY_PREFIX_VAR: '',
+    };
+  }
+  throw new Error(`Unsupported E2E_STREAM_STORE "${streamStore}"`);
+}
+
 export function getBaseE2EEnv(): Record<string, string> {
   const baseURL = getE2EBaseURL();
   const { host, port } = getE2EServerAddress(baseURL);
@@ -96,6 +122,7 @@ export function getBaseE2EEnv(): Record<string, string> {
     SESSION_EXPIRY: process.env.SESSION_EXPIRY ?? '3600000',
     ALLOW_REGISTRATION: 'true',
     REFRESH_TOKEN_EXPIRY: process.env.REFRESH_TOKEN_EXPIRY ?? '3600000',
+    ...getStreamStoreEnv(),
   };
 }
 

e2e/setup/fake-model.js

@@ -26,6 +26,7 @@ const ASSERT_AGENT_CONTEXT_MARKER = 'E2E_ASSERT_AGENT_CONTEXT:';
 const ASSERT_QUOTE_MARKER = 'E2E_ASSERT_QUOTE:';
 const REPLY_MARKER = 'E2E_REPLY:';
 const COUNTED_REPLY_MARKER = 'E2E_COUNTED_REPLY:';
+const ORDERED_REPLY_MARKER = 'E2E_ORDERED_REPLY:';
 const SLOW_REPLY_MARKER = 'E2E_SLOW_REPLY:';
 const SLOW_COUNTED_REPLY_MARKER = 'E2E_SLOW_COUNTED_REPLY:';
 const STEER_TOOL_REPLY_MARKER = 'E2E_STEER_TOOL_REPLY:';
@@ -57,6 +58,8 @@ const STEER_LATE_FINAL_TEXT = 'E2E steer late reply done';
 const ACTIVITY_FINAL_TEXT = 'E2E activity reply done';
 const STEER_TOOL_NAME_PREFIX = 'remember_fact';
 const SLOW_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_SLOW_CHUNK_DELAY_MS) || 35;
+const ORDERED_CHUNK_DELAY_MS = 2;
+const ORDERED_REPLY_PIECES = 64;
 const SLOW_REPLY_CHUNKS = 160;
 const RESUME_ICON_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_RESUME_ICON_CHUNK_DELAY_MS) || 60;
 const RESUME_ICON_REPLY_CHUNKS = 240;
@@ -402,6 +405,18 @@ function replyResponses(text) {
     };
   }
 
+  const orderedName = getMarkerValue(text, ORDERED_REPLY_MARKER);
+  if (orderedName) {
+    const pieces = Array.from(
+      { length: ORDERED_REPLY_PIECES },
+      (_, index) => `piece-${String(index).padStart(3, '0')}`,
+    ).join(' ');
+    return {
+      responses: [`E2E ordered reply ${orderedName} ${pieces}`],
+      sleep: ORDERED_CHUNK_DELAY_MS,
+    };
+  }
+
   const slowName = getMarkerValue(text, SLOW_REPLY_MARKER);
   if (slowName) {
     const chunks = Array.from(

e2e/setup/start-server.js

@@ -5,6 +5,8 @@ require('dotenv').config();
 
 const DEFAULT_MONGO_URI = 'mongodb://127.0.0.1:27017/LibreChat-e2e';
 const DEFAULT_RUNTIME_ENV_PATH = path.resolve(__dirname, '../specs/.test-results/runtime-env.json');
+const REDIS_STREAM_STARTUP_TIMEOUT_MS = 15_000;
+const REDIS_PING_TIMEOUT_MS = 10_000;
 let mongoServer;
 
 function decodeMongoValue(value) {
@@ -156,6 +158,46 @@ async function shutdown() {
   }
 }
 
+async function requireRedisStreams() {
+  if (process.env.E2E_REQUIRE_REDIS_STREAMS !== 'true') {
+    return;
+  }
+  const { ioredisClient } = require('@librechat/api');
+  if (!ioredisClient) {
+    throw new Error('[e2e] Redis stream mode was required but no Redis client was configured');
+  }
+  let timeout;
+  try {
+    await Promise.race([
+      ioredisClient.ping(),
+      new Promise((_, reject) => {
+        timeout = setTimeout(
+          () => reject(new Error(`[e2e] Redis did not respond within ${REDIS_PING_TIMEOUT_MS}ms`)),
+          REDIS_PING_TIMEOUT_MS,
+        );
+      }),
+    ]);
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+async function verifyRedisStreams() {
+  if (process.env.E2E_REQUIRE_REDIS_STREAMS !== 'true') {
+    return;
+  }
+  const { GenerationJobManager } = require('@librechat/api');
+  const deadline = Date.now() + REDIS_STREAM_STARTUP_TIMEOUT_MS;
+  while (Date.now() < deadline) {
+    if (GenerationJobManager.isRedis) {
+      console.log('[e2e] Verified Redis-backed generation streams');
+      return;
+    }
+    await new Promise((resolve) => setTimeout(resolve, 50));
+  }
+  throw new Error('[e2e] Redis stream mode was required but the server fell back to memory');
+}
+
 process.once('SIGINT', async () => {
   await shutdown();
   process.exit(130);
@@ -168,8 +210,10 @@ process.once('SIGTERM', async () => {
 
 function startServer() {
   return maybeStartMemoryMongo()
-    .then(() => {
+    .then(requireRedisStreams)
+    .then(async () => {
       require(path.resolve(__dirname, '../../api/server/index.js'));
+      await verifyRedisStreams();
     })
     .catch((error) => {
       console.error('[e2e] Failed to start test server:', error);

e2e/specs/mock/streaming.spec.ts

@@ -0,0 +1,41 @@
+import { expect, test } from '@playwright/test';
+import {
+  MOCK_ENDPOINTS,
+  NEW_CHAT_PATH,
+  messagesView,
+  selectMockEndpoint,
+  sendMessage,
+} from './helpers';
+
+const ORDERED_PIECE_COUNT = 64;
+
+const orderedPieces = () =>
+  Array.from(
+    { length: ORDERED_PIECE_COUNT },
+    (_, index) => `piece-${String(index).padStart(3, '0')}`,
+  );
+
+test.describe('stream transport fidelity', () => {
+  test('renders and persists every LLM chunk exactly once and in order', async ({ page }) => {
+    const label = `ordered-${Date.now()}`;
+    const expected = `E2E ordered reply ${label} ${orderedPieces().join(' ')}`;
+
+    await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
+    await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
+
+    const response = await sendMessage(page, `E2E_ORDERED_REPLY:${label}`);
+    expect(response.ok()).toBeTruthy();
+
+    const assistantContent = messagesView(page)
+      .locator('.message-render')
+      .last()
+      .locator('.message-content');
+    await expect(assistantContent).toContainText('piece-010', { timeout: 30000 });
+    await expect(assistantContent).toHaveText(expected, { timeout: 30000 });
+
+    await page.reload({ timeout: 10000 });
+    await expect(
+      messagesView(page).locator('.message-render').last().locator('.message-content'),
+    ).toHaveText(expected, { timeout: 30000 });
+  });
+});

package.json

@@ -59,6 +59,7 @@
     "e2e:a11y": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.a11y.ts --headed",
     "e2e:ci": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.ts",
     "e2e:mock": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.mock.ts",
+    "e2e:mock:redis": "npm run e2e:prepare && cross-env E2E_STREAM_STORE=redis playwright test --config=e2e/playwright.config.mock.ts",
     "e2e:benchmark:agents": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.benchmark.ts agent-startup.latency.spec.ts",
     "e2e:mock:enforce": "npm run e2e:prepare && cross-env E2E_MODEL_SPECS_ENFORCE=true playwright test --config=e2e/playwright.config.mock.ts enforced-model-specs.spec.ts",
     "e2e:mock:ci": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.mock.ts",