From ee21066590c846007eae1692a6594477a5c66203 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 14 Aug 2026 12:54:30 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=8F=8E=EF=B8=8F=20ci:=20Focus=20Redis=20E?= =?UTF-8?q?2E=20Coverage=20(#14842)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/playwright-mock.yml | 35 +++++++++++++++++++-------- e2e/README.md | 11 +++++++-- e2e/playwright.config.redis.ts | 18 ++++++++++++++ package.json | 1 + 4 files changed, 53 insertions(+), 12 deletions(-) create mode 100644 e2e/playwright.config.redis.ts diff --git a/.github/workflows/playwright-mock.yml b/.github/workflows/playwright-mock.yml index 1b677dcc76..d764961d0d 100644 --- a/.github/workflows/playwright-mock.yml +++ b/.github/workflows/playwright-mock.yml @@ -7,6 +7,8 @@ on: - '!**.md' - '!.github/workflows/**' - '.github/workflows/playwright-mock.yml' + schedule: + - cron: '0 5 * * *' workflow_dispatch: inputs: reason: @@ -27,8 +29,9 @@ env: jobs: e2e_shards: - name: e2e (${{ matrix.stream_store }}, shard ${{ matrix.shard }}/2) + name: e2e (${{ matrix.name }}) if: >- + github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request != null && @@ -37,12 +40,15 @@ jobs: timeout-minutes: 30 strategy: fail-fast: false - matrix: - stream_store: [memory, redis] - shard: [1, 2] + matrix: >- + ${{ + github.event_name == 'pull_request' && + fromJSON('{"include":[{"name":"memory, shard 1/3","stream_store":"memory","redis_image":"","suite":"full","shard":"1/3","artifact":"memory-1-of-3"},{"name":"memory, shard 2/3","stream_store":"memory","redis_image":"","suite":"full","shard":"2/3","artifact":"memory-2-of-3"},{"name":"memory, shard 3/3","stream_store":"memory","redis_image":"","suite":"full","shard":"3/3","artifact":"memory-3-of-3"},{"name":"redis transport","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"","artifact":"redis-transport"}]}') || + fromJSON('{"include":[{"name":"memory, shard 1/2","stream_store":"memory","redis_image":"","suite":"full","shard":"1/2","artifact":"memory-1-of-2"},{"name":"memory, shard 2/2","stream_store":"memory","redis_image":"","suite":"full","shard":"2/2","artifact":"memory-2-of-2"},{"name":"redis, shard 1/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"1/2","artifact":"redis-1-of-2"},{"name":"redis, shard 2/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"2/2","artifact":"redis-2-of-2"}]}') + }} services: redis: - image: redis:7-alpine + image: ${{ matrix.redis_image }} ports: - 6379:6379 options: >- @@ -141,8 +147,15 @@ jobs: google-chrome --version npx playwright install-deps chrome - - name: Run mock-LLM Tier-1 e2e - run: npx playwright test --config=e2e/playwright.config.mock.ts --shard=${{ matrix.shard }}/2 + - name: Run full mock-LLM Tier-1 e2e + if: matrix.suite == 'full' + run: npx playwright test --config=e2e/playwright.config.mock.ts --shard=${{ matrix.shard }} + env: + CI: 'true' + + - name: Run Redis stream transport e2e + if: matrix.suite == 'transport' + run: npx playwright test --config=e2e/playwright.config.redis.ts env: CI: 'true' @@ -150,7 +163,7 @@ jobs: if: ${{ !cancelled() }} uses: actions/upload-artifact@v6 with: - name: playwright-report-${{ matrix.stream_store }}-${{ matrix.shard }} + name: playwright-report-${{ matrix.artifact }} path: e2e/playwright-report/** retention-days: 7 if-no-files-found: ignore @@ -159,7 +172,7 @@ jobs: if: failure() uses: actions/upload-artifact@v6 with: - name: playwright-test-results-${{ matrix.stream_store }}-${{ matrix.shard }} + name: playwright-test-results-${{ matrix.artifact }} path: e2e/specs/.test-results/** retention-days: 7 if-no-files-found: ignore @@ -167,6 +180,7 @@ jobs: mcp_tool_list_changed: name: MCP list_changed (replica count ${{ matrix.replicas }}) if: >- + github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request != null && @@ -278,7 +292,8 @@ jobs: name: e2e if: >- always() && - (github.event_name == 'workflow_dispatch' || + (github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request != null && contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association))) diff --git a/e2e/README.md b/e2e/README.md index 49afe34305..cdae3a7521 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -12,12 +12,19 @@ npm run e2e:mock:redis Memory mode explicitly disables Redis. Redis mode defaults to database 15 with a `LibreChatE2E` key prefix, and fails closed: the test server pings Redis and verifies that the generation job manager did not silently fall back to memory. Override `REDIS_URI` or `E2E_REDIS_KEY_PREFIX` when needed. -CI runs the complete mock suite in both stream modes. Each mode is split across four Playwright shards, while each shard keeps one worker so tests do not contend for the shard's authenticated user and database: +Pull request CI runs the complete mock suite in memory mode across three shards, plus a +focused Redis transport suite. The Redis suite covers streaming fidelity, steering, +interrupts, resumptions, HITL approvals, completion, thread folding, model icons, and usage: ```sh -npx playwright test --config=e2e/playwright.config.mock.ts --shard=1/4 +npx playwright test --config=e2e/playwright.config.mock.ts --shard=1/3 +npm run e2e:mock:redis:transport ``` +The nightly schedule and manual workflow dispatch run the complete mock suite in both stream +modes across two shards per mode. Every shard keeps one worker so tests do not contend for its +authenticated user and database. + ## Property-based browser testing Bombadil explores randomized sequences across the core chat loop, message branches, diff --git a/e2e/playwright.config.redis.ts b/e2e/playwright.config.redis.ts new file mode 100644 index 0000000000..548cb262db --- /dev/null +++ b/e2e/playwright.config.redis.ts @@ -0,0 +1,18 @@ +import { defineConfig } from '@playwright/test'; +import mockConfig from './playwright.config.mock'; + +/** Browser scenarios whose behavior crosses the generation stream-store boundary. */ +export default defineConfig({ + ...mockConfig, + testMatch: [ + /completion\.spec\.ts/, + /deferred-tools-hitl\.spec\.ts/, + /model-spec-icons\.spec\.ts/, + /steering\.spec\.ts/, + /steering-escalation\.spec\.ts/, + /streaming\.spec\.ts/, + /thread-fold\.spec\.ts/, + /tool-approvals\.spec\.ts/, + /usage\.spec\.ts/, + ], +}); diff --git a/package.json b/package.json index 69a88e5834..4e0aaf6620 100644 --- a/package.json +++ b/package.json @@ -67,6 +67,7 @@ "e2e:ci": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.ts", "e2e:mock": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.mock.ts", "e2e:mock:redis": "npm run e2e:prepare && cross-env E2E_STREAM_STORE=redis playwright test --config=e2e/playwright.config.mock.ts", + "e2e:mock:redis:transport": "npm run e2e:prepare && cross-env E2E_STREAM_STORE=redis playwright test --config=e2e/playwright.config.redis.ts", "e2e:benchmark:agents": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.benchmark.ts agent-startup.latency.spec.ts", "e2e:bombadil": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.bombadil.ts", "e2e:bombadil:run": "playwright test --config=e2e/playwright.config.bombadil.ts",