diff --git a/.github/workflows/playwright-mock.yml b/.github/workflows/playwright-mock.yml index aa56ad71d3..43122021e9 100644 --- a/.github/workflows/playwright-mock.yml +++ b/.github/workflows/playwright-mock.yml @@ -555,6 +555,81 @@ jobs: ./redis-config/stop-cluster.sh || true redis-cli -p 6379 shutdown || true + mcp_apps: + name: MCP Apps (browser and server) + # Always run the dedicated fixture on eligible PRs, schedule, and manual runs. The regular + # mock shards skip it unless E2E_MCP_APPS=true; codegraph gating must not omit this lane. + if: >- + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'pull_request' && + github.event.pull_request != null && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association)) + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + CI: 'true' + E2E_CHROMIUM_CHANNEL: chrome + steps: + - uses: actions/checkout@v5 + + - name: Use Node.js 24.16.0 + uses: actions/setup-node@v5 + with: + node-version: '24.16.0' + + - name: Restore node_modules cache + id: cache-node-modules + uses: actions/cache@v5 + with: + path: | + node_modules + client/node_modules + packages/client/node_modules + packages/data-provider/node_modules + packages/data-schemas/node_modules + packages/api/node_modules + api/node_modules + key: node-modules-e2e-${{ runner.os }}-24.16.0-${{ hashFiles('package-lock.json') }} + + - name: Install dependencies + if: steps.cache-node-modules.outputs.cache-hit != 'true' + run: npm ci + + - name: Verify Chrome is present + run: google-chrome --version + + - name: Check MCP Apps result gating + run: node --test e2e/mcp-apps/results.test.mjs + + - name: Check real SDK server fixture + run: node --test e2e/mcp-apps/server.test.mjs + + # The runner builds once, then runs all four browser policy phases against one disposable + # MongoDB. It also rejects a Playwright-successful phase if every test was skipped. + - name: Run MCP Apps browser fixture (enabled, disabled, omitted, quota) + run: node e2e/mcp-apps/run.mjs + + - name: Upload per-phase Playwright reports + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v6 + with: + name: playwright-mcp-apps-reports + path: | + e2e/playwright-report/mcp-apps-*/** + e2e/.generated/mcp-apps-result-*.json + retention-days: 7 + if-no-files-found: warn + + - name: Upload MCP Apps traces and screenshots + if: failure() + uses: actions/upload-artifact@v6 + with: + name: playwright-mcp-apps-results + path: e2e/specs/.test-results/mcp-apps-*/** + retention-days: 7 + if-no-files-found: ignore + e2e: name: e2e # `!cancelled()`, not `always()`: the gate has to survive a failed shard to report it, but a @@ -569,13 +644,14 @@ jobs: (github.event_name == 'pull_request' && github.event.pull_request != null && contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association))) - needs: [codegraph_select, e2e_shards, mcp_tool_list_changed] + needs: [codegraph_select, e2e_shards, mcp_tool_list_changed, mcp_apps] runs-on: ubuntu-latest steps: # A codegraph-skipped MCP lane reports `skipped`; that is a decision, not a failure. - name: Verify every Playwright job passed if: >- needs.e2e_shards.result != 'success' || + needs.mcp_apps.result != 'success' || (needs.mcp_tool_list_changed.result != 'success' && !(needs.mcp_tool_list_changed.result == 'skipped' && needs.codegraph_select.outputs.mcp_run == 'false')) run: exit 1 diff --git a/docs/mcp-apps.md b/docs/mcp-apps.md index d1991b4770..2d0ff3995e 100644 --- a/docs/mcp-apps.md +++ b/docs/mcp-apps.md @@ -205,9 +205,12 @@ Use Node 24 with the exact lockfile installed and Playwright Chromium available. root, run the profile with one command: ```bash +node --test e2e/mcp-apps/server.test.mjs node e2e/mcp-apps/run.mjs ``` +The Playwright E2E workflow runs these checks on eligible pull requests, the nightly schedule, and manual dispatch, regardless of codegraph selection. Its required `e2e` gate fails if this dedicated job fails. The runner verifies that enabled, disabled, omitted-policy, and quota phases each executed passing browser tests; a successful Playwright exit with only skipped tests does not pass. CI uploads separate reports for each phase and preserves failing traces and screenshots. + The runner builds the client with the chat at `http://127.0.0.1:3080` and the Sandbox Proxy at `http://localhost:3080`. Those URLs reach the same disposable LibreChat process but have distinct browser origins. The aliases are only a disposable local substitute for separate deployment diff --git a/e2e/mcp-apps/results.mjs b/e2e/mcp-apps/results.mjs new file mode 100644 index 0000000000..628d60efb7 --- /dev/null +++ b/e2e/mcp-apps/results.mjs @@ -0,0 +1,35 @@ +import assert from 'node:assert/strict'; + +/** Playwright exits successfully when every test is skipped. The dedicated CI lane must not. */ +export function assertMCPAppsPhaseResults(report, phase, minimumPassed) { + assert.ok( + Array.isArray(report?.suites) && Array.isArray(report.errors), + `MCP Apps ${phase}: missing Playwright JSON results`, + ); + assert.equal(report.errors.length, 0, `MCP Apps ${phase}: Playwright reported errors`); + + const collect = (suites) => + (suites ?? []).reduce( + (passed, suite) => + passed + + (suite.specs ?? []).reduce( + (count, spec) => + count + + (spec.tests ?? []).filter( + (test) => + test.expectedStatus === 'passed' && + test.results?.some((result) => result.status === 'passed'), + ).length, + 0, + ) + + collect(suite.suites), + 0, + ); + + const passed = collect(report.suites); + assert.ok( + passed >= minimumPassed, + `MCP Apps ${phase}: expected at least ${minimumPassed} executed passing tests, got ${passed}`, + ); + return passed; +} diff --git a/e2e/mcp-apps/results.test.mjs b/e2e/mcp-apps/results.test.mjs new file mode 100644 index 0000000000..2fd9c79b76 --- /dev/null +++ b/e2e/mcp-apps/results.test.mjs @@ -0,0 +1,49 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { assertMCPAppsPhaseResults } from './results.mjs'; + +const result = (status, expectedStatus = 'passed') => ({ + expectedStatus, + results: [{ status }], +}); +const report = (...tests) => ({ + errors: [], + suites: [{ suites: [{ specs: [{ tests }] }] }], +}); + +test('requires the expected number of executed, passing tests for each phase', () => { + assert.equal( + assertMCPAppsPhaseResults( + report(result('passed'), result('passed'), result('skipped', 'skipped')), + 'true', + 2, + ), + 2, + ); + assert.equal(assertMCPAppsPhaseResults(report(result('passed')), 'quota', 1), 1); +}); + +test('rejects skipped-only and missing-test reports even when Playwright exits successfully', () => { + assert.throws( + () => assertMCPAppsPhaseResults(report(result('skipped', 'skipped')), 'false', 1), + /expected at least 1 executed passing tests, got 0/, + ); + assert.throws(() => assertMCPAppsPhaseResults(report(), 'omitted', 1), /got 0/); + assert.throws(() => assertMCPAppsPhaseResults({}, 'quota', 1), /missing Playwright JSON results/); +}); + +test('does not count a skipped test as a passing test on retry', () => { + assert.throws( + () => assertMCPAppsPhaseResults(report(result('passed', 'skipped')), 'true', 1), + /got 0/, + ); + assert.throws( + () => + assertMCPAppsPhaseResults( + { ...report(result('passed')), errors: [{ message: 'failed' }] }, + 'quota', + 1, + ), + /Playwright reported errors/, + ); +}); diff --git a/e2e/mcp-apps/run.mjs b/e2e/mcp-apps/run.mjs index 4276a338f8..1c22481269 100644 --- a/e2e/mcp-apps/run.mjs +++ b/e2e/mcp-apps/run.mjs @@ -1,10 +1,12 @@ #!/usr/bin/env node +import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { randomBytes } from 'node:crypto'; import { spawn } from 'node:child_process'; import { createRequire } from 'node:module'; +import { assertMCPAppsPhaseResults } from './results.mjs'; const baseUrl = new URL(process.env.E2E_BASE_URL || 'http://127.0.0.1:3080'); const sandboxHost = baseUrl.hostname === 'localhost' ? '127.0.0.1' : 'localhost'; @@ -52,12 +54,13 @@ function run(executable, args, overrides = {}) { } const phases = [ - { name: 'true', policy: 'true' }, - { name: 'false', policy: 'false' }, - { name: 'omitted', policy: 'omitted' }, + { name: 'true', policy: 'true', minimumPassed: 2 }, + { name: 'false', policy: 'false', minimumPassed: 1 }, + { name: 'omitted', policy: 'omitted', minimumPassed: 1 }, { name: 'quota', policy: 'true', + minimumPassed: 1, E2E_MCP_APP_RESOURCE_LIMIT: '2', E2E_MCP_APP_TOOL_CALL_LIMIT: '2', }, @@ -74,12 +77,32 @@ try { env.MONGO_URI = mongo.getUri('LibreChat-e2e'); for (const phase of phases) { - await run('npx', ['playwright', 'test', '--config=e2e/playwright.config.mcp-apps.ts'], { - E2E_MCP_APPS_PHASE: phase.name, - E2E_MCP_APPS_POLICY: phase.policy, - E2E_MCP_APP_RESOURCE_LIMIT: phase.E2E_MCP_APP_RESOURCE_LIMIT || '', - E2E_MCP_APP_TOOL_CALL_LIMIT: phase.E2E_MCP_APP_TOOL_CALL_LIMIT || '', - }); + const resultsFile = path.join(root, `e2e/.generated/mcp-apps-result-${phase.name}.json`); + fs.rmSync(resultsFile, { force: true }); + await run( + 'npx', + [ + 'playwright', + 'test', + '--config=e2e/playwright.config.mcp-apps.ts', + '--reporter=line,html,json', + ], + { + E2E_MCP_APPS_PHASE: phase.name, + E2E_MCP_APPS_POLICY: phase.policy, + E2E_MCP_APP_RESOURCE_LIMIT: phase.E2E_MCP_APP_RESOURCE_LIMIT || '', + E2E_MCP_APP_TOOL_CALL_LIMIT: phase.E2E_MCP_APP_TOOL_CALL_LIMIT || '', + PLAYWRIGHT_HTML_OPEN: 'never', + PLAYWRIGHT_HTML_OUTPUT_DIR: path.join(root, `e2e/playwright-report/mcp-apps-${phase.name}`), + PLAYWRIGHT_JSON_OUTPUT_FILE: resultsFile, + }, + ); + const passed = assertMCPAppsPhaseResults( + JSON.parse(fs.readFileSync(resultsFile, 'utf8')), + phase.name, + phase.minimumPassed, + ); + console.log(`MCP Apps ${phase.name}: ${passed} browser test(s) passed`); } } finally { await mongo?.stop();