🥼 ci: Gate MCP Apps on Executed Browser Tests (#16283)

Co-authored-by: Lia <lia@librechat.ai>
(cherry picked from commit 1c54ffa411)

Original-PR: https://github.com/LibreChat-AI/LibreChat/pull/16283
This commit is contained in:
lia-by-librechat[bot] 2026-09-24 06:04:01 -04:00 • committed by Lia
parent 7c60a20275
commit 627ded714b
5 changed files with 196 additions and 10 deletions

View file

@ -555,6 +555,81 @@ jobs:
./redis-config/stop-cluster.sh || true
redis-cli -p 6379 shutdown || true
mcp_apps:
name: MCP Apps (browser and server)
# Always run the dedicated fixture on eligible PRs, schedule, and manual runs. The regular
# mock shards skip it unless E2E_MCP_APPS=true; codegraph gating must not omit this lane.
if: >-
github.event_name == 'schedule' ||
github.event_name == 'workflow_dispatch' ||
(github.event_name == 'pull_request' &&
github.event.pull_request != null &&
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association))
runs-on: ubuntu-latest
timeout-minutes: 45
env:
CI: 'true'
E2E_CHROMIUM_CHANNEL: chrome
steps:
- uses: actions/checkout@v5
- name: Use Node.js 24.16.0
uses: actions/setup-node@v5
with:
node-version: '24.16.0'
- name: Restore node_modules cache
id: cache-node-modules
uses: actions/cache@v5
with:
path: |
node_modules
client/node_modules
packages/client/node_modules
packages/data-provider/node_modules
packages/data-schemas/node_modules
packages/api/node_modules
api/node_modules
key: node-modules-e2e-${{ runner.os }}-24.16.0-${{ hashFiles('package-lock.json') }}
- name: Install dependencies
if: steps.cache-node-modules.outputs.cache-hit != 'true'
run: npm ci
- name: Verify Chrome is present
run: google-chrome --version
- name: Check MCP Apps result gating
run: node --test e2e/mcp-apps/results.test.mjs
- name: Check real SDK server fixture
run: node --test e2e/mcp-apps/server.test.mjs
# The runner builds once, then runs all four browser policy phases against one disposable
# MongoDB. It also rejects a Playwright-successful phase if every test was skipped.
- name: Run MCP Apps browser fixture (enabled, disabled, omitted, quota)
run: node e2e/mcp-apps/run.mjs
- name: Upload per-phase Playwright reports
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v6
with:
name: playwright-mcp-apps-reports
path: |
e2e/playwright-report/mcp-apps-*/**
e2e/.generated/mcp-apps-result-*.json
retention-days: 7
if-no-files-found: warn
- name: Upload MCP Apps traces and screenshots
if: failure()
uses: actions/upload-artifact@v6
with:
name: playwright-mcp-apps-results
path: e2e/specs/.test-results/mcp-apps-*/**
retention-days: 7
if-no-files-found: ignore
e2e:
name: e2e
# `!cancelled()`, not `always()`: the gate has to survive a failed shard to report it, but a
@ -569,13 +644,14 @@ jobs:
(github.event_name == 'pull_request' &&
github.event.pull_request != null &&
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association)))
needs: [codegraph_select, e2e_shards, mcp_tool_list_changed]
needs: [codegraph_select, e2e_shards, mcp_tool_list_changed, mcp_apps]
runs-on: ubuntu-latest
steps:
# A codegraph-skipped MCP lane reports `skipped`; that is a decision, not a failure.
- name: Verify every Playwright job passed
if: >-
needs.e2e_shards.result != 'success' ||
needs.mcp_apps.result != 'success' ||
(needs.mcp_tool_list_changed.result != 'success' &&
!(needs.mcp_tool_list_changed.result == 'skipped' && needs.codegraph_select.outputs.mcp_run == 'false'))
run: exit 1

View file

@ -205,9 +205,12 @@ Use Node 24 with the exact lockfile installed and Playwright Chromium available.
root, run the profile with one command:
```bash
node --test e2e/mcp-apps/server.test.mjs
node e2e/mcp-apps/run.mjs
```
The Playwright E2E workflow runs these checks on eligible pull requests, the nightly schedule, and manual dispatch, regardless of codegraph selection. Its required `e2e` gate fails if this dedicated job fails. The runner verifies that enabled, disabled, omitted-policy, and quota phases each executed passing browser tests; a successful Playwright exit with only skipped tests does not pass. CI uploads separate reports for each phase and preserves failing traces and screenshots.
The runner builds the client with the chat at `http://127.0.0.1:3080` and the Sandbox Proxy at
`http://localhost:3080`. Those URLs reach the same disposable LibreChat process but have distinct
browser origins. The aliases are only a disposable local substitute for separate deployment

35
e2e/mcp-apps/results.mjs Normal file
View file

@ -0,0 +1,35 @@
import assert from 'node:assert/strict';
/** Playwright exits successfully when every test is skipped. The dedicated CI lane must not. */
export function assertMCPAppsPhaseResults(report, phase, minimumPassed) {
assert.ok(
Array.isArray(report?.suites) && Array.isArray(report.errors),
`MCP Apps ${phase}: missing Playwright JSON results`,
);
assert.equal(report.errors.length, 0, `MCP Apps ${phase}: Playwright reported errors`);
const collect = (suites) =>
(suites ?? []).reduce(
(passed, suite) =>
passed +
(suite.specs ?? []).reduce(
(count, spec) =>
count +
(spec.tests ?? []).filter(
(test) =>
test.expectedStatus === 'passed' &&
test.results?.some((result) => result.status === 'passed'),
).length,
0,
) +
collect(suite.suites),
0,
);
const passed = collect(report.suites);
assert.ok(
passed >= minimumPassed,
`MCP Apps ${phase}: expected at least ${minimumPassed} executed passing tests, got ${passed}`,
);
return passed;
}

View file

@ -0,0 +1,49 @@
import assert from 'node:assert/strict';
import { test } from 'node:test';
import { assertMCPAppsPhaseResults } from './results.mjs';
const result = (status, expectedStatus = 'passed') => ({
expectedStatus,
results: [{ status }],
});
const report = (...tests) => ({
errors: [],
suites: [{ suites: [{ specs: [{ tests }] }] }],
});
test('requires the expected number of executed, passing tests for each phase', () => {
assert.equal(
assertMCPAppsPhaseResults(
report(result('passed'), result('passed'), result('skipped', 'skipped')),
'true',
2,
),
2,
);
assert.equal(assertMCPAppsPhaseResults(report(result('passed')), 'quota', 1), 1);
});
test('rejects skipped-only and missing-test reports even when Playwright exits successfully', () => {
assert.throws(
() => assertMCPAppsPhaseResults(report(result('skipped', 'skipped')), 'false', 1),
/expected at least 1 executed passing tests, got 0/,
);
assert.throws(() => assertMCPAppsPhaseResults(report(), 'omitted', 1), /got 0/);
assert.throws(() => assertMCPAppsPhaseResults({}, 'quota', 1), /missing Playwright JSON results/);
});
test('does not count a skipped test as a passing test on retry', () => {
assert.throws(
() => assertMCPAppsPhaseResults(report(result('passed', 'skipped')), 'true', 1),
/got 0/,
);
assert.throws(
() =>
assertMCPAppsPhaseResults(
{ ...report(result('passed')), errors: [{ message: 'failed' }] },
'quota',
1,
),
/Playwright reported errors/,
);
});

View file

@ -1,10 +1,12 @@
#!/usr/bin/env node
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { randomBytes } from 'node:crypto';
import { spawn } from 'node:child_process';
import { createRequire } from 'node:module';
import { assertMCPAppsPhaseResults } from './results.mjs';
const baseUrl = new URL(process.env.E2E_BASE_URL || 'http://127.0.0.1:3080');
const sandboxHost = baseUrl.hostname === 'localhost' ? '127.0.0.1' : 'localhost';
@ -52,12 +54,13 @@ function run(executable, args, overrides = {}) {
}
const phases = [
{ name: 'true', policy: 'true' },
{ name: 'false', policy: 'false' },
{ name: 'omitted', policy: 'omitted' },
{ name: 'true', policy: 'true', minimumPassed: 2 },
{ name: 'false', policy: 'false', minimumPassed: 1 },
{ name: 'omitted', policy: 'omitted', minimumPassed: 1 },
{
name: 'quota',
policy: 'true',
minimumPassed: 1,
E2E_MCP_APP_RESOURCE_LIMIT: '2',
E2E_MCP_APP_TOOL_CALL_LIMIT: '2',
},
@ -74,12 +77,32 @@ try {
env.MONGO_URI = mongo.getUri('LibreChat-e2e');
for (const phase of phases) {
await run('npx', ['playwright', 'test', '--config=e2e/playwright.config.mcp-apps.ts'], {
E2E_MCP_APPS_PHASE: phase.name,
E2E_MCP_APPS_POLICY: phase.policy,
E2E_MCP_APP_RESOURCE_LIMIT: phase.E2E_MCP_APP_RESOURCE_LIMIT || '',
E2E_MCP_APP_TOOL_CALL_LIMIT: phase.E2E_MCP_APP_TOOL_CALL_LIMIT || '',
});
const resultsFile = path.join(root, `e2e/.generated/mcp-apps-result-${phase.name}.json`);
fs.rmSync(resultsFile, { force: true });
await run(
'npx',
[
'playwright',
'test',
'--config=e2e/playwright.config.mcp-apps.ts',
'--reporter=line,html,json',
],
{
E2E_MCP_APPS_PHASE: phase.name,
E2E_MCP_APPS_POLICY: phase.policy,
E2E_MCP_APP_RESOURCE_LIMIT: phase.E2E_MCP_APP_RESOURCE_LIMIT || '',
E2E_MCP_APP_TOOL_CALL_LIMIT: phase.E2E_MCP_APP_TOOL_CALL_LIMIT || '',
PLAYWRIGHT_HTML_OPEN: 'never',
PLAYWRIGHT_HTML_OUTPUT_DIR: path.join(root, `e2e/playwright-report/mcp-apps-${phase.name}`),
PLAYWRIGHT_JSON_OUTPUT_FILE: resultsFile,
},
);
const passed = assertMCPAppsPhaseResults(
JSON.parse(fs.readFileSync(resultsFile, 'utf8')),
phase.name,
phase.minimumPassed,
);
console.log(`MCP Apps ${phase.name}: ${passed} browser test(s) passed`);
}
} finally {
await mongo?.stop();