mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-19 14:33:38 +00:00
🥼 ci: Gate MCP Apps on Executed Browser Tests (#16283)
Co-authored-by: Lia <lia@librechat.ai>
(cherry picked from commit 1c54ffa411)
Original-PR: https://github.com/LibreChat-AI/LibreChat/pull/16283
This commit is contained in:
parent
7c60a20275
commit
627ded714b
5 changed files with 196 additions and 10 deletions
78
.github/workflows/playwright-mock.yml
vendored
78
.github/workflows/playwright-mock.yml
vendored
|
|
@ -555,6 +555,81 @@ jobs:
|
|||
./redis-config/stop-cluster.sh || true
|
||||
redis-cli -p 6379 shutdown || true
|
||||
|
||||
mcp_apps:
|
||||
name: MCP Apps (browser and server)
|
||||
# Always run the dedicated fixture on eligible PRs, schedule, and manual runs. The regular
|
||||
# mock shards skip it unless E2E_MCP_APPS=true; codegraph gating must not omit this lane.
|
||||
if: >-
|
||||
github.event_name == 'schedule' ||
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
(github.event_name == 'pull_request' &&
|
||||
github.event.pull_request != null &&
|
||||
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
CI: 'true'
|
||||
E2E_CHROMIUM_CHANNEL: chrome
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
- name: Use Node.js 24.16.0
|
||||
uses: actions/setup-node@v5
|
||||
with:
|
||||
node-version: '24.16.0'
|
||||
|
||||
- name: Restore node_modules cache
|
||||
id: cache-node-modules
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
node_modules
|
||||
client/node_modules
|
||||
packages/client/node_modules
|
||||
packages/data-provider/node_modules
|
||||
packages/data-schemas/node_modules
|
||||
packages/api/node_modules
|
||||
api/node_modules
|
||||
key: node-modules-e2e-${{ runner.os }}-24.16.0-${{ hashFiles('package-lock.json') }}
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.cache-node-modules.outputs.cache-hit != 'true'
|
||||
run: npm ci
|
||||
|
||||
- name: Verify Chrome is present
|
||||
run: google-chrome --version
|
||||
|
||||
- name: Check MCP Apps result gating
|
||||
run: node --test e2e/mcp-apps/results.test.mjs
|
||||
|
||||
- name: Check real SDK server fixture
|
||||
run: node --test e2e/mcp-apps/server.test.mjs
|
||||
|
||||
# The runner builds once, then runs all four browser policy phases against one disposable
|
||||
# MongoDB. It also rejects a Playwright-successful phase if every test was skipped.
|
||||
- name: Run MCP Apps browser fixture (enabled, disabled, omitted, quota)
|
||||
run: node e2e/mcp-apps/run.mjs
|
||||
|
||||
- name: Upload per-phase Playwright reports
|
||||
if: ${{ !cancelled() }}
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: playwright-mcp-apps-reports
|
||||
path: |
|
||||
e2e/playwright-report/mcp-apps-*/**
|
||||
e2e/.generated/mcp-apps-result-*.json
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload MCP Apps traces and screenshots
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: playwright-mcp-apps-results
|
||||
path: e2e/specs/.test-results/mcp-apps-*/**
|
||||
retention-days: 7
|
||||
if-no-files-found: ignore
|
||||
|
||||
e2e:
|
||||
name: e2e
|
||||
# `!cancelled()`, not `always()`: the gate has to survive a failed shard to report it, but a
|
||||
|
|
@ -569,13 +644,14 @@ jobs:
|
|||
(github.event_name == 'pull_request' &&
|
||||
github.event.pull_request != null &&
|
||||
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association)))
|
||||
needs: [codegraph_select, e2e_shards, mcp_tool_list_changed]
|
||||
needs: [codegraph_select, e2e_shards, mcp_tool_list_changed, mcp_apps]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# A codegraph-skipped MCP lane reports `skipped`; that is a decision, not a failure.
|
||||
- name: Verify every Playwright job passed
|
||||
if: >-
|
||||
needs.e2e_shards.result != 'success' ||
|
||||
needs.mcp_apps.result != 'success' ||
|
||||
(needs.mcp_tool_list_changed.result != 'success' &&
|
||||
!(needs.mcp_tool_list_changed.result == 'skipped' && needs.codegraph_select.outputs.mcp_run == 'false'))
|
||||
run: exit 1
|
||||
|
|
|
|||
|
|
@ -205,9 +205,12 @@ Use Node 24 with the exact lockfile installed and Playwright Chromium available.
|
|||
root, run the profile with one command:
|
||||
|
||||
```bash
|
||||
node --test e2e/mcp-apps/server.test.mjs
|
||||
node e2e/mcp-apps/run.mjs
|
||||
```
|
||||
|
||||
The Playwright E2E workflow runs these checks on eligible pull requests, the nightly schedule, and manual dispatch, regardless of codegraph selection. Its required `e2e` gate fails if this dedicated job fails. The runner verifies that enabled, disabled, omitted-policy, and quota phases each executed passing browser tests; a successful Playwright exit with only skipped tests does not pass. CI uploads separate reports for each phase and preserves failing traces and screenshots.
|
||||
|
||||
The runner builds the client with the chat at `http://127.0.0.1:3080` and the Sandbox Proxy at
|
||||
`http://localhost:3080`. Those URLs reach the same disposable LibreChat process but have distinct
|
||||
browser origins. The aliases are only a disposable local substitute for separate deployment
|
||||
|
|
|
|||
35
e2e/mcp-apps/results.mjs
Normal file
35
e2e/mcp-apps/results.mjs
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
import assert from 'node:assert/strict';
|
||||
|
||||
/** Playwright exits successfully when every test is skipped. The dedicated CI lane must not. */
|
||||
export function assertMCPAppsPhaseResults(report, phase, minimumPassed) {
|
||||
assert.ok(
|
||||
Array.isArray(report?.suites) && Array.isArray(report.errors),
|
||||
`MCP Apps ${phase}: missing Playwright JSON results`,
|
||||
);
|
||||
assert.equal(report.errors.length, 0, `MCP Apps ${phase}: Playwright reported errors`);
|
||||
|
||||
const collect = (suites) =>
|
||||
(suites ?? []).reduce(
|
||||
(passed, suite) =>
|
||||
passed +
|
||||
(suite.specs ?? []).reduce(
|
||||
(count, spec) =>
|
||||
count +
|
||||
(spec.tests ?? []).filter(
|
||||
(test) =>
|
||||
test.expectedStatus === 'passed' &&
|
||||
test.results?.some((result) => result.status === 'passed'),
|
||||
).length,
|
||||
0,
|
||||
) +
|
||||
collect(suite.suites),
|
||||
0,
|
||||
);
|
||||
|
||||
const passed = collect(report.suites);
|
||||
assert.ok(
|
||||
passed >= minimumPassed,
|
||||
`MCP Apps ${phase}: expected at least ${minimumPassed} executed passing tests, got ${passed}`,
|
||||
);
|
||||
return passed;
|
||||
}
|
||||
49
e2e/mcp-apps/results.test.mjs
Normal file
49
e2e/mcp-apps/results.test.mjs
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
import assert from 'node:assert/strict';
|
||||
import { test } from 'node:test';
|
||||
import { assertMCPAppsPhaseResults } from './results.mjs';
|
||||
|
||||
const result = (status, expectedStatus = 'passed') => ({
|
||||
expectedStatus,
|
||||
results: [{ status }],
|
||||
});
|
||||
const report = (...tests) => ({
|
||||
errors: [],
|
||||
suites: [{ suites: [{ specs: [{ tests }] }] }],
|
||||
});
|
||||
|
||||
test('requires the expected number of executed, passing tests for each phase', () => {
|
||||
assert.equal(
|
||||
assertMCPAppsPhaseResults(
|
||||
report(result('passed'), result('passed'), result('skipped', 'skipped')),
|
||||
'true',
|
||||
2,
|
||||
),
|
||||
2,
|
||||
);
|
||||
assert.equal(assertMCPAppsPhaseResults(report(result('passed')), 'quota', 1), 1);
|
||||
});
|
||||
|
||||
test('rejects skipped-only and missing-test reports even when Playwright exits successfully', () => {
|
||||
assert.throws(
|
||||
() => assertMCPAppsPhaseResults(report(result('skipped', 'skipped')), 'false', 1),
|
||||
/expected at least 1 executed passing tests, got 0/,
|
||||
);
|
||||
assert.throws(() => assertMCPAppsPhaseResults(report(), 'omitted', 1), /got 0/);
|
||||
assert.throws(() => assertMCPAppsPhaseResults({}, 'quota', 1), /missing Playwright JSON results/);
|
||||
});
|
||||
|
||||
test('does not count a skipped test as a passing test on retry', () => {
|
||||
assert.throws(
|
||||
() => assertMCPAppsPhaseResults(report(result('passed', 'skipped')), 'true', 1),
|
||||
/got 0/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
assertMCPAppsPhaseResults(
|
||||
{ ...report(result('passed')), errors: [{ message: 'failed' }] },
|
||||
'quota',
|
||||
1,
|
||||
),
|
||||
/Playwright reported errors/,
|
||||
);
|
||||
});
|
||||
|
|
@ -1,10 +1,12 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { createRequire } from 'node:module';
|
||||
import { assertMCPAppsPhaseResults } from './results.mjs';
|
||||
|
||||
const baseUrl = new URL(process.env.E2E_BASE_URL || 'http://127.0.0.1:3080');
|
||||
const sandboxHost = baseUrl.hostname === 'localhost' ? '127.0.0.1' : 'localhost';
|
||||
|
|
@ -52,12 +54,13 @@ function run(executable, args, overrides = {}) {
|
|||
}
|
||||
|
||||
const phases = [
|
||||
{ name: 'true', policy: 'true' },
|
||||
{ name: 'false', policy: 'false' },
|
||||
{ name: 'omitted', policy: 'omitted' },
|
||||
{ name: 'true', policy: 'true', minimumPassed: 2 },
|
||||
{ name: 'false', policy: 'false', minimumPassed: 1 },
|
||||
{ name: 'omitted', policy: 'omitted', minimumPassed: 1 },
|
||||
{
|
||||
name: 'quota',
|
||||
policy: 'true',
|
||||
minimumPassed: 1,
|
||||
E2E_MCP_APP_RESOURCE_LIMIT: '2',
|
||||
E2E_MCP_APP_TOOL_CALL_LIMIT: '2',
|
||||
},
|
||||
|
|
@ -74,12 +77,32 @@ try {
|
|||
env.MONGO_URI = mongo.getUri('LibreChat-e2e');
|
||||
|
||||
for (const phase of phases) {
|
||||
await run('npx', ['playwright', 'test', '--config=e2e/playwright.config.mcp-apps.ts'], {
|
||||
E2E_MCP_APPS_PHASE: phase.name,
|
||||
E2E_MCP_APPS_POLICY: phase.policy,
|
||||
E2E_MCP_APP_RESOURCE_LIMIT: phase.E2E_MCP_APP_RESOURCE_LIMIT || '',
|
||||
E2E_MCP_APP_TOOL_CALL_LIMIT: phase.E2E_MCP_APP_TOOL_CALL_LIMIT || '',
|
||||
});
|
||||
const resultsFile = path.join(root, `e2e/.generated/mcp-apps-result-${phase.name}.json`);
|
||||
fs.rmSync(resultsFile, { force: true });
|
||||
await run(
|
||||
'npx',
|
||||
[
|
||||
'playwright',
|
||||
'test',
|
||||
'--config=e2e/playwright.config.mcp-apps.ts',
|
||||
'--reporter=line,html,json',
|
||||
],
|
||||
{
|
||||
E2E_MCP_APPS_PHASE: phase.name,
|
||||
E2E_MCP_APPS_POLICY: phase.policy,
|
||||
E2E_MCP_APP_RESOURCE_LIMIT: phase.E2E_MCP_APP_RESOURCE_LIMIT || '',
|
||||
E2E_MCP_APP_TOOL_CALL_LIMIT: phase.E2E_MCP_APP_TOOL_CALL_LIMIT || '',
|
||||
PLAYWRIGHT_HTML_OPEN: 'never',
|
||||
PLAYWRIGHT_HTML_OUTPUT_DIR: path.join(root, `e2e/playwright-report/mcp-apps-${phase.name}`),
|
||||
PLAYWRIGHT_JSON_OUTPUT_FILE: resultsFile,
|
||||
},
|
||||
);
|
||||
const passed = assertMCPAppsPhaseResults(
|
||||
JSON.parse(fs.readFileSync(resultsFile, 'utf8')),
|
||||
phase.name,
|
||||
phase.minimumPassed,
|
||||
);
|
||||
console.log(`MCP Apps ${phase.name}: ${passed} browser test(s) passed`);
|
||||
}
|
||||
} finally {
|
||||
await mongo?.stop();
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue