🎈 ci: Gate Page Loads Under Database Latency (#15646)

Co-authored-by: Danny Avila <danny@librechat.ai>

.github/workflows/lighthouse.yml

@@ -0,0 +1,71 @@
+name: Lighthouse CI
+
+on:
+  pull_request:
+    paths:
+      - 'api/**'
+      - 'client/**'
+      - 'config/**'
+      - 'packages/**'
+      - 'e2e/**'
+      - 'package.json'
+      - 'package-lock.json'
+      - '.github/workflows/lighthouse.yml'
+      - '!**.md'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pull-requests: write
+
+concurrency:
+  group: lighthouse-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lighthouse:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    env:
+      E2E_CHROMIUM_CHANNEL: chrome
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-node@v5
+        with:
+          node-version: '24.16.0'
+          cache: npm
+      - run: npm ci
+      - run: npm run e2e:prepare
+      - run: google-chrome --version
+      - name: Audit serial database latency
+        id: audit
+        shell: bash
+        run: |
+          set -o pipefail
+          npm run lighthouse:run 2>&1 | tee lighthouse-ci.log
+      - name: Comment Lighthouse findings
+        if: ${{ failure() && steps.audit.outcome == 'failure' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          {
+            echo 'Lighthouse CI failed. The last 80 log lines contain the measured budgets and assertion failures.'
+            echo
+            echo '```text'
+            tail -n 80 lighthouse-ci.log
+            echo '```'
+            echo
+            echo "[Open the full run]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)"
+          } > lighthouse-comment.md
+          gh pr comment "$PR_NUMBER" --body-file lighthouse-comment.md
+      - name: Upload Lighthouse reports
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v6
+        with:
+          name: lighthouse
+          path: |
+            .lighthouseci/
+            lighthouse-ci.log
+          include-hidden-files: true
+          retention-days: 7

.gitignore

@@ -189,3 +189,7 @@ CLAUDE.md
 .gsd
 codedb.snapshot
 scripts/activity-labels/results/
+
+!e2e/lighthouse/librechat.yaml
+/.lighthouseci/
+/e2e/lighthouse/.test-results/

AGENTS.md

@@ -12,6 +12,11 @@ default branch, so close linked issues by hand. Worktrees share one stash stack,
 
 ## Verification
 
+For startup, auth, config, file, or message-loading changes, avoid serial database
+reads and reuse loaded request data. Run `npm run lighthouse` before completion:
+the CI lane adds 250 ms per Mongo query and checks the visible conversation's LCP.
+See [budgets, reproduction and failure diagnosis](e2e/lighthouse/README.md).
+
 A green build is not a typecheck: `packages/api`, `packages/client` and `packages/data-schemas` build
 with `tsdown`, which emits without checking types. Run `npx tsc --noEmit` in the workspace you
 changed. `packages/client` excludes `*.spec.ts(x)` and `*.test.ts(x)` from typechecking entirely.

e2e/lighthouse/README.md

@@ -0,0 +1,77 @@
+# Serial database latency Lighthouse CI
+
+Run from the repository root with Node 24 and Chrome installed:
+
+```sh
+npm ci
+E2E_CHROMIUM_CHANNEL=chrome npm run lighthouse
+# Reuse the production build:
+E2E_CHROMIUM_CHANNEL=chrome npm run lighthouse:run
+# Negative control: this MUST exit nonzero with an LCP assertion failure:
+E2E_CHROMIUM_CHANNEL=chrome npm run lighthouse:regression
+```
+
+`E2E_BASE_URL=http://localhost:3098` selects another local port. Each run starts a
+disposable MongoDB and the real Express server, registers a local user, and seeds
+a conversation. No model inference is needed. Do not point this test at a deployed
+service. Playwright refuses to reuse an existing server.
+
+The existing `benchmarks/mongoose-latency-hook.cjs` adds **250 ms per Mongoose
+Query/Aggregate execution** in the server process. Independent queries can overlap;
+serial queries compound. This is a deterministic approximation of remote database
+latency, not a replica topology or network emulator. It also delays query-based
+writes; native driver calls, bulk operations and cursor batches are outside its
+coverage. Use a TCP latency proxy if those paths need coverage.
+
+Lighthouse CI makes three cold browser navigations to a populated conversation,
+using the production client build and real authentication, config, file and message
+routes. It uses desktop settings with `throttlingMethod: provided` so Lighthouse
+does not replace the measured server delays with simulated network timing.
+Median budgets are LCP **4,500 ms**, CLS **0.1**, and TBT **500 ms**. These are lab
+regression budgets, not field web-vitals percentiles; Lighthouse does not measure INP.
+The test also requires the seeded transcript to be the LCP element, so a fast
+login page, spinner, or empty shell cannot pass.
+
+## When the gate fails
+
+1. Read the failed audit's actual value and limit in the job log or
+   `.lighthouseci/assertion-results.json`.
+2. Open a `.lighthouseci/lhr-*.html` report. The console also prints API request
+   start/end times. A late request start suggests a browser dependency; a long
+   request suggests server work or serial database reads.
+3. Inspect the relevant path before changing the budget:
+
+| Request / symptom                           | Code to inspect                                                                           | Performance change this protects                                                                                                                                                            |
+| ------------------------------------------- | ----------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Config waits for repeated user lookups      | `packages/api/src/app/service.ts`, `api/server/middleware/config/app.js`                  | [#14101](https://github.com/danny-avila/LibreChat/pull/14101)                                                                                                                               |
+| Message authorization and read run serially | `api/server/routes/messages.js`, `packages/api/src/middleware/messageValidation.ts`       | [#14101](https://github.com/danny-avila/LibreChat/pull/14101)                                                                                                                               |
+| Messages wait for the file map              | `client/src/data-provider/Messages/queries.ts`, `client/src/components/Chat/ChatView.tsx` | [#14188](https://github.com/danny-avila/LibreChat/pull/14188)                                                                                                                               |
+| Startup repeats auth user reads             | `api/server/controllers/AuthController.js`, `packages/api/src/auth/userDocCache.ts`       | [#14187](https://github.com/danny-avila/LibreChat/pull/14187), [#14343](https://github.com/danny-avila/LibreChat/pull/14343), [#14747](https://github.com/danny-avila/LibreChat/pull/14747) |
+
+Reuse already-loaded user/config data. Start independent reads together, but keep
+every read scoped to the authenticated user/tenant and wait for authorization
+before returning data. Do not raise a threshold to hide added round trips.
+
+## Adding another scenario
+
+`auditPage({ url, cookies, configPath })` in `audit.ts` collects the reports,
+redacts authentication headers, prints API timings and enforces the selected LHCI
+budgets. It returns the reports so each scenario can assert its expected final
+URL and LCP content. `load.spec.ts` owns local login state, conversation seeding
+and transcript assertions; the runner does not depend on them.
+
+A downstream fork can import this runner from a separate launch spec, supply
+cookies from its own authentication fixture and use a separate LHCI config for
+launch budgets. The database delay hook can be preloaded by that fixture's server
+configuration. Keep provider-specific authentication and provisioning in that spec.
+For launch flows that navigate across documents, also measure entry-to-ready time
+with Playwright: LCP resets on a new document, so final-page LCP alone does not
+cover the entire launch. This lane does not exercise OpenID/Redis cache priming.
+
+`lighthouse:regression` preloads a test-only hook that adds 16 real, sequential
+user reads before message retrieval (at least four extra seconds). It uses the
+same page and LHCI assertions, and never modifies production source. Confirm
+that `largest-contentful-paint` fails, rather than treating any process error as
+proof. Run the normal command again to restore baseline reports. Reports stay
+local or in GitHub job artifacts; session cookie values are redacted before upload.
+Same-repository pull requests also receive the last 80 log lines as a failure comment.

e2e/lighthouse/audit.ts

@@ -0,0 +1,72 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { promisify } from 'node:util';
+import { execFile } from 'node:child_process';
+import { expect } from '@playwright/test';
+import type { Cookie } from '@playwright/test';
+import type Result from 'lighthouse/types/lhr/lhr';
+
+const exec = promisify(execFile);
+
+export async function auditPage({
+  url,
+  cookies,
+  configPath = path.resolve(__dirname, 'lighthouserc.cjs'),
+}: {
+  url: string;
+  cookies: Cookie[];
+  configPath?: string;
+}): Promise<Result[]> {
+  const env = {
+    ...process.env,
+    LIGHTHOUSE_URL: url,
+    LIGHTHOUSE_COOKIE: cookies.map(({ name, value }) => `${name}=${value}`).join('; '),
+  };
+  const cli = require.resolve('@lhci/cli/src/cli.js');
+  const config = `--config=${configPath}`;
+  const run = (command: string) => exec(process.execPath, [cli, command, config], { env });
+  const directory = path.resolve('.lighthouseci');
+  try {
+    console.log((await run('collect')).stdout);
+  } finally {
+    // LHCI embeds extraHeaders in reports; keep disposable session cookies out of artifacts.
+    for (const file of fs.existsSync(directory) ? fs.readdirSync(directory) : []) {
+      if (!/\.(json|html)$/.test(file)) continue;
+      const filename = path.join(directory, file);
+      const content = fs.readFileSync(filename, 'utf8');
+      fs.writeFileSync(
+        filename,
+        env.LIGHTHOUSE_COOKIE ? content.replaceAll(env.LIGHTHOUSE_COOKIE, '[redacted]') : content,
+      );
+    }
+  }
+  const reports = fs.readdirSync(directory).filter((file) => /^lhr-.*\.json$/.test(file));
+  expect(reports.length, 'Lighthouse must produce reports').toBeGreaterThan(0);
+  const results = reports.map(
+    (file) => JSON.parse(fs.readFileSync(path.join(directory, file), 'utf8')) as Result,
+  );
+  for (const report of results) {
+    console.log(
+      `${report.finalDisplayedUrl}: LCP ${report.audits['largest-contentful-paint'].displayValue}`,
+    );
+    const requests = report.audits['network-requests'].details;
+    if (requests?.type === 'table') {
+      console.table(
+        requests.items
+          .filter((item) => typeof item.url === 'string' && item.url.includes('/api/'))
+          .map((item) => ({
+            url: item.url,
+            startMs: item.networkRequestTime,
+            endMs: item.networkEndTime,
+            status: item.statusCode,
+          })),
+      );
+    }
+    expect(report.runtimeError, 'Lighthouse navigation must succeed').toBeUndefined();
+  }
+  console.log(
+    'Inspect .lighthouseci HTML/JSON and e2e/lighthouse/README.md. Reuse loaded user/config data; overlap independent reads without bypassing authorization.',
+  );
+  console.log((await run('assert')).stdout);
+  return results;
+}

e2e/lighthouse/librechat.yaml

@@ -0,0 +1,5 @@
+version: 1.3.6
+cache: true
+endpoints:
+  openAI:
+    titleConvo: false

e2e/lighthouse/lighthouserc.cjs

@@ -0,0 +1,24 @@
+module.exports = {
+  ci: {
+    collect: {
+      url: [process.env.LIGHTHOUSE_URL],
+      numberOfRuns: 3,
+      settings: {
+        preset: 'desktop',
+        throttlingMethod: 'provided',
+        onlyCategories: ['performance'],
+        extraHeaders: { Cookie: process.env.LIGHTHOUSE_COOKIE },
+      },
+    },
+    assert: {
+      assertions: {
+        'largest-contentful-paint': [
+          'error',
+          { maxNumericValue: 4500, aggregationMethod: 'median' },
+        ],
+        'cumulative-layout-shift': ['error', { maxNumericValue: 0.1, aggregationMethod: 'median' }],
+        'total-blocking-time': ['error', { maxNumericValue: 500, aggregationMethod: 'median' }],
+      },
+    },
+  },
+};

e2e/lighthouse/load.spec.ts

@@ -0,0 +1,42 @@
+import { expect, test } from '@playwright/test';
+import { seedConversations, seedMessages } from '../specs/mock/db';
+import { getE2EUser } from '../setup/user';
+import { auditPage } from './audit';
+
+const conversationId = '16390000-0000-4000-8000-000000000001';
+const title = 'Lighthouse latency audit';
+const transcriptMarker = 'Serial database latency transcript';
+
+test('serial database latency stays within web-vitals budgets', async ({ page, baseURL }) => {
+  const email = getE2EUser().email;
+  await seedConversations(email, [{ conversationId, title, updatedAt: new Date() }]);
+  await seedMessages(email, conversationId, [
+    {
+      messageId: '16390000-0000-4000-8000-000000000002',
+      parentMessageId: '00000000-0000-0000-0000-000000000000',
+      text: 'Explain why independent database reads should start together.',
+      isCreatedByUser: true,
+      sender: 'User',
+    },
+    {
+      messageId: '16390000-0000-4000-8000-000000000003',
+      parentMessageId: '16390000-0000-4000-8000-000000000002',
+      text: `${transcriptMarker}. Independent reads can overlap. Serial reads each add another database round trip and delay the visible conversation. Reuse loaded user data and preserve authorization checks when starting reads in parallel.`,
+      isCreatedByUser: false,
+      sender: 'Assistant',
+    },
+  ]);
+  const url = `${baseURL}/c/${conversationId}`;
+  await page.goto(url);
+  await expect(page.locator('.message-render').filter({ hasText: transcriptMarker })).toBeVisible();
+  const cookies = await page.context().cookies();
+  const reports = await auditPage({ url, cookies });
+  expect(reports, 'Lighthouse must produce all three reports').toHaveLength(3);
+  for (const report of reports) {
+    expect(report.finalDisplayedUrl, 'Do not measure a login redirect').toBe(url);
+    expect(
+      JSON.stringify(report.audits['largest-contentful-paint-element'].details),
+      'The measured LCP must be the seeded transcript, not the shell or a spinner',
+    ).toContain(transcriptMarker);
+  }
+});

e2e/lighthouse/regression.cjs

@@ -0,0 +1,11 @@
+const mongoose = require('mongoose');
+
+const originalExec = mongoose.Query.prototype.exec;
+mongoose.Query.prototype.exec = async function serialUserReads(...args) {
+  if (this.model.collection.name === 'messages' && this.op === 'find') {
+    for (let i = 0; i < 16; i++) {
+      await mongoose.model('User').findById(this.getFilter().user).lean().exec();
+    }
+  }
+  return originalExec.apply(this, args);
+};

e2e/playwright.config.lighthouse.ts

@@ -0,0 +1,40 @@
+import path from 'node:path';
+import { defineConfig } from '@playwright/test';
+import mockConfig from './playwright.config.mock';
+
+const latencyHook = path.resolve(__dirname, 'benchmarks/mongoose-latency-hook.cjs');
+const regressionHook = path.resolve(__dirname, 'lighthouse/regression.cjs');
+const servers = (Array.isArray(mockConfig.webServer) ? mockConfig.webServer : []).filter((server) =>
+  server.command.endsWith('start-server.js'),
+);
+if (servers.length !== 1) {
+  throw new Error('Lighthouse requires the isolated single-server harness (E2E_REPLICAS=1).');
+}
+
+export default defineConfig({
+  ...mockConfig,
+  testDir: 'lighthouse',
+  outputDir: 'lighthouse/.test-results',
+  timeout: 300_000,
+  retries: 0,
+  reporter: [['line']],
+  webServer: servers.map((server) => ({
+    ...server,
+    env: {
+      ...server.env,
+      CONFIG_PATH: path.resolve(__dirname, 'lighthouse/librechat.yaml'),
+      ENDPOINTS: 'openAI',
+      OPENAI_MODELS: 'gpt-4o-mini',
+      ASSISTANTS_API_KEY: '',
+      E2E_USE_MEMORY_MONGO: 'true',
+      E2E_LATENCY_MONGO_DELAY_MS: '250',
+      NODE_OPTIONS: [
+        server.env.NODE_OPTIONS,
+        `--require=${latencyHook}`,
+        ...(process.env.LIGHTHOUSE_REGRESSION === 'serial-reads'
+          ? [`--require=${regressionHook}`]
+          : []),
+      ].join(' '),
+    },
+  })),
+});

package.json

@@ -140,7 +140,10 @@
     "migrate:orphaned-agent-files:batch": "node config/migrate-orphaned-agent-files.js --batch-size=50",
     "migrate:code-file-duplicates:dry-run": "node config/migrate-code-file-duplicates.js --dry-run",
     "migrate:code-file-duplicates": "node config/migrate-code-file-duplicates.js",
-    "migrate:code-file-duplicates:batch": "node config/migrate-code-file-duplicates.js --batch-size=50"
+    "migrate:code-file-duplicates:batch": "node config/migrate-code-file-duplicates.js --batch-size=50",
+    "lighthouse": "npm run e2e:prepare && npm run lighthouse:run",
+    "lighthouse:run": "playwright test --config=e2e/playwright.config.lighthouse.ts",
+    "lighthouse:regression": "cross-env LIGHTHOUSE_REGRESSION=serial-reads npm run lighthouse:run"
   },
   "repository": {
     "type": "git",
@@ -158,6 +161,7 @@
     "@eslint/compat": "^1.2.6",
     "@eslint/eslintrc": "^3.3.4",
     "@eslint/js": "^9.20.0",
+    "@lhci/cli": "0.15.1",
     "@playwright/test": "^1.62.1",
     "@types/react-virtualized": "^9.22.0",
     "brace-expansion": "^2.1.2",
@@ -178,6 +182,7 @@
     "globals": "^15.14.0",
     "husky": "^9.1.7",
     "jest": "^30.2.0",
+    "lighthouse": "12.6.1",
     "lint-staged": "^15.4.3",
     "prettier": "^3.5.0",
     "prettier-plugin-tailwindcss": "^0.6.11",