diff --git a/e2e/README.md b/e2e/README.md index 89cfe53689..89bff8a2b9 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -139,6 +139,17 @@ handler in the middle of one throws `Route is already handled!` and fails the te first adds a route that matches nothing, which keeps the page intercepting: handlers already running answer their requests, and later requests reach the server. +## Harness servers and idle connections + +Playwright sends `route.fetch()`, the `request` fixture and `page.request` through one keep-alive +agent per worker that never closes an idle socket and ignores a server's `Keep-Alive: timeout=N` +hint. A Node server closes an idle socket after `keepAliveTimeout` plus `keepAliveTimeoutBuffer` +(6 s by default), so a request that reuses the socket at that moment fails with `read ECONNRESET` +or `socket hang up`. The harness therefore runs LibreChat with `HTTP_KEEP_ALIVE_TIMEOUT_MS=0`, and +every fixture server the mock profiles start as a `webServer` passes its server through `keepIdleConnections()` from +`e2e/setup/keepalive.js`. A new fixture server that specs reach from Node does the same. +`fixtures.spec.ts` checks that each harness server answers without an idle deadline. + ## Property-based browser testing Bombadil explores randomized sequences across the core chat loop, message branches, diff --git a/e2e/setup/env.ts b/e2e/setup/env.ts index dc584a400e..c2f8577ae9 100644 --- a/e2e/setup/env.ts +++ b/e2e/setup/env.ts @@ -138,6 +138,8 @@ export function getBaseE2EEnv(): Record { SESSION_EXPIRY: process.env.SESSION_EXPIRY ?? '3600000', ALLOW_REGISTRATION: 'true', REFRESH_TOKEN_EXPIRY: process.env.REFRESH_TOKEN_EXPIRY ?? '3600000', + /** Playwright's Node-side requests keep idle sockets indefinitely; see `keepalive.js`. */ + HTTP_KEEP_ALIVE_TIMEOUT_MS: '0', ...getStreamStoreEnv(), }; } diff --git a/e2e/setup/fake-assistants-server.js b/e2e/setup/fake-assistants-server.js index 251a8f4d5d..9f83efbcc8 100644 --- a/e2e/setup/fake-assistants-server.js +++ b/e2e/setup/fake-assistants-server.js @@ -8,6 +8,7 @@ */ const http = require('http'); const { randomUUID } = require('crypto'); +const { keepIdleConnections } = require('./keepalive'); const PORT = Number(process.env.E2E_ASSISTANTS_PORT) || 8890; const DEFAULT_REPLY = process.env.E2E_ASSISTANTS_REPLY || 'E2E mock assistant reply: pong'; @@ -630,6 +631,7 @@ const server = http.createServer(async (req, res) => { } }); +keepIdleConnections(server); server.listen(PORT, '127.0.0.1', () => { console.log(`[e2e] fake assistants server listening on http://127.0.0.1:${PORT}`); }); diff --git a/e2e/setup/fake-code-server.js b/e2e/setup/fake-code-server.js index 29047c93d6..a5db3bface 100644 --- a/e2e/setup/fake-code-server.js +++ b/e2e/setup/fake-code-server.js @@ -22,6 +22,7 @@ const http = require('http'); const busboy = require('busboy'); const { randomUUID } = require('crypto'); +const { keepIdleConnections } = require('./keepalive'); const PORT = parseInt(process.env.E2E_CODE_API_PORT || '8790', 10); const HOST = '127.0.0.1'; @@ -344,6 +345,7 @@ const server = http.createServer((req, res) => { }); }); +keepIdleConnections(server); server.listen(PORT, HOST, () => { console.log(`[e2e] fake code API listening on http://${HOST}:${PORT}/v1`); }); diff --git a/e2e/setup/fake-flux-server.js b/e2e/setup/fake-flux-server.js index 8639cbb66a..ed59a23372 100644 --- a/e2e/setup/fake-flux-server.js +++ b/e2e/setup/fake-flux-server.js @@ -14,6 +14,7 @@ */ const http = require('http'); const sharp = require('sharp'); +const { keepIdleConnections } = require('./keepalive'); const PORT = Number(process.env.E2E_FLUX_PORT) || 8794; const ORIGIN = `http://127.0.0.1:${PORT}`; @@ -149,6 +150,8 @@ const server = http.createServer(async (req, res) => { return send(res, 404, { detail: 'Not found' }); }); +keepIdleConnections(server); + Promise.all([createSample(2048), createSample(1200)]) .then(([large, boundary]) => { const limit = 5 * 1024 * 1024; diff --git a/e2e/setup/fake-git-host-server.js b/e2e/setup/fake-git-host-server.js index 6b97666a1e..911d0d718f 100644 --- a/e2e/setup/fake-git-host-server.js +++ b/e2e/setup/fake-git-host-server.js @@ -12,6 +12,7 @@ */ const fs = require('fs'); const https = require('https'); +const { keepIdleConnections } = require('./keepalive'); const PORT = Number(process.env.E2E_GIT_HOST_PORT) || 8793; const HOST = `localhost:${PORT}`; @@ -120,6 +121,8 @@ const server = https.createServer( }, ); +keepIdleConnections(server); + /** Every local address, since `localhost` may resolve to either family. */ server.listen(PORT, () => { console.log(`[fake-git-host] listening on https://${HOST}`); diff --git a/e2e/setup/fake-label-server.js b/e2e/setup/fake-label-server.js index 1c08013b98..6b2fb79b9c 100644 --- a/e2e/setup/fake-label-server.js +++ b/e2e/setup/fake-label-server.js @@ -15,6 +15,7 @@ * reached the model) rather than just that some text rendered. */ const http = require('http'); +const { keepIdleConnections } = require('./keepalive'); const PORT = Number(process.env.E2E_LABEL_PORT) || 8889; const PHASE_PROMPT_MARKER = 'Summarize what this phase of an agent run accomplished'; @@ -209,6 +210,7 @@ const server = http.createServer(async (req, res) => { sendJson(res, 404, { error: { message: `Unhandled ${req.method} ${url.pathname}` } }); }); +keepIdleConnections(server); server.listen(PORT, '127.0.0.1', () => { console.log(`[e2e] fake label server listening on http://127.0.0.1:${PORT}`); }); diff --git a/e2e/setup/fake-mcp-app-server.mjs b/e2e/setup/fake-mcp-app-server.mjs index ff17bbd10d..f5fa4f87a6 100644 --- a/e2e/setup/fake-mcp-app-server.mjs +++ b/e2e/setup/fake-mcp-app-server.mjs @@ -13,6 +13,7 @@ import { registerAppTool, } from '@modelcontextprotocol/ext-apps/server'; import { z } from 'zod/v4'; +import { keepIdleConnections } from './keepalive.js'; const require = createRequire(import.meta.url); const HOST = '127.0.0.1'; @@ -427,6 +428,7 @@ const httpServer = http.createServer(async (req, res) => { } }); +keepIdleConnections(httpServer); httpServer.listen(PORT, HOST, () => { console.log(`[e2e] MCP App fixture listening on ${ORIGIN}/mcp`); }); diff --git a/e2e/setup/fake-mcp-dynamic-network-server.js b/e2e/setup/fake-mcp-dynamic-network-server.js index 381fa91c09..90e9cb1d93 100644 --- a/e2e/setup/fake-mcp-dynamic-network-server.js +++ b/e2e/setup/fake-mcp-dynamic-network-server.js @@ -8,6 +8,7 @@ const { StreamableHTTPServerTransport, } = require('@modelcontextprotocol/sdk/server/streamableHttp.js'); const { watchDynamicTool } = require('./dynamic-mcp-tools'); +const { keepIdleConnections } = require('./keepalive'); const PORT = Number.parseInt(process.env.E2E_MCP_DYNAMIC_PORT || '8766', 10); const HOST = '127.0.0.1'; @@ -120,6 +121,7 @@ async function shutdown() { process.once('SIGINT', shutdown); process.once('SIGTERM', shutdown); +keepIdleConnections(httpServer); httpServer.listen(PORT, HOST, () => { console.log(`[e2e] dynamic MCP server listening on http://${HOST}:${PORT}`); }); diff --git a/e2e/setup/fake-mcp-http-server.js b/e2e/setup/fake-mcp-http-server.js index 503e82d9e2..93003e1718 100644 --- a/e2e/setup/fake-mcp-http-server.js +++ b/e2e/setup/fake-mcp-http-server.js @@ -15,6 +15,7 @@ const http = require('http'); const { randomUUID } = require('crypto'); +const { keepIdleConnections } = require('./keepalive'); const { McpServer } = require('@modelcontextprotocol/sdk/server/mcp.js'); const { StreamableHTTPServerTransport, @@ -73,6 +74,7 @@ const httpServer = http.createServer(async (req, res) => { } }); +keepIdleConnections(httpServer); httpServer.listen(PORT, HOST, () => { console.log(`[e2e] fake HTTP MCP server listening on http://${HOST}:${PORT}/mcp`); }); diff --git a/e2e/setup/fake-mcp-oauth-server.js b/e2e/setup/fake-mcp-oauth-server.js index a9f1a93772..adeb6f85e2 100644 --- a/e2e/setup/fake-mcp-oauth-server.js +++ b/e2e/setup/fake-mcp-oauth-server.js @@ -7,6 +7,7 @@ */ const http = require('node:http'); +const { keepIdleConnections } = require('./keepalive'); const PORT = Number(process.env.E2E_MCP_OAUTH_PORT || 8767); const HOST = '127.0.0.1'; @@ -45,6 +46,7 @@ const server = http.createServer((req, res) => { res.end(); }); +keepIdleConnections(server); server.listen(PORT, HOST, () => { console.log(`[e2e] fake OAuth MCP resource listening on http://${HOST}:${PORT}/mcp`); }); diff --git a/e2e/setup/fake-rag-server.js b/e2e/setup/fake-rag-server.js index 47af262056..69342c3bc1 100644 --- a/e2e/setup/fake-rag-server.js +++ b/e2e/setup/fake-rag-server.js @@ -27,6 +27,7 @@ const http = require('http'); const busboy = require('busboy'); +const { keepIdleConnections } = require('./keepalive'); const PORT = parseInt(process.env.E2E_RAG_API_PORT || '8791', 10); /** Mirrors ENTITY_EMBEDDINGS_PROBE_K in packages/api/src/files/rag.ts; file_search asks for 5. */ @@ -212,6 +213,7 @@ const server = http.createServer((req, res) => { }); }); +keepIdleConnections(server); server.listen(PORT, HOST, () => { console.log(`[e2e] fake RAG API listening on http://${HOST}:${PORT}`); }); diff --git a/e2e/setup/keepalive.js b/e2e/setup/keepalive.js new file mode 100644 index 0000000000..a3a17cfd15 --- /dev/null +++ b/e2e/setup/keepalive.js @@ -0,0 +1,22 @@ +/** + * Leaves idle keep-alive connections open until the client closes them. + * + * Playwright sends its Node-side requests (the `request` fixture, `page.request`, `route.fetch`) + * through one process-wide keep-alive agent with no idle timeout. Node's agent honors a server's + * `Keep-Alive: timeout=N` hint only by shortening a timeout of its own, so this one keeps an idle + * socket indefinitely. A Node server, by default, closes an idle socket once `keepAliveTimeout` + * plus `keepAliveTimeoutBuffer` (6 s) pass; a request that reuses the socket at that moment, or + * reaches a server too busy to read it before the deadline fires, is reset (`read ECONNRESET`) or + * dropped (`socket hang up`). Every server the suite drives from Playwright's Node side therefore + * disables its idle deadline; the backend gets the same through `HTTP_KEEP_ALIVE_TIMEOUT_MS`. + * + * @template {import('node:http').Server} T + * @param {T} server + * @returns {T} + */ +function keepIdleConnections(server) { + server.keepAliveTimeout = 0; + return server; +} + +module.exports = { keepIdleConnections }; diff --git a/e2e/setup/start-server-cluster.js b/e2e/setup/start-server-cluster.js index 4cf5bf9ece..8d6a88283b 100644 --- a/e2e/setup/start-server-cluster.js +++ b/e2e/setup/start-server-cluster.js @@ -3,6 +3,7 @@ const fs = require('fs'); const http = require('http'); const path = require('path'); const { MongoMemoryServer } = require('mongodb-memory-server'); +const { keepIdleConnections } = require('./keepalive'); const DEFAULT_BASE_URL = 'http://localhost:3080'; const DEFAULT_RUNTIME_ENV_PATH = path.resolve(__dirname, '../specs/.test-results/runtime-env.json'); @@ -102,6 +103,7 @@ function startProxy(baseURL, targetPort) { }); request.pipe(upstream); }); + keepIdleConnections(proxyServer); proxyServer.listen(Number(baseURL.port || 80), baseURL.hostname, () => { console.log( `[e2e] Replica proxy listening at ${baseURL.origin}; primary target is ${targetPort}`, diff --git a/e2e/specs/mock/fixtures.spec.ts b/e2e/specs/mock/fixtures.spec.ts index a66c6d74d9..7e5d0c597d 100644 --- a/e2e/specs/mock/fixtures.spec.ts +++ b/e2e/specs/mock/fixtures.spec.ts @@ -105,3 +105,35 @@ test.describe('route removal', () => { expect(await readConfig(page)).not.toHaveProperty('answeredBy'); }); }); + +/** The servers every mock profile starts, each at the route its readiness check probes. */ +const port = (name: string, fallback: string) => process.env[name] || fallback; +const HARNESS_SERVERS = [ + CONFIG_PATH, + `http://127.0.0.1:${port('E2E_LABEL_PORT', '8889')}/`, + `http://127.0.0.1:${port('E2E_ASSISTANTS_PORT', '8890')}/`, + `http://127.0.0.1:${port('E2E_FLUX_PORT', '8794')}/`, + `http://127.0.0.1:${port('E2E_CODE_API_PORT', '8790')}/health`, + `http://127.0.0.1:${port('E2E_RAG_API_PORT', '8791')}/health`, + `http://127.0.0.1:${port('E2E_MCP_HTTP_PORT', '8765')}/`, + `http://127.0.0.1:${port('E2E_MCP_OAUTH_PORT', '8767')}/`, + `https://localhost:${port('E2E_GIT_HOST_PORT', '8793')}/`, +]; + +test.describe('harness connections', () => { + /** + * Playwright's Node-side requests reuse idle sockets for as long as the worker lives, so a + * server that announces an idle deadline (`Keep-Alive: timeout=N`) and closes on it resets + * whichever request next reuses the socket at that moment (see `e2e/setup/keepalive.js`). + */ + test('every harness server keeps idle connections open for the client to close', async ({ + request, + }) => { + for (const url of HARNESS_SERVERS) { + const response = await request.get(url, { ignoreHTTPSErrors: true }); + const headers = response.headers(); + expect(headers.connection, url).toBe('keep-alive'); + expect(headers['keep-alive'], url).toBeUndefined(); + } + }); +});