From a1bfa3b298b877901cf4cf8075499ebc5e92da53 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 4 Jun 2026 08:33:28 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=AD=20test:=20Run=20Mock=20E2E=20Suite?= =?UTF-8?q?=20Through=20`createRun`=20With=20In-Process=20Fake=20Model=20(?= =?UTF-8?q?#13508)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🎭 test: Run Mock E2E Suite Through createRun With In-Process Fake Model Replace the standalone HTTP mock LLM server with an in-process fake model injected into the real createRun -> Run.create pipeline via run.Graph.overrideTestModel, so the mock suite exercises the agents integration end-to-end without a live provider or a separate server. - Bump @librechat/agents to 3.2.2 for the FakeChatModel/createFakeStreamingLLM exports - Add an env-gated applyTestRunHook seam in packages/api createRun (no /api changes) - Add e2e/setup/fake-model.js to drive default replies + the skill-authoring tool-call flow - Drop the mock-llm webServer from playwright.config.mock.ts and set LIBRECHAT_TEST_RUN_HOOK * 🧹 test: Retire Standalone Mock LLM Server From E2E Recorder Migrate the `--profile=mock` recorder onto the same in-process fake model as the Playwright mock suite, then delete the now-unused HTTP mock server so the fake-LLM logic lives in a single place. - Point record.js mock profile at the fake model via LIBRECHAT_TEST_RUN_HOOK - Remove the mock-llm-server spawn/wait and MOCK_LLM_PORT plumbing from record.js - Delete e2e/setup/mock-llm-server.js (e2e/setup/fake-model.js is now the only source) - Update e2e/README.md to describe the in-process fake LLM * 🏷️ ci: Rename Playwright Mock E2E Check to Playwright E2E Tests --- .github/workflows/playwright-mock.yml | 3 +- api/package.json | 2 +- e2e/README.md | 6 +- e2e/config/librechat.e2e.yaml | 5 +- e2e/playwright.config.mock.ts | 36 +-- e2e/setup/fake-model.js | 213 ++++++++++++++ e2e/setup/mock-llm-server.js | 392 -------------------------- e2e/setup/record.js | 37 +-- package-lock.json | 10 +- packages/api/package.json | 2 +- packages/api/src/agents/index.ts | 1 + packages/api/src/agents/run.ts | 6 +- packages/api/src/agents/testHook.ts | 40 +++ 13 files changed, 286 insertions(+), 467 deletions(-) create mode 100644 e2e/setup/fake-model.js delete mode 100644 e2e/setup/mock-llm-server.js create mode 100644 packages/api/src/agents/testHook.ts diff --git a/.github/workflows/playwright-mock.yml b/.github/workflows/playwright-mock.yml index 91a2f9910b..f0d0d1323d 100644 --- a/.github/workflows/playwright-mock.yml +++ b/.github/workflows/playwright-mock.yml @@ -1,4 +1,4 @@ -name: Playwright E2E (Mock LLM) +name: Playwright E2E Tests on: pull_request: @@ -22,7 +22,6 @@ env: jobs: e2e: - name: Tier-1 smoke (headless Chrome) runs-on: ubuntu-latest timeout-minutes: 30 env: diff --git a/api/package.json b/api/package.json index 9c4da2df2e..652cec9ccd 100644 --- a/api/package.json +++ b/api/package.json @@ -46,7 +46,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.0.1", "@keyv/redis": "^4.3.3", - "@librechat/agents": "^3.2.1", + "@librechat/agents": "^3.2.2", "@librechat/api": "*", "@librechat/data-schemas": "*", "@microsoft/microsoft-graph-client": "^3.0.7", diff --git a/e2e/README.md b/e2e/README.md index 822297103f..86ec912447 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -1,6 +1,6 @@ # LibreChat e2e -The mock e2e profile is the safest default for generated tests. It starts LibreChat with `e2e/config/librechat.e2e.yaml`, points custom endpoints at the local mock LLM server, creates an authenticated e2e user, and avoids real provider credentials. +The mock e2e profile is the safest default for generated tests. It starts LibreChat with `e2e/config/librechat.e2e.yaml`, injects an in-process fake LLM (via `LIBRECHAT_TEST_RUN_HOOK`), creates an authenticated e2e user, and avoids real provider credentials. ## Recording Tests @@ -10,9 +10,9 @@ Use Playwright codegen when you want to turn an exploratory browser session into npm run e2e:record ``` -That command builds the app, starts the mock LLM and LibreChat test server when needed, writes `e2e/storageState.json`, and opens Playwright codegen at `/c/new`. The npm script uses `http://localhost:3333` so it does not collide with a normal dev server on `3080`. Raw recordings are written to `e2e/recordings/` and ignored by git. +That command builds the app, starts the LibreChat test server (with an in-process fake LLM) when needed, writes `e2e/storageState.json`, and opens Playwright codegen at `/c/new`. The npm script uses `http://localhost:3333` so it does not collide with a normal dev server on `3080`. Raw recordings are written to `e2e/recordings/` and ignored by git. -For a real local LibreChat config instead of the mock LLM profile: +For a real local LibreChat config instead of the mock profile: ```sh npm run e2e:record:local diff --git a/e2e/config/librechat.e2e.yaml b/e2e/config/librechat.e2e.yaml index e7e9e62271..8f73e53e8c 100644 --- a/e2e/config/librechat.e2e.yaml +++ b/e2e/config/librechat.e2e.yaml @@ -1,5 +1,6 @@ -# Credential-free e2e config template. e2e/playwright.config.mock.ts writes -# an ignored runtime copy and rewrites the mock LLM port from MOCK_LLM_PORT. +# Credential-free e2e config template. e2e/playwright.config.mock.ts copies this +# to an ignored runtime path; the in-process fake model (e2e/setup/fake-model.js) +# overrides each run, so the placeholder baseURL below is never contacted. version: 1.3.11 cache: true diff --git a/e2e/playwright.config.mock.ts b/e2e/playwright.config.mock.ts index a78d51817c..08284efeed 100644 --- a/e2e/playwright.config.mock.ts +++ b/e2e/playwright.config.mock.ts @@ -5,15 +5,12 @@ import { getLocalE2EEnv, getE2EBaseURL } from './setup/env'; const rootPath = path.resolve(__dirname, '..'); const serverPath = path.resolve(rootPath, 'e2e/setup/start-server.js'); -const mockLlmPath = path.resolve(rootPath, 'e2e/setup/mock-llm-server.js'); +const fakeModelHookPath = path.resolve(rootPath, 'e2e/setup/fake-model.js'); const configTemplatePath = path.resolve(rootPath, 'e2e/config/librechat.e2e.yaml'); const configPath = path.resolve(rootPath, 'e2e/.generated/librechat.e2e.yaml'); const reportPath = path.resolve(rootPath, 'e2e/playwright-report'); const baseURL = getE2EBaseURL(); -const mockLlmPort = getMockLlmPort(); -const defaultMockLlmBaseURL = 'http://127.0.0.1:8889/v1'; -const mockLlmBaseURL = `http://127.0.0.1:${mockLlmPort}/v1`; const chromiumChannel = process.env.E2E_CHROMIUM_CHANNEL || undefined; const vanillaOverrides = { @@ -30,7 +27,8 @@ const vanillaOverrides = { const baseEnv = { ...getLocalE2EEnv(), CONFIG_PATH: configPath, - MOCK_LLM_PORT: mockLlmPort, + /** Loaded in-process by `@librechat/api`'s `createRun` to swap in a fake model. */ + LIBRECHAT_TEST_RUN_HOOK: fakeModelHookPath, ...vanillaOverrides, }; @@ -41,23 +39,15 @@ const preservedCredentialEnvKeys = new Set([ 'E2E_USER_B_PASSWORD', ]); -function getMockLlmPort() { - const port = process.env.MOCK_LLM_PORT ?? '8889'; - if (!/^\d+$/.test(port)) { - throw new Error('MOCK_LLM_PORT must be a numeric port'); - } - return port; -} - +/** + * The custom endpoints in the template point at an unreachable baseURL; the fake + * model injected via `LIBRECHAT_TEST_RUN_HOOK` overrides the run before any + * request is made, so no real (or mock HTTP) provider is contacted. + */ function writeRuntimeMockConfig() { const template = fs.readFileSync(configTemplatePath, 'utf8'); - - if (!template.includes(defaultMockLlmBaseURL)) { - throw new Error(`Expected mock config template to include ${defaultMockLlmBaseURL}`); - } - fs.mkdirSync(path.dirname(configPath), { recursive: true }); - fs.writeFileSync(configPath, template.replaceAll(defaultMockLlmBaseURL, mockLlmBaseURL)); + fs.writeFileSync(configPath, template); } function neutralizeCredentialEnv(env: NodeJS.ProcessEnv, keep: Set) { @@ -128,14 +118,6 @@ export default defineConfig({ }, ], webServer: [ - { - command: `node ${mockLlmPath}`, - cwd: rootPath, - url: `http://127.0.0.1:${mockLlmPort}/health`, - stdout: 'pipe', - timeout: 30_000, - reuseExistingServer: false, - }, { command: `node ${serverPath}`, cwd: rootPath, diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.js new file mode 100644 index 0000000000..04a2f9b47b --- /dev/null +++ b/e2e/setup/fake-model.js @@ -0,0 +1,213 @@ +/** + * In-process fake LLM for credential-free e2e tests. Loaded by `@librechat/api`'s + * `createRun` via the `LIBRECHAT_TEST_RUN_HOOK` env var (set by the mock + * Playwright config and the `--profile=mock` recorder), it swaps the run's model + * for the agents package's own `FakeChatModel` through + * `run.Graph.overrideTestModel(...)`. + * + * This exercises the real `Run.create` -> graph -> tool-node pipeline end to end + * without a live provider or a standalone HTTP mock server: responses are decided + * from the conversation and the agents' advertised tools. + */ +const MOCK_REPLY = process.env.MOCK_LLM_REPLY || 'E2E mock reply: pong'; +const CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_CHUNK_DELAY_MS) || 10; + +const CREATE_SKILL_MARKER = 'E2E_CREATE_SKILL:'; +const EDIT_SKILL_MARKER = 'E2E_EDIT_SKILL:'; +const CREATE_FILE_AUTHORING_FINAL_TEXT = 'E2E file authoring complete'; +const EDIT_FILE_AUTHORING_FINAL_TEXT = 'E2E file edit complete'; +const CREATE_FILE_TOOL_NAME = 'create_file'; +const EDIT_FILE_TOOL_NAME = 'edit_file'; +const BASH_TOOL_NAME = 'bash_tool'; +const CREATE_SKILL_TOOL_CALL_ID = 'call_e2e_create_skill'; +const EDIT_SKILL_TOOL_CALL_ID = 'call_e2e_edit_skill'; +const SKILL_DESCRIPTION = + 'Use this skill to verify LibreChat skill file authoring in mock end-to-end tests.'; +const EDITED_SKILL_DESCRIPTION = + 'Use this edited skill to verify LibreChat skill file authoring in mock end-to-end tests.'; + +function messageType(message) { + if (typeof message.getType === 'function') { + return message.getType(); + } + if (typeof message._getType === 'function') { + return message._getType(); + } + return message.role || message.type || ''; +} + +function getContentText(content) { + if (typeof content === 'string') { + return content; + } + if (!Array.isArray(content)) { + return ''; + } + return content + .map((part) => { + if (typeof part === 'string') { + return part; + } + if (part && typeof part === 'object' && typeof part.text === 'string') { + return part.text; + } + return ''; + }) + .join('\n'); +} + +function getLatestUserText(messages) { + if (!Array.isArray(messages)) { + return ''; + } + for (let index = messages.length - 1; index >= 0; index--) { + const message = messages[index]; + if (!message) { + continue; + } + const type = messageType(message); + if (type === 'human' || type === 'user') { + return getContentText(message.content); + } + } + return ''; +} + +function getRequestedSkillName(text, marker) { + const markerIndex = text.indexOf(marker); + if (markerIndex === -1) { + return ''; + } + const afterMarker = text.slice(markerIndex + marker.length); + return afterMarker.match(/[a-z0-9][a-z0-9-]*/)?.[0] ?? ''; +} + +function collectToolNames(agents) { + const names = new Set(); + const add = (name) => { + if (typeof name === 'string' && name) { + names.add(name); + } + }; + for (const agent of agents ?? []) { + if (!agent) { + continue; + } + for (const tool of agent.tools ?? []) { + add(tool?.name); + } + for (const def of agent.toolDefinitions ?? []) { + add(def?.name); + } + if (agent.toolRegistry && typeof agent.toolRegistry.keys === 'function') { + for (const name of agent.toolRegistry.keys()) { + add(name); + } + } + } + return names; +} + +function buildSkillBody(skillName) { + return `--- +name: ${skillName} +description: ${SKILL_DESCRIPTION} +--- + +# ${skillName} + +Created by the Playwright mock e2e suite to verify host file authoring without code execution.`; +} + +function buildCreateSkillArgs(skillName) { + return { + file_path: `skills/${skillName}/SKILL.md`, + content: buildSkillBody(skillName), + overwrite: false, + }; +} + +function buildEditSkillArgs(skillName) { + return { + file_path: `skills/${skillName}/SKILL.md`, + old_text: `description: ${SKILL_DESCRIPTION}`, + new_text: `description: ${EDITED_SKILL_DESCRIPTION}`, + }; +} + +/** + * Pick the fake-model script for a skill file-authoring turn. The graph runs two + * model turns: turn 1 streams the (empty) preamble and emits the tool call, the + * tool node writes the SKILL.md, then turn 2 streams the final text. The guards + * assert the feature advertised the host file-authoring tool and did NOT enable + * code execution. + */ +function fileAuthoringResponses(operation, toolNames) { + if (!toolNames.has(operation.toolName)) { + return { + responses: [`E2E file authoring unavailable: ${operation.toolName} was not advertised.`], + }; + } + if (toolNames.has(BASH_TOOL_NAME)) { + return { + responses: [`E2E file authoring unavailable: ${BASH_TOOL_NAME} was unexpectedly advertised.`], + }; + } + return { + responses: ['', `${operation.finalText}: ${operation.skillName}`], + toolCalls: [ + { + id: operation.toolCallId, + name: operation.toolName, + args: operation.args, + type: 'tool_call', + }, + ], + }; +} + +function resolveResponses(text, toolNames) { + const createSkillName = getRequestedSkillName(text, CREATE_SKILL_MARKER); + if (createSkillName) { + return fileAuthoringResponses( + { + skillName: createSkillName, + toolName: CREATE_FILE_TOOL_NAME, + toolCallId: CREATE_SKILL_TOOL_CALL_ID, + finalText: CREATE_FILE_AUTHORING_FINAL_TEXT, + args: buildCreateSkillArgs(createSkillName), + }, + toolNames, + ); + } + + const editSkillName = getRequestedSkillName(text, EDIT_SKILL_MARKER); + if (editSkillName) { + return fileAuthoringResponses( + { + skillName: editSkillName, + toolName: EDIT_FILE_TOOL_NAME, + toolCallId: EDIT_SKILL_TOOL_CALL_ID, + finalText: EDIT_FILE_AUTHORING_FINAL_TEXT, + args: buildEditSkillArgs(editSkillName), + }, + toolNames, + ); + } + + return { responses: [MOCK_REPLY] }; +} + +/** @type {import('@librechat/api').TestRunHook} */ +module.exports = function fakeModelHook(run, context) { + const graph = run?.Graph; + if (!graph || typeof graph.overrideTestModel !== 'function') { + console.warn('[e2e] fake-model hook: run.Graph.overrideTestModel unavailable'); + return; + } + + const text = getLatestUserText(context?.messages); + const toolNames = collectToolNames(context?.agents); + const { responses, toolCalls } = resolveResponses(text, toolNames); + graph.overrideTestModel(responses, CHUNK_DELAY_MS, toolCalls); +}; diff --git a/e2e/setup/mock-llm-server.js b/e2e/setup/mock-llm-server.js deleted file mode 100644 index 4f34190e9c..0000000000 --- a/e2e/setup/mock-llm-server.js +++ /dev/null @@ -1,392 +0,0 @@ -/** - * OpenAI-compatible mock server for credential-free e2e tests. Answers - * `${baseURL}/chat/completions` with deterministic content. Run standalone - * (Playwright `webServer`) or import `startMockLlm()` for programmatic control. - */ -const http = require('http'); - -const DEFAULT_PORT = 8889; -const MOCK_REPLY = process.env.MOCK_LLM_REPLY || 'E2E mock reply: pong'; -const MODEL_FALLBACK = 'mock-model'; -const STREAM_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_CHUNK_DELAY_MS) || 60; -const CREATE_SKILL_MARKER = 'E2E_CREATE_SKILL:'; -const EDIT_SKILL_MARKER = 'E2E_EDIT_SKILL:'; -const CREATE_FILE_AUTHORING_FINAL_TEXT = 'E2E file authoring complete'; -const EDIT_FILE_AUTHORING_FINAL_TEXT = 'E2E file edit complete'; -const CREATE_FILE_TOOL_NAME = 'create_file'; -const EDIT_FILE_TOOL_NAME = 'edit_file'; -const BASH_TOOL_NAME = 'bash_tool'; -const CREATE_SKILL_TOOL_CALL_ID = 'call_e2e_create_skill'; -const EDIT_SKILL_TOOL_CALL_ID = 'call_e2e_edit_skill'; -const SKILL_DESCRIPTION = - 'Use this skill to verify LibreChat skill file authoring in mock end-to-end tests.'; -const EDITED_SKILL_DESCRIPTION = - 'Use this edited skill to verify LibreChat skill file authoring in mock end-to-end tests.'; - -const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); - -function readJsonBody(req) { - return new Promise((resolve) => { - let raw = ''; - req.on('data', (chunk) => { - raw += chunk; - }); - req.on('end', () => { - try { - resolve(raw ? JSON.parse(raw) : {}); - } catch { - resolve({}); - } - }); - }); -} - -function toChunks(text) { - return text.match(/\S+\s*/g) || [text]; -} - -function getToolName(tool) { - if (!tool || typeof tool !== 'object') { - return ''; - } - if (typeof tool.name === 'string') { - return tool.name; - } - if (tool.function && typeof tool.function.name === 'string') { - return tool.function.name; - } - return ''; -} - -function hasTool(body, name) { - const tools = Array.isArray(body.tools) ? body.tools : []; - const functions = Array.isArray(body.functions) ? body.functions : []; - return [...tools, ...functions].some((tool) => getToolName(tool) === name); -} - -function getContentText(content) { - if (typeof content === 'string') { - return content; - } - if (!Array.isArray(content)) { - return ''; - } - return content - .map((part) => { - if (typeof part === 'string') { - return part; - } - if (part && typeof part === 'object' && typeof part.text === 'string') { - return part.text; - } - return ''; - }) - .join('\n'); -} - -function getLatestUserText(body) { - const messages = Array.isArray(body.messages) ? body.messages : []; - for (let index = messages.length - 1; index >= 0; index--) { - const message = messages[index]; - if (!message || message.role !== 'user') { - continue; - } - return getContentText(message.content); - } - return ''; -} - -function getRequestedSkillName(body, marker) { - const text = getLatestUserText(body); - const markerIndex = text.indexOf(marker); - if (markerIndex === -1) { - return ''; - } - const afterMarker = text.slice(markerIndex + marker.length); - const skillName = afterMarker.match(/[a-z0-9][a-z0-9-]*/)?.[0]; - if (skillName) { - return skillName; - } - return ''; -} - -function hasFileAuthoringToolResult(body, toolCallId) { - const messages = Array.isArray(body.messages) ? body.messages : []; - return messages.some( - (message) => message && message.role === 'tool' && message.tool_call_id === toolCallId, - ); -} - -function buildSkillBody(skillName) { - return `--- -name: ${skillName} -description: ${SKILL_DESCRIPTION} ---- - -# ${skillName} - -Created by the Playwright mock e2e suite to verify host file authoring without code execution.`; -} - -function buildCreateSkillArgs(skillName) { - return { - file_path: `skills/${skillName}/SKILL.md`, - content: buildSkillBody(skillName), - overwrite: false, - }; -} - -function buildEditSkillArgs(skillName) { - return { - file_path: `skills/${skillName}/SKILL.md`, - old_text: `description: ${SKILL_DESCRIPTION}`, - new_text: `description: ${EDITED_SKILL_DESCRIPTION}`, - }; -} - -async function streamTextCompletion(res, model, text = MOCK_REPLY) { - res.writeHead(200, { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - }); - if (typeof res.flushHeaders === 'function') { - res.flushHeaders(); - } - - const id = 'chatcmpl-e2e-mock'; - const created = 1700000000; - const base = { id, object: 'chat.completion.chunk', created, model }; - - const send = (delta, finishReason = null) => { - const payload = { - ...base, - choices: [{ index: 0, delta, finish_reason: finishReason }], - }; - res.write(`data: ${JSON.stringify(payload)}\n\n`); - }; - - send({ role: 'assistant', content: '' }); - for (const chunk of toChunks(text)) { - await delay(STREAM_CHUNK_DELAY_MS); - send({ content: chunk }); - } - send({}, 'stop'); - res.write('data: [DONE]\n\n'); - res.end(); -} - -async function streamToolCall(res, model, { toolName, toolCallId, args }) { - res.writeHead(200, { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - }); - if (typeof res.flushHeaders === 'function') { - res.flushHeaders(); - } - - const id = 'chatcmpl-e2e-mock'; - const created = 1700000000; - const base = { id, object: 'chat.completion.chunk', created, model }; - const send = (delta, finishReason = null) => { - const payload = { - ...base, - choices: [{ index: 0, delta, finish_reason: finishReason }], - }; - res.write(`data: ${JSON.stringify(payload)}\n\n`); - }; - - send({ role: 'assistant', content: '' }); - await delay(STREAM_CHUNK_DELAY_MS); - send({ - tool_calls: [ - { - index: 0, - id: toolCallId, - type: 'function', - function: { name: toolName, arguments: '' }, - }, - ], - }); - await delay(STREAM_CHUNK_DELAY_MS); - send({ - tool_calls: [ - { - index: 0, - function: { arguments: JSON.stringify(args) }, - }, - ], - }); - send({}, 'tool_calls'); - res.write('data: [DONE]\n\n'); - res.end(); -} - -function jsonCompletion(res, model, content = MOCK_REPLY) { - const payload = { - id: 'chatcmpl-e2e-mock', - object: 'chat.completion', - created: 1700000000, - model, - choices: [ - { - index: 0, - message: { role: 'assistant', content }, - finish_reason: 'stop', - }, - ], - usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, - }; - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify(payload)); -} - -function jsonToolCall(res, model, { toolName, toolCallId, args }) { - const payload = { - id: 'chatcmpl-e2e-mock', - object: 'chat.completion', - created: 1700000000, - model, - choices: [ - { - index: 0, - message: { - role: 'assistant', - content: null, - tool_calls: [ - { - id: toolCallId, - type: 'function', - function: { - name: toolName, - arguments: JSON.stringify(args), - }, - }, - ], - }, - finish_reason: 'tool_calls', - }, - ], - usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, - }; - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify(payload)); -} - -async function handleFileAuthoringOperation(body, res, model, operation) { - const finalText = `${operation.finalText}: ${operation.skillName}`; - if (hasFileAuthoringToolResult(body, operation.toolCallId)) { - if (body.stream) { - await streamTextCompletion(res, model, finalText); - } else { - jsonCompletion(res, model, finalText); - } - return true; - } - - if (!hasTool(body, operation.toolName)) { - const unavailable = `E2E file authoring unavailable: ${operation.toolName} was not advertised.`; - if (body.stream) { - await streamTextCompletion(res, model, unavailable); - } else { - jsonCompletion(res, model, unavailable); - } - return true; - } - - if (hasTool(body, BASH_TOOL_NAME)) { - const unexpected = `E2E file authoring unavailable: ${BASH_TOOL_NAME} was unexpectedly advertised.`; - if (body.stream) { - await streamTextCompletion(res, model, unexpected); - } else { - jsonCompletion(res, model, unexpected); - } - return true; - } - - const toolCall = { - toolName: operation.toolName, - toolCallId: operation.toolCallId, - args: operation.args, - }; - if (body.stream) { - await streamToolCall(res, model, toolCall); - } else { - jsonToolCall(res, model, toolCall); - } - return true; -} - -async function handleFileAuthoringE2E(body, res, model) { - const createSkillName = getRequestedSkillName(body, CREATE_SKILL_MARKER); - if (createSkillName) { - return await handleFileAuthoringOperation(body, res, model, { - skillName: createSkillName, - toolName: CREATE_FILE_TOOL_NAME, - toolCallId: CREATE_SKILL_TOOL_CALL_ID, - finalText: CREATE_FILE_AUTHORING_FINAL_TEXT, - args: buildCreateSkillArgs(createSkillName), - }); - } - - const editSkillName = getRequestedSkillName(body, EDIT_SKILL_MARKER); - if (editSkillName) { - return await handleFileAuthoringOperation(body, res, model, { - skillName: editSkillName, - toolName: EDIT_FILE_TOOL_NAME, - toolCallId: EDIT_SKILL_TOOL_CALL_ID, - finalText: EDIT_FILE_AUTHORING_FINAL_TEXT, - args: buildEditSkillArgs(editSkillName), - }); - } - - return false; -} - -async function handleRequest(req, res) { - if (req.method === 'GET' && (req.url === '/health' || req.url === '/')) { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ status: 'ok' })); - return; - } - - if (req.method === 'POST' && req.url && req.url.endsWith('/chat/completions')) { - const body = await readJsonBody(req); - const model = body.model || MODEL_FALLBACK; - if (await handleFileAuthoringE2E(body, res, model)) { - return; - } - if (body.stream) { - await streamTextCompletion(res, model); - } else { - jsonCompletion(res, model); - } - return; - } - - res.writeHead(404, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ error: 'not found' })); -} - -function startMockLlm(port = Number(process.env.MOCK_LLM_PORT) || DEFAULT_PORT) { - const server = http.createServer((req, res) => { - handleRequest(req, res).catch(() => { - res.writeHead(500, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ error: 'mock server error' })); - }); - }); - - return new Promise((resolve) => { - server.listen(port, '127.0.0.1', () => { - console.log(`[e2e] Mock LLM server listening on http://127.0.0.1:${port}`); - resolve(server); - }); - }); -} - -if (require.main === module) { - startMockLlm(); -} - -module.exports = { startMockLlm, MOCK_REPLY }; diff --git a/e2e/setup/record.js b/e2e/setup/record.js index 6bbb1d0c98..70052a2b78 100644 --- a/e2e/setup/record.js +++ b/e2e/setup/record.js @@ -8,9 +8,7 @@ const baseURL = process.env.E2E_BASE_URL || 'http://localhost:3080'; const storageStatePath = path.resolve(rootPath, 'e2e/storageState.json'); const configTemplatePath = path.resolve(rootPath, 'e2e/config/librechat.e2e.yaml'); const configPath = path.resolve(rootPath, 'e2e/.generated/librechat.e2e.yaml'); -const defaultMockLlmBaseURL = 'http://127.0.0.1:8889/v1'; -const mockLlmPort = getMockLlmPort(); -const mockLlmBaseURL = `http://127.0.0.1:${mockLlmPort}/v1`; +const fakeModelHookPath = path.resolve(rootPath, 'e2e/setup/fake-model.js'); const defaultUser = { email: 'testuser@example.com', name: 'Test User', @@ -49,7 +47,8 @@ const rateLimitOverrides = { const mockOverrides = { CONFIG_PATH: configPath, - MOCK_LLM_PORT: mockLlmPort, + /** Loaded in-process by `@librechat/api`'s `createRun` to swap in a fake model. */ + LIBRECHAT_TEST_RUN_HOOK: fakeModelHookPath, OPENAI_API_KEY: 'user_provided', TENANT_ISOLATION_STRICT: 'false', OPENID_CLIENT_ID: '', @@ -75,14 +74,6 @@ const preservedCredentialEnvKeys = new Set([ ]); const npxBin = process.platform === 'win32' ? 'npx.cmd' : 'npx'; -function getMockLlmPort() { - const port = process.env.MOCK_LLM_PORT || '8889'; - if (!/^\d+$/.test(port)) { - throw new Error('MOCK_LLM_PORT must be a numeric port'); - } - return port; -} - function appURL(pathname = '') { const normalizedBaseURL = baseURL.endsWith('/') ? baseURL : `${baseURL}/`; return new URL(pathname.replace(/^\/+/, ''), normalizedBaseURL).toString(); @@ -181,13 +172,8 @@ function formatDate(date) { function writeRuntimeMockConfig() { const template = fs.readFileSync(configTemplatePath, 'utf8'); - - if (!template.includes(defaultMockLlmBaseURL)) { - throw new Error(`Expected mock config template to include ${defaultMockLlmBaseURL}`); - } - fs.mkdirSync(path.dirname(configPath), { recursive: true }); - fs.writeFileSync(configPath, template.replaceAll(defaultMockLlmBaseURL, mockLlmBaseURL)); + fs.writeFileSync(configPath, template); } function parseArgs(argv) { @@ -406,21 +392,6 @@ async function main() { try { if (options.profile === 'mock') { writeRuntimeMockConfig(); - - const mockURL = `http://127.0.0.1:${mockLlmPort}/health`; - if (!(await waitForURL(mockURL, 1000))) { - children.push( - spawnProcess( - 'mock-llm', - 'node', - [path.resolve(rootPath, 'e2e/setup/mock-llm-server.js')], - env, - ), - ); - if (!(await waitForURL(mockURL, 30000))) { - throw new Error(`Mock LLM server did not become ready at ${mockURL}`); - } - } } if (await waitForURL(baseURL, 1000)) { diff --git a/package-lock.json b/package-lock.json index e5b549fa47..12e9841163 100644 --- a/package-lock.json +++ b/package-lock.json @@ -61,7 +61,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.0.1", "@keyv/redis": "^4.3.3", - "@librechat/agents": "^3.2.1", + "@librechat/agents": "^3.2.2", "@librechat/api": "*", "@librechat/data-schemas": "*", "@microsoft/microsoft-graph-client": "^3.0.7", @@ -11734,9 +11734,9 @@ } }, "node_modules/@librechat/agents": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-3.2.1.tgz", - "integrity": "sha512-CusD5Iif0rd/mTAC/3J32ZZwm0EjArRkOUGEXQoMWlh5hrU2pGw/XjNK54GwSaqOAKV31bD569BhJ12BR2WXBQ==", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-3.2.2.tgz", + "integrity": "sha512-ibjAoUCWQ4zgIJfnhAr3O8JkUfRXbcvyH2/4NFNc9YgRHvxS5tPoJAWUnBmC6t0AOt89vaJAQQwTMWo2YEB9qw==", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "^0.92.0", @@ -44016,7 +44016,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.0.1", "@keyv/redis": "^4.3.3", - "@librechat/agents": "^3.2.1", + "@librechat/agents": "^3.2.2", "@librechat/data-schemas": "*", "@modelcontextprotocol/sdk": "^1.29.0", "@opentelemetry/api": "^1.9.0", diff --git a/packages/api/package.json b/packages/api/package.json index e8fe6d74ad..d1cc530352 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -104,7 +104,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.0.1", "@keyv/redis": "^4.3.3", - "@librechat/agents": "^3.2.1", + "@librechat/agents": "^3.2.2", "@librechat/data-schemas": "*", "@modelcontextprotocol/sdk": "^1.29.0", "@opentelemetry/api": "^1.9.0", diff --git a/packages/api/src/agents/index.ts b/packages/api/src/agents/index.ts index 31d2485a01..14e0f480fd 100644 --- a/packages/api/src/agents/index.ts +++ b/packages/api/src/agents/index.ts @@ -22,6 +22,7 @@ export * from './skillConfigurable'; export * from './skillFiles'; export * from './codeFilesSession'; export * from './run'; +export * from './testHook'; export * from './tools'; export * from './validation'; export * from './added'; diff --git a/packages/api/src/agents/run.ts b/packages/api/src/agents/run.ts index 85b614f4f7..2549ec7984 100644 --- a/packages/api/src/agents/run.ts +++ b/packages/api/src/agents/run.ts @@ -35,6 +35,7 @@ import type * as t from '~/types'; import { getProviderConfig } from '~/endpoints/config/providers'; import { resolveHeaders, createSafeUser } from '~/utils/env'; import { getOpenAIConfig } from '~/endpoints/openai/config'; +import { applyTestRunHook } from '~/agents/testHook'; import { isUserProvided } from '~/utils/common'; /** Expected shape of JSON tool search results */ @@ -1030,7 +1031,7 @@ export async function createRun({ */ const enableToolOutputReferences = anyAgentHasCodeEnv(agents); - return Run.create({ + const run = await Run.create({ runId, graphConfig, tokenCounter, @@ -1043,4 +1044,7 @@ export async function createRun({ toolOutputReferences: { enabled: true }, }), }); + + applyTestRunHook(run, { messages, agents }); + return run; } diff --git a/packages/api/src/agents/testHook.ts b/packages/api/src/agents/testHook.ts new file mode 100644 index 0000000000..1599669c82 --- /dev/null +++ b/packages/api/src/agents/testHook.ts @@ -0,0 +1,40 @@ +import { logger } from '@librechat/data-schemas'; +import type { Run, IState } from '@librechat/agents'; +import type { BaseMessage } from '@librechat/agents/langchain/messages'; + +/** + * Context handed to a test run hook so it can shape fake-model behavior from + * the conversation and the agents' advertised tools. + */ +export interface TestRunHookContext { + messages?: BaseMessage[]; + agents: ReadonlyArray<{ tools?: ReadonlyArray<{ name: string }> }>; +} + +export type TestRunHook = (run: Run, context: TestRunHookContext) => void; + +/** + * Env-gated extension point used only by the e2e harness. When + * `LIBRECHAT_TEST_RUN_HOOK` points at a module, it is loaded and invoked with + * the freshly created run so a test can swap in a fake model via + * `run.Graph.overrideTestModel(...)` instead of reaching a live provider. A + * no-op (returns immediately) in normal operation since the env var is unset. + */ +export function applyTestRunHook(run: Run, context: TestRunHookContext): void { + const hookPath = process.env.LIBRECHAT_TEST_RUN_HOOK; + if (!hookPath) { + return; + } + try { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const loaded = require(hookPath) as TestRunHook | { default?: TestRunHook }; + const hook = typeof loaded === 'function' ? loaded : loaded.default; + if (typeof hook !== 'function') { + logger.warn(`[applyTestRunHook] ${hookPath} did not export a function`); + return; + } + hook(run, context); + } catch (error) { + logger.error(`[applyTestRunHook] Failed to apply hook from ${hookPath}`, error); + } +}