diff --git a/e2e/config/librechat.e2e.yaml b/e2e/config/librechat.e2e.yaml index 4ed7224f53..a852ae1594 100644 --- a/e2e/config/librechat.e2e.yaml +++ b/e2e/config/librechat.e2e.yaml @@ -4,6 +4,16 @@ version: 1.3.11 cache: true +mcpServers: + e2e-memory: + type: stdio + command: node + args: + - e2e/setup/fake-mcp-server.js + title: E2E Memory + description: Local MCP fixture used by mock end-to-end tests. + timeout: 30000 + endpoints: custom: - name: 'Mock Provider A' @@ -15,6 +25,8 @@ endpoints: fetch: false titleConvo: false modelDisplayLabel: 'Mock Provider A' + customParams: + defaultParamsEndpoint: anthropic - name: 'Mock Provider B' apiKey: 'e2e-mock-key-b' diff --git a/e2e/setup/fake-mcp-server.js b/e2e/setup/fake-mcp-server.js new file mode 100644 index 0000000000..87419df88a --- /dev/null +++ b/e2e/setup/fake-mcp-server.js @@ -0,0 +1,53 @@ +#!/usr/bin/env node + +const { McpServer } = require('@modelcontextprotocol/sdk/server/mcp.js'); +const { StdioServerTransport } = require('@modelcontextprotocol/sdk/server/stdio.js'); +const z = require('zod/v4'); + +const server = new McpServer({ + name: 'e2e-memory', + version: '1.0.0', +}); + +server.registerTool( + 'remember_fact', + { + description: 'Stores a deterministic fact for LibreChat mock end-to-end tests.', + inputSchema: { + fact: z.string().optional(), + }, + }, + async ({ fact = 'LibreChat MCP e2e fact' }) => ({ + content: [ + { + type: 'text', + text: `E2E MCP memory noted: ${fact}`, + }, + ], + }), +); + +server.registerTool( + 'recall_fact', + { + description: 'Returns a deterministic fact for LibreChat mock end-to-end tests.', + inputSchema: {}, + }, + async () => ({ + content: [ + { + type: 'text', + text: 'E2E MCP memory recalled: LibreChat can persist MCP tools on agents.', + }, + ], + }), +); + +async function main() { + await server.connect(new StdioServerTransport()); +} + +main().catch((error) => { + console.error('[fake-mcp-server] failed to start', error); + process.exit(1); +}); diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.js index 277adc85cc..9702f5282a 100644 --- a/e2e/setup/fake-model.js +++ b/e2e/setup/fake-model.js @@ -23,6 +23,7 @@ const COUNTED_REPLY_MARKER = 'E2E_COUNTED_REPLY:'; const SLOW_REPLY_MARKER = 'E2E_SLOW_REPLY:'; const RESUME_ICON_REPLY_MARKER = 'E2E_RESUME_ICON_REPLY:'; const FORCED_ERROR_MARKER = 'E2E_FORCED_ERROR:'; +const MARKDOWN_REPLY_MARKER = 'E2E_MARKDOWN_REPLY'; const CREATE_FILE_AUTHORING_FINAL_TEXT = 'E2E file authoring complete'; const EDIT_FILE_AUTHORING_FINAL_TEXT = 'E2E file edit complete'; const MODEL_SPEC_SKILL_ASSERTION_FINAL_TEXT = 'E2E model spec skill assertion passed'; @@ -228,6 +229,24 @@ function providerFileAssertionResponses({ messages, text }) { } function replyResponses(text) { + if (text.includes(MARKDOWN_REPLY_MARKER)) { + return { + responses: [ + [ + '## E2E markdown heading', + '', + '**E2E bold text**', + '', + '- E2E list item', + '', + '```javascript', + 'const e2eSyntaxHighlight = "ok";', + '```', + ].join('\n'), + ], + }; + } + const errorName = getMarkerValue(text, FORCED_ERROR_MARKER); if (errorName) { return { diff --git a/e2e/specs/mock/agents.helpers.ts b/e2e/specs/mock/agents.helpers.ts new file mode 100644 index 0000000000..bf5566f6af --- /dev/null +++ b/e2e/specs/mock/agents.helpers.ts @@ -0,0 +1,130 @@ +import { expect } from '@playwright/test'; +import type { Page } from '@playwright/test'; +import { MOCK_ENDPOINTS, NEW_CHAT_PATH, fetchJson, getAccessToken, requestJson } from './helpers'; + +export const AGENT_EDIT_PERMISSION = 2; + +export type AgentSummary = { + _id: string; + id: string; + name?: string; + description?: string; + provider: string; + model: string; + category?: string; +}; + +export type ModelParameters = { + maxContextTokens?: number; + maxOutputTokens?: number; + temperature?: number; + topP?: number; + topK?: number; + thinkingBudget?: number; + fileTokenLimit?: number; + resendFiles?: boolean; + promptCache?: boolean; + thinking?: boolean; + web_search?: boolean; +}; + +export type AgentDetail = AgentSummary & { + instructions?: string; + model_parameters?: ModelParameters; + tools?: string[]; + mcpServerNames?: string[]; +}; + +export const uniqueAgentName = (prefix: string) => + `${prefix} ${Date.now()}-${Math.floor(Math.random() * 1e4)}`; + +export async function findAgent( + page: Page, + agentName: string, + token: string, +): Promise { + const body = await fetchJson<{ data?: AgentSummary[] }>( + page, + `/api/agents?search=${encodeURIComponent(agentName)}&limit=10&requiredPermission=${AGENT_EDIT_PERMISSION}`, + token, + ); + return body.data?.find((agent) => agent.name === agentName) ?? null; +} + +export async function waitForPersistedAgent( + page: Page, + agentName: string, + description: string, +): Promise { + const token = await getAccessToken(page); + let latestAgent: AgentDetail | null = null; + + for (let attempt = 0; attempt < 20; attempt++) { + const agent = await findAgent(page, agentName, token); + if (agent) { + latestAgent = await fetchJson( + page, + `/api/agents/${encodeURIComponent(agent.id)}/expanded`, + token, + ); + if (latestAgent.description === description) { + return latestAgent; + } + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + + expect(latestAgent, `Expected agent "${agentName}" to be persisted`).not.toBeNull(); + expect(latestAgent?.description).toBe(description); + return latestAgent!; +} + +export async function openAgentBuilder(page: Page) { + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + + const form = page.getByRole('form', { name: 'Agent configuration form' }); + const builderVisible = await form + .waitFor({ state: 'visible', timeout: 1000 }) + .then(() => true) + .catch(() => false); + if (!builderVisible) { + const agentBuilderButton = page.getByRole('button', { name: 'Agent Builder' }); + await expect(agentBuilderButton).toBeVisible(); + if ((await agentBuilderButton.getAttribute('aria-pressed')) !== 'true') { + await agentBuilderButton.click(); + } + } + await expect(form).toBeVisible(); + return form; +} + +export async function selectMockModel(page: Page, clickBackToBuilder = false) { + const form = page.getByRole('form', { name: 'Agent configuration form' }); + + await form.locator('label[for="provider"] + button').click(); + await expect(form.getByText('Model Parameters', { exact: true })).toBeVisible(); + + await form.getByRole('combobox', { name: 'Provider' }).click(); + await page.getByRole('option', { name: MOCK_ENDPOINTS[0].label }).click(); + + await form.getByRole('combobox', { name: 'Model' }).click(); + await page.getByRole('option', { name: MOCK_ENDPOINTS[0].model, exact: true }).click(); + + if (clickBackToBuilder) { + await form.getByRole('button', { name: 'Back to builder' }).click(); + await expect(page.getByRole('form', { name: 'Agent configuration form' })).toBeVisible(); + } +} + +export async function cleanupAgent(page: Page, agentId?: string) { + if (!agentId) { + return; + } + + const token = await getAccessToken(page); + await requestJson<{ message?: string }>(page, { + path: `/api/agents/${encodeURIComponent(agentId)}`, + token, + method: 'DELETE', + }); +} diff --git a/e2e/specs/mock/agents.spec.ts b/e2e/specs/mock/agents.spec.ts new file mode 100644 index 0000000000..b066234aaf --- /dev/null +++ b/e2e/specs/mock/agents.spec.ts @@ -0,0 +1,195 @@ +import { expect, test } from '@playwright/test'; +import type { Locator, Page } from '@playwright/test'; +import type { AgentDetail } from './agents.helpers'; +import { + cleanupAgent, + openAgentBuilder, + selectMockModel, + uniqueAgentName, + waitForPersistedAgent, +} from './agents.helpers'; +import { MOCK_ENDPOINTS, mockReply, sendMessage } from './helpers'; + +const DESCRIPTION = 'Use this agent to verify LibreChat agent creation in mock end-to-end tests.'; +const INSTRUCTIONS = + 'Reply through the mock e2e model and keep this agent available for UI persistence checks.'; +const MODEL_PARAMETERS = { + maxContextTokens: 32000, + maxOutputTokens: 4096, + temperature: 0.25, + topP: 0.8, + topK: 12, + resendFiles: false, + promptCache: true, + thinking: true, + thinkingBudget: 2000, + web_search: true, + fileTokenLimit: 12000, +}; +const PERSISTED_MODEL_PARAMETERS = { + maxContextTokens: MODEL_PARAMETERS.maxContextTokens, + maxOutputTokens: MODEL_PARAMETERS.maxOutputTokens, + temperature: MODEL_PARAMETERS.temperature, + topP: MODEL_PARAMETERS.topP, + topK: MODEL_PARAMETERS.topK, + resendFiles: MODEL_PARAMETERS.resendFiles, + web_search: MODEL_PARAMETERS.web_search, + fileTokenLimit: MODEL_PARAMETERS.fileTokenLimit, +}; + +async function setSwitch(form: Locator, name: string, checked: boolean) { + const control = form.getByRole('switch', { name, exact: true }); + await expect(control).toBeVisible(); + if ((await control.getAttribute('aria-checked')) !== String(checked)) { + await control.click(); + } + await expect(control).toHaveAttribute('aria-checked', String(checked)); +} + +async function fillAnthropicStyleModelParameters(page: Page) { + const form = page.getByRole('form', { name: 'Agent configuration form' }); + + await expect(form.getByLabel('Max Context Tokens')).toBeVisible(); + await expect(form.getByLabel('Max Output Tokens')).toBeVisible(); + await expect(form.getByText('Temperature', { exact: true })).toBeVisible(); + await expect(form.getByText('Top P', { exact: true })).toBeVisible(); + await expect(form.getByText('Top K', { exact: true })).toBeVisible(); + await expect(form.getByText('Effort', { exact: true })).toBeVisible(); + await expect(form.getByText('Thought Visibility', { exact: true })).toBeVisible(); + + await form + .locator('#maxContextTokens-dynamic-input') + .fill(`${MODEL_PARAMETERS.maxContextTokens}`); + await form.locator('#maxOutputTokens-dynamic-input').fill(`${MODEL_PARAMETERS.maxOutputTokens}`); + await form + .locator('#temperature-dynamic-setting-input-number') + .fill(`${MODEL_PARAMETERS.temperature}`); + await form.locator('#topP-dynamic-setting-input-number').fill(`${MODEL_PARAMETERS.topP}`); + await form.locator('#topK-dynamic-setting-input-number').fill(`${MODEL_PARAMETERS.topK}`); + await form.locator('#thinkingBudget-dynamic-input').fill(`${MODEL_PARAMETERS.thinkingBudget}`); + await form.locator('#fileTokenLimit-dynamic-input').fill(`${MODEL_PARAMETERS.fileTokenLimit}`); + + await setSwitch(form, 'Resend Files', MODEL_PARAMETERS.resendFiles); + await setSwitch(form, 'Use Prompt Caching', MODEL_PARAMETERS.promptCache); + await setSwitch(form, 'Thinking', MODEL_PARAMETERS.thinking); + await setSwitch(form, 'Web Search', MODEL_PARAMETERS.web_search); + + // DynamicInput/DynamicSlider values use a 450ms debounced form update. + await page.waitForTimeout(600); +} + +async function expectAnthropicStyleModelParameters(page: Page) { + const form = page.getByRole('form', { name: 'Agent configuration form' }); + + await expect(form.locator('#maxContextTokens-dynamic-input')).toHaveValue( + `${MODEL_PARAMETERS.maxContextTokens}`, + ); + await expect(form.locator('#maxOutputTokens-dynamic-input')).toHaveValue( + `${MODEL_PARAMETERS.maxOutputTokens}`, + ); + await expect(form.locator('#temperature-dynamic-setting-input-number')).toHaveValue( + `${MODEL_PARAMETERS.temperature}`, + ); + await expect(form.locator('#topP-dynamic-setting-input-number')).toHaveValue( + MODEL_PARAMETERS.topP.toFixed(2), + ); + await expect(form.locator('#topK-dynamic-setting-input-number')).toHaveValue( + `${MODEL_PARAMETERS.topK}`, + ); + await expect(form.locator('#thinkingBudget-dynamic-input')).toHaveValue( + `${MODEL_PARAMETERS.thinkingBudget}`, + ); + await expect(form.locator('#fileTokenLimit-dynamic-input')).toHaveValue( + `${MODEL_PARAMETERS.fileTokenLimit}`, + ); + + await expect(form.getByRole('switch', { name: 'Resend Files', exact: true })).toHaveAttribute( + 'aria-checked', + String(MODEL_PARAMETERS.resendFiles), + ); + await expect( + form.getByRole('switch', { name: 'Use Prompt Caching', exact: true }), + ).toHaveAttribute('aria-checked', String(MODEL_PARAMETERS.promptCache)); + await expect(form.getByRole('switch', { name: 'Thinking', exact: true })).toHaveAttribute( + 'aria-checked', + String(MODEL_PARAMETERS.thinking), + ); + await expect(form.getByRole('switch', { name: 'Web Search', exact: true })).toHaveAttribute( + 'aria-checked', + String(MODEL_PARAMETERS.web_search), + ); +} + +test.describe('agent builder', () => { + test('creates an agent, persists its configuration, and can chat with it', async ({ page }) => { + test.setTimeout(120000); + + const agentName = uniqueAgentName('E2E Agent'); + let createdAgentId: string | undefined; + + try { + let form = await openAgentBuilder(page); + + await form.getByLabel('Agent name').fill(agentName); + await form.getByLabel('Agent description').fill(DESCRIPTION); + await form.getByLabel('Agent instructions').fill(INSTRUCTIONS); + + await selectMockModel(page); + await fillAnthropicStyleModelParameters(page); + await page + .getByRole('form', { name: 'Agent configuration form' }) + .getByRole('button', { + name: 'Back to builder', + }) + .click(); + form = page.getByRole('form', { name: 'Agent configuration form' }); + + const [createResponse] = await Promise.all([ + page.waitForResponse( + (response) => + response.request().method() === 'POST' && + new URL(response.url()).pathname === '/api/agents' && + response.status() === 201, + { timeout: 30000 }, + ), + form.getByRole('button', { name: 'Create' }).click(), + ]); + const createdAgent = (await createResponse.json()) as AgentDetail; + createdAgentId = createdAgent.id; + + await expect(page.getByText(`Successfully created ${agentName}`)).toBeVisible(); + + const persistedAgent = await waitForPersistedAgent(page, agentName, DESCRIPTION); + expect(persistedAgent).toMatchObject({ + id: createdAgentId, + name: agentName, + description: DESCRIPTION, + instructions: INSTRUCTIONS, + provider: MOCK_ENDPOINTS[0].label, + model: MOCK_ENDPOINTS[0].model, + category: 'general', + }); + expect(persistedAgent.model_parameters).toMatchObject(PERSISTED_MODEL_PARAMETERS); + + form = await openAgentBuilder(page); + await form.getByRole('combobox', { name: 'Agent', exact: true }).click(); + await page.getByRole('option', { name: agentName }).click(); + + await expect(form.getByLabel('Agent name')).toHaveValue(agentName); + await expect(form.getByLabel('Agent description')).toHaveValue(DESCRIPTION); + await expect(form.getByLabel('Agent instructions')).toHaveValue(INSTRUCTIONS); + + await form.locator('label[for="provider"] + button').click(); + await expectAnthropicStyleModelParameters(page); + await form.getByRole('button', { name: 'Back to builder' }).click(); + + await form.getByRole('button', { name: 'Select Agent' }).click(); + + const response = await sendMessage(page, `hello from ${agentName}`); + expect(response.ok()).toBeTruthy(); + await expect(mockReply(page)).toBeVisible({ timeout: 30000 }); + } finally { + await cleanupAgent(page, createdAgentId); + } + }); +}); diff --git a/e2e/specs/mock/app-load.spec.ts b/e2e/specs/mock/app-load.spec.ts index aadb3ec4db..9c5d686501 100644 --- a/e2e/specs/mock/app-load.spec.ts +++ b/e2e/specs/mock/app-load.spec.ts @@ -1,10 +1,12 @@ import { expect, test } from '@playwright/test'; +import { getPrimaryE2EUser } from '../../setup/users.mock'; import { NEW_CHAT_PATH } from './helpers'; test.describe('app loads cleanly', () => { test('authenticated user lands on a rendered chat view without runtime errors', async ({ page, }) => { + const user = getPrimaryE2EUser(); const pageErrors: string[] = []; page.on('pageerror', (error) => pageErrors.push(error.message)); @@ -12,6 +14,7 @@ test.describe('app loads cleanly', () => { await expect(page).toHaveURL(/\/c\/new$/); await expect(page.getByRole('main')).toBeVisible(); + await expect(page.getByRole('main')).toContainText(user.name); await expect(page.getByRole('textbox', { name: 'Message input' })).toBeVisible(); await expect(page.getByRole('button', { name: 'Select a model' }).first()).toBeVisible(); await expect(page.getByTestId('nav-user')).toBeVisible(); diff --git a/e2e/specs/mock/chat.spec.ts b/e2e/specs/mock/chat.spec.ts index 36466420af..0fcfb8bcbd 100644 --- a/e2e/specs/mock/chat.spec.ts +++ b/e2e/specs/mock/chat.spec.ts @@ -1,13 +1,83 @@ import { expect, test } from '@playwright/test'; +import type { Page } from '@playwright/test'; import { isAgentsStream, MOCK_ENDPOINTS, NEW_CHAT_PATH, + messagesView, mockReply, selectMockEndpoint, sendMessage, } from './helpers'; +type UploadFixture = { + name: string; + mimeType: string; + buffer: Buffer; +}; + +const pdfFixture: UploadFixture = { + name: 'provider-context.pdf', + mimeType: 'application/pdf', + buffer: Buffer.from( + `%PDF-1.4 +1 0 obj +<< /Type /Catalog /Pages 2 0 R >> +endobj +2 0 obj +<< /Type /Pages /Count 0 >> +endobj +trailer +<< /Root 1 0 R >> +%%EOF +`, + ), +}; + +const textFixture: UploadFixture = { + name: 'provider-context.txt', + mimeType: 'text/plain', + buffer: Buffer.from('This text attachment should be available to the mock model.\n'), +}; + +const imageFixture: UploadFixture = { + name: 'provider-context.png', + mimeType: 'image/png', + buffer: Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=', + 'base64', + ), +}; + +const composer = (page: Page) => page.locator('form'); + +async function openProviderFileChooser(page: Page) { + await page.getByRole('button', { name: 'Attach File Options' }).click(); + await expect(page.getByText('Upload to Provider')).toBeVisible(); + + const fileChooserPromise = page.waitForEvent('filechooser'); + await page.getByText('Upload to Provider').click(); + const fileChooser = await fileChooserPromise; + expect(await fileChooser.element().getAttribute('type')).toBe('file'); + return fileChooser; +} + +async function uploadProviderFile(page: Page, fixture: UploadFixture) { + const fileChooser = await openProviderFileChooser(page); + const uploadResponsePromise = page.waitForResponse( + (response) => + response.url().includes('/api/files') && + response.request().method() === 'POST' && + response.status() === 200, + { timeout: 30000 }, + ); + await fileChooser.setFiles(fixture); + const uploadResponse = await uploadResponsePromise; + expect(uploadResponse.ok()).toBeTruthy(); + await page.waitForTimeout(350); + return uploadResponse; +} + test.describe('core chat loop', () => { test('streams a response, saves the conversation, and persists across reload', async ({ page, @@ -22,6 +92,11 @@ test.describe('core chat loop', () => { expect(response.ok()).toBeTruthy(); await expect(page.getByText(userMessage)).toBeVisible(); + const userMessageTurn = messagesView(page) + .locator('.message-render') + .filter({ hasText: userMessage }); + await expect(userMessageTurn.locator('.user-turn')).toBeVisible(); + await expect(userMessageTurn.locator('.agent-turn')).toHaveCount(0); await expect(mockReply(page)).toBeVisible(); await expect(page).toHaveURL(/\/c\/[0-9a-fA-F-]{36}$/); @@ -36,6 +111,48 @@ test.describe('core chat loop', () => { await expect(page.getByTestId('convo-item').first()).toBeVisible(); }); + test('keeps send disabled until the composer has message text', async ({ page }) => { + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await selectMockEndpoint(page, MOCK_ENDPOINTS[0]); + + const input = page.getByRole('textbox', { name: 'Message input' }); + const sendButton = page.getByTestId('send-button'); + + await expect(sendButton).toBeDisabled(); + await input.fill('ready to send'); + await expect(sendButton).toBeEnabled(); + await input.fill(' '); + await expect(sendButton).toBeDisabled(); + }); + + test('renders assistant markdown and syntax-highlighted code blocks', async ({ page }) => { + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await selectMockEndpoint(page, MOCK_ENDPOINTS[0]); + + const response = await sendMessage(page, 'E2E_MARKDOWN_REPLY'); + expect(response.ok()).toBeTruthy(); + + const assistantMessage = messagesView(page) + .locator('.message-render') + .filter({ hasText: 'E2E markdown heading' }) + .last(); + await expect(assistantMessage.locator('.agent-turn')).toBeVisible(); + await expect( + assistantMessage.getByRole('heading', { name: 'E2E markdown heading' }), + ).toBeVisible(); + await expect( + assistantMessage.locator('strong').filter({ hasText: 'E2E bold text' }), + ).toBeVisible(); + await expect( + assistantMessage.getByRole('listitem').filter({ hasText: 'E2E list item' }), + ).toBeVisible(); + + const codeBlock = assistantMessage.locator('code').filter({ hasText: 'e2eSyntaxHighlight' }); + await expect(codeBlock).toBeVisible(); + await expect(codeBlock).toHaveClass(/hljs/); + await expect(codeBlock).toHaveClass(/language-javascript/); + }); + test('can switch back to the previous branch after regenerating an earlier response', async ({ page, }) => { @@ -54,10 +171,7 @@ test.describe('core chat loop', () => { expect(response.ok()).toBeTruthy(); await expect(page.getByText(followUpMessage)).toBeVisible(); - const firstAssistantMessage = page - .getByTestId('messages-view') - .locator('.message-render') - .nth(1); + const firstAssistantMessage = messagesView(page).locator('.message-render').nth(1); await firstAssistantMessage.hover(); const regenerateButton = firstAssistantMessage.locator('button[title="Regenerate"]').last(); await expect(regenerateButton).toBeVisible(); @@ -79,31 +193,19 @@ test.describe('core chat loop', () => { }) => { test.setTimeout(90000); - const filename = 'provider-upload.csv'; + const csvFixture: UploadFixture = { + name: 'provider-upload.csv', + mimeType: 'text/csv', + buffer: Buffer.from('name,value\nalpha,1\n'), + }; + const filename = csvFixture.name; const assertionText = `E2E_ASSERT_PROVIDER_FILE:${filename}`; - const fileChip = page.getByTestId('messages-view').getByRole('button', { name: filename }); + const fileChip = messagesView(page).getByRole('button', { name: filename }); await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); await selectMockEndpoint(page, MOCK_ENDPOINTS[0]); - await page.getByRole('button', { name: 'Attach File Options' }).click(); - const fileChooserPromise = page.waitForEvent('filechooser'); - await page.getByText('Upload to Provider').click(); - const fileChooser = await fileChooserPromise; - - const uploadResponsePromise = page.waitForResponse( - (response) => - response.url().includes('/api/files') && - response.request().method() === 'POST' && - response.status() === 200, - { timeout: 30000 }, - ); - await fileChooser.setFiles({ - name: filename, - mimeType: 'text/csv', - buffer: Buffer.from('name,value\nalpha,1\n'), - }); - await uploadResponsePromise; + await uploadProviderFile(page, csvFixture); await expect(page.getByRole('button', { name: filename })).toBeVisible(); @@ -119,9 +221,7 @@ test.describe('core chat loop', () => { expect(response.ok()).toBeTruthy(); await expect( - page - .getByTestId('messages-view') - .getByText(`E2E provider file assertion passed: ${filename}`), + messagesView(page).getByText(`E2E provider file assertion passed: ${filename}`), ).toBeVisible(); await expect(fileChip).toBeVisible(); @@ -131,4 +231,46 @@ test.describe('core chat loop', () => { await expect(page).toHaveURL(conversationUrl); await expect(fileChip).toBeVisible(); }); + + test('supports attaching, removing, and sending provider files from the composer', async ({ + page, + }) => { + test.setTimeout(90000); + + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await selectMockEndpoint(page, MOCK_ENDPOINTS[0]); + + await uploadProviderFile(page, pdfFixture); + const pdfChip = composer(page).getByRole('button', { name: pdfFixture.name }); + await expect(pdfChip).toBeVisible(); + + await composer(page).getByRole('button', { name: 'Remove file' }).click(); + await expect(pdfChip).toHaveCount(0); + + await uploadProviderFile(page, imageFixture); + await expect( + composer(page).getByRole('button', { name: 'View Preview image in full size' }), + ).toBeVisible(); + + await uploadProviderFile(page, textFixture); + const textChip = composer(page).getByRole('button', { name: textFixture.name }); + await expect(textChip).toBeVisible(); + + const assertionText = `E2E_ASSERT_PROVIDER_FILE:${textFixture.name}`; + const input = page.getByRole('textbox', { name: 'Message input' }); + await input.click(); + await input.fill(assertionText); + await expect(page.getByTestId('send-button')).toBeEnabled(); + + const [response] = await Promise.all([ + page.waitForResponse(isAgentsStream, { timeout: 30000 }), + page.getByTestId('send-button').click(), + ]); + expect(response.ok()).toBeTruthy(); + + await expect( + messagesView(page).getByText(`E2E provider file assertion passed: ${textFixture.name}`), + ).toBeVisible(); + await expect(messagesView(page).getByRole('button', { name: textFixture.name })).toBeVisible(); + }); }); diff --git a/e2e/specs/mock/conversation-management.spec.ts b/e2e/specs/mock/conversation-management.spec.ts new file mode 100644 index 0000000000..c289aa1699 --- /dev/null +++ b/e2e/specs/mock/conversation-management.spec.ts @@ -0,0 +1,120 @@ +import { expect, test } from '@playwright/test'; +import type { Locator, Page } from '@playwright/test'; +import { + MOCK_ENDPOINTS, + NEW_CHAT_PATH, + messagesView, + replyPrompt, + replyText, + selectMockEndpoint, + sendMessage, +} from './helpers'; + +const firstConversation = (page: Page) => page.getByTestId('convo-item').first(); + +function uniqueLabel(prefix: string) { + return `${prefix}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`; +} + +async function openMockChat(page: Page) { + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await selectMockEndpoint(page, MOCK_ENDPOINTS[0]); +} + +async function sendAndExpectReply(page: Page, label: string) { + const prompt = replyPrompt(label); + const reply = replyText(label); + const response = await sendMessage(page, prompt); + expect(response.ok()).toBeTruthy(); + await expect(messagesView(page).getByText(prompt)).toBeVisible(); + await expect(messagesView(page).getByText(reply)).toBeVisible(); + return { prompt, reply }; +} + +async function openConversationMenu(conversation: Locator) { + await conversation.hover(); + await conversation.getByRole('button', { name: 'Conversation Menu Options' }).click(); +} + +async function renameConversation(page: Page, conversation: Locator, title: string) { + await openConversationMenu(conversation); + await page.getByRole('menuitem', { name: 'Rename' }).click(); + const titleInput = conversation.getByRole('textbox', { name: 'New Conversation Title' }); + await expect(titleInput).toBeVisible(); + await titleInput.fill(title); + await conversation.getByRole('button', { name: 'Save' }).click(); + await expect(conversation).toContainText(title); +} + +test.describe('conversation management', () => { + test('loads a past sidebar conversation with its message history', async ({ page }) => { + const firstLabel = uniqueLabel('sidebar-history-first'); + const secondLabel = uniqueLabel('sidebar-history-second'); + + await openMockChat(page); + const firstTurn = await sendAndExpectReply(page, firstLabel); + const secondTurn = await sendAndExpectReply(page, secondLabel); + const conversationUrl = page.url(); + + await expect(page).toHaveURL(/\/c\/[0-9a-fA-F-]{36}$/); + await expect(firstConversation(page)).toBeVisible(); + + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await expect(page).toHaveURL(/\/c\/new$/); + await expect(messagesView(page).getByText(firstTurn.prompt)).toHaveCount(0); + await expect(messagesView(page).getByText(secondTurn.prompt)).toHaveCount(0); + + await firstConversation(page).click(); + await expect(page).toHaveURL(conversationUrl); + await expect(messagesView(page).getByText(firstTurn.prompt)).toBeVisible(); + await expect(messagesView(page).getByText(firstTurn.reply)).toBeVisible(); + await expect(messagesView(page).getByText(secondTurn.prompt)).toBeVisible(); + await expect(messagesView(page).getByText(secondTurn.reply)).toBeVisible(); + }); + + test('renames a conversation from the sidebar', async ({ page }) => { + const label = uniqueLabel('sidebar-rename'); + const renamedTitle = `Renamed ${label}`; + + await openMockChat(page); + await sendAndExpectReply(page, label); + + await renameConversation(page, firstConversation(page), renamedTitle); + await page.reload({ timeout: 10000 }); + await expect(page.getByTestId('convo-item').filter({ hasText: renamedTitle })).toBeVisible(); + }); + + test('deletes a conversation from the sidebar and blocks direct URL access', async ({ page }) => { + const label = uniqueLabel('sidebar-delete'); + const renamedTitle = `Delete ${label}`; + + await openMockChat(page); + const turn = await sendAndExpectReply(page, label); + const conversationUrl = page.url(); + + const conversation = firstConversation(page); + await renameConversation(page, conversation, renamedTitle); + await openConversationMenu(conversation); + await page.getByRole('menuitem', { name: 'Delete' }).click(); + + const dialog = page.getByRole('dialog', { name: 'Delete chat?' }); + await expect(dialog).toBeVisible(); + const [deleteResponse] = await Promise.all([ + page.waitForResponse( + (response) => + response.request().method() === 'DELETE' && response.url().includes('/api/convos'), + { timeout: 30000 }, + ), + dialog.getByRole('button', { name: 'Delete' }).click(), + ]); + expect(deleteResponse.ok()).toBeTruthy(); + + await expect(page).toHaveURL(/\/c\/new$/); + await expect(page.getByTestId('convo-item').filter({ hasText: renamedTitle })).toHaveCount(0); + + await page.goto(conversationUrl, { timeout: 10000 }); + await expect(page.getByRole('textbox', { name: 'Message input' })).toBeVisible(); + await expect(messagesView(page).getByText(turn.prompt)).toHaveCount(0); + await expect(messagesView(page).getByText(turn.reply)).toHaveCount(0); + }); +}); diff --git a/e2e/specs/mock/helpers.ts b/e2e/specs/mock/helpers.ts index 08538b28d6..42d98eaa82 100644 --- a/e2e/specs/mock/helpers.ts +++ b/e2e/specs/mock/helpers.ts @@ -36,7 +36,7 @@ export function isAgentGenerationStart(response: Response) { const modelSelectorTrigger = (page: Page) => page.getByRole('button', { name: 'Select a model' }).first(); -const escapeRegExp = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +export const escapeRegExp = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); /** Open the model selector, choose an endpoint, then its model (committed on the model click). */ export async function selectMockEndpoint(page: Page, endpoint: MockEndpoint) { @@ -70,9 +70,16 @@ export async function enableSkills(page: Page) { await expect(page.getByRole('button', { name: 'Skills' })).toBeVisible(); } +/** The conversation messages container. */ +export const messagesView = (page: Page) => page.getByTestId('messages-view'); + +/** Build the mock-model reply trigger and its expected rendered text for a label. */ +export const replyPrompt = (label: string) => `E2E_REPLY:${label}`; +export const replyText = (label: string) => `E2E reply ${label}`; + /** The mock reply as rendered in the conversation, scoped to the messages view. */ export function mockReply(page: Page) { - return page.getByTestId('messages-view').getByText(new RegExp(MOCK_REPLY_TEXT, 'i')); + return messagesView(page).getByText(new RegExp(MOCK_REPLY_TEXT, 'i')); } /** Type a message, send it, and wait for the streamed `/api/agents` response. */ diff --git a/e2e/specs/mock/mcp.spec.ts b/e2e/specs/mock/mcp.spec.ts new file mode 100644 index 0000000000..0a2b4fd466 --- /dev/null +++ b/e2e/specs/mock/mcp.spec.ts @@ -0,0 +1,150 @@ +import { expect, test } from '@playwright/test'; +import type { Locator, Page } from '@playwright/test'; +import type { AgentDetail } from './agents.helpers'; +import { + cleanupAgent, + openAgentBuilder, + selectMockModel, + uniqueAgentName, + waitForPersistedAgent, +} from './agents.helpers'; +import { MOCK_ENDPOINTS, fetchJson, getAccessToken, mockReply, sendMessage } from './helpers'; + +const MCP_SERVER_NAME = 'e2e-memory'; +const MCP_SERVER_TITLE = 'E2E Memory'; +const MCP_TOOL_NAME = 'remember_fact'; +const MCP_SERVER_TOOL_ID = `sys__server__sys_mcp_${MCP_SERVER_NAME}`; +const MCP_TOOL_ID = `${MCP_TOOL_NAME}_mcp_${MCP_SERVER_NAME}`; +const DESCRIPTION = 'Use this agent to verify LibreChat MCP tool selection in mock e2e tests.'; +const INSTRUCTIONS = + 'Keep the selected MCP server tools available while replying through the mock e2e model.'; + +type MCPToolsResponse = { + servers?: Record< + string, + { + tools?: Array<{ + name: string; + pluginKey: string; + description?: string; + }>; + } + >; +}; + +async function waitForMCPTools(page: Page) { + const token = await getAccessToken(page); + let latestTools: MCPToolsResponse | null = null; + + for (let attempt = 0; attempt < 20; attempt++) { + latestTools = await fetchJson(page, '/api/mcp/tools', token); + const serverTools = latestTools.servers?.[MCP_SERVER_NAME]?.tools ?? []; + if (serverTools.some((tool) => tool.pluginKey === MCP_TOOL_ID)) { + return serverTools; + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + + expect( + latestTools?.servers?.[MCP_SERVER_NAME]?.tools, + `Expected ${MCP_SERVER_NAME} to expose ${MCP_TOOL_ID}`, + ).toEqual(expect.arrayContaining([expect.objectContaining({ pluginKey: MCP_TOOL_ID })])); + return latestTools!.servers![MCP_SERVER_NAME].tools!; +} + +async function addMCPServerTools(page: Page, form: Locator) { + await expect(form.getByText('MCP Servers', { exact: true })).toBeVisible(); + await form.getByRole('button', { name: 'Add MCP Server Tools' }).click(); + + const dialogTitle = page.getByText('MCP Server Tools', { exact: true }); + await expect(dialogTitle).toBeVisible(); + await expect(page.getByText(MCP_SERVER_TITLE, { exact: true })).toBeVisible(); + + await page.getByRole('button', { name: `Add ${MCP_SERVER_TITLE}` }).click(); + await expect(page.getByRole('button', { name: `Remove ${MCP_SERVER_TITLE}` })).toBeVisible({ + timeout: 30000, + }); + + await page.getByRole('button', { name: 'Close dialog' }).click(); + await expect(dialogTitle).toBeHidden(); + + await expect(form.getByText(MCP_SERVER_NAME, { exact: true })).toBeVisible(); + await form.getByText(MCP_SERVER_NAME, { exact: true }).click(); + await expect(form.getByLabel(MCP_TOOL_NAME)).toBeVisible(); +} + +async function expectSelectedMCPServerTools(form: Locator) { + await expect(form.getByText(MCP_SERVER_NAME, { exact: true })).toBeVisible(); + await form.getByText(MCP_SERVER_NAME, { exact: true }).click(); + await expect(form.getByLabel(MCP_TOOL_NAME)).toBeVisible(); +} + +test.describe('agent builder MCP tools', () => { + test('creates an agent with MCP server tools and persists the selection', async ({ page }) => { + test.setTimeout(120000); + + const agentName = uniqueAgentName('E2E MCP Agent'); + let createdAgentId: string | undefined; + + try { + const form = await openAgentBuilder(page); + await waitForMCPTools(page); + + await form.getByLabel('Agent name').fill(agentName); + await form.getByLabel('Agent description').fill(DESCRIPTION); + await form.getByLabel('Agent instructions').fill(INSTRUCTIONS); + await selectMockModel(page, true); + + await addMCPServerTools(page, form); + + const [createResponse] = await Promise.all([ + page.waitForResponse( + (response) => + response.request().method() === 'POST' && + new URL(response.url()).pathname === '/api/agents' && + response.status() === 201, + { timeout: 30000 }, + ), + form.getByRole('button', { name: 'Create' }).click(), + ]); + const createdAgent = (await createResponse.json()) as AgentDetail; + createdAgentId = createdAgent.id; + + await expect( + page.getByText(`Successfully created ${agentName}`, { exact: true }), + ).toBeVisible(); + + const persistedAgent = await waitForPersistedAgent(page, agentName, DESCRIPTION); + expect(persistedAgent).toMatchObject({ + id: createdAgentId, + name: agentName, + description: DESCRIPTION, + instructions: INSTRUCTIONS, + provider: MOCK_ENDPOINTS[0].label, + model: MOCK_ENDPOINTS[0].model, + category: 'general', + }); + expect(persistedAgent.tools).toEqual( + expect.arrayContaining([MCP_SERVER_TOOL_ID, MCP_TOOL_ID]), + ); + expect(persistedAgent.mcpServerNames).toEqual(expect.arrayContaining([MCP_SERVER_NAME])); + + const reopenedForm = await openAgentBuilder(page); + await reopenedForm.getByRole('combobox', { name: 'Agent', exact: true }).click(); + await page.getByRole('option', { name: agentName }).click(); + + await expect(reopenedForm.getByLabel('Agent name')).toHaveValue(agentName); + await expect(reopenedForm.getByLabel('Agent description')).toHaveValue(DESCRIPTION); + await expect(reopenedForm.getByLabel('Agent instructions')).toHaveValue(INSTRUCTIONS); + await expectSelectedMCPServerTools(reopenedForm); + + await reopenedForm.getByRole('button', { name: 'Select Agent' }).click(); + + const response = await sendMessage(page, `hello from ${agentName}`); + expect(response.ok()).toBeTruthy(); + await expect(mockReply(page)).toBeVisible({ timeout: 30000 }); + } finally { + await cleanupAgent(page, createdAgentId); + } + }); +}); diff --git a/e2e/specs/mock/prompts.spec.ts b/e2e/specs/mock/prompts.spec.ts new file mode 100644 index 0000000000..b0e39604c2 --- /dev/null +++ b/e2e/specs/mock/prompts.spec.ts @@ -0,0 +1,203 @@ +import { expect, test } from '@playwright/test'; +import type { Page } from '@playwright/test'; +import { + isAgentsStream, + MOCK_ENDPOINTS, + NEW_CHAT_PATH, + escapeRegExp, + fetchJson, + getAccessToken, + replyPrompt, + replyText, + requestJson, + selectMockEndpoint, +} from './helpers'; + +const DESCRIPTION = 'Use this prompt to verify LibreChat prompt creation in mock e2e tests.'; +const COMMAND = 'e2e-prompt'; + +type Prompt = { + _id?: string; + groupId: string; + prompt: string; + type: 'text' | 'chat'; +}; + +type PromptGroup = { + _id?: string; + name: string; + oneliner?: string; + command?: string; + productionPrompt?: Pick | null; +}; + +type PromptGroupListResponse = { + promptGroups?: PromptGroup[]; +}; + +const uniquePromptName = () => `E2E Prompt ${Date.now()}-${Math.floor(Math.random() * 1e4)}`; + +async function findPromptGroup( + page: Page, + promptName: string, + token: string, +): Promise { + const body = await fetchJson( + page, + `/api/prompts/groups?name=${encodeURIComponent(promptName)}&limit=10`, + token, + ); + return body.promptGroups?.find((group) => group.name === promptName) ?? null; +} + +async function waitForPersistedPrompt( + page: Page, + promptName: string, + expectedPrompt: string, +): Promise<{ group: PromptGroup; prompts: Prompt[] }> { + const token = await getAccessToken(page); + let latestGroup: PromptGroup | null = null; + let latestPrompts: Prompt[] = []; + + for (let attempt = 0; attempt < 20; attempt++) { + const group = await findPromptGroup(page, promptName, token); + if (group?._id) { + latestGroup = await fetchJson( + page, + `/api/prompts/groups/${encodeURIComponent(group._id)}`, + token, + ); + latestPrompts = await fetchJson( + page, + `/api/prompts?groupId=${encodeURIComponent(group._id)}`, + token, + ); + + const hasExpectedPrompt = latestPrompts.some((prompt) => prompt.prompt === expectedPrompt); + if ( + latestGroup.oneliner === DESCRIPTION && + latestGroup.command === COMMAND && + latestGroup.productionPrompt?.prompt === expectedPrompt && + hasExpectedPrompt + ) { + return { group: latestGroup, prompts: latestPrompts }; + } + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + + expect(latestGroup, `Expected prompt group "${promptName}" to be persisted`).not.toBeNull(); + expect(latestGroup?.oneliner).toBe(DESCRIPTION); + expect(latestGroup?.command).toBe(COMMAND); + expect(latestGroup?.productionPrompt?.prompt).toBe(expectedPrompt); + expect(latestPrompts.some((prompt) => prompt.prompt === expectedPrompt)).toBe(true); + return { group: latestGroup!, prompts: latestPrompts }; +} + +async function cleanupPromptGroup(page: Page, groupId?: string) { + if (!groupId) { + return; + } + + const token = await getAccessToken(page); + await requestJson<{ message?: string }>(page, { + path: `/api/prompts/groups/${encodeURIComponent(groupId)}`, + token, + method: 'DELETE', + }); +} + +async function openPromptsPanel(page: Page) { + const promptsButton = page.getByRole('button', { name: 'Prompts', exact: true }); + await expect(promptsButton).toBeVisible(); + if ((await promptsButton.getAttribute('aria-pressed')) !== 'true') { + await promptsButton.click(); + } + await expect(page.getByRole('search')).toBeVisible(); +} + +async function ensureAutoSendPrompts(page: Page) { + const autoSend = page.getByRole('button', { name: 'Send prompts on select' }); + await expect(autoSend).toBeVisible(); + if ((await autoSend.getAttribute('aria-pressed')) !== 'true') { + await autoSend.click(); + } + await expect(autoSend).toHaveAttribute('aria-pressed', 'true'); +} + +test.describe('prompt manager', () => { + test('creates a prompt and can send it from chat', async ({ page }) => { + test.setTimeout(120000); + + const promptName = uniquePromptName(); + const label = promptName.toLowerCase().replace(/[^a-z0-9]+/g, '-'); + const promptText = replyPrompt(label); + let createdGroupId: string | undefined; + + try { + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await openPromptsPanel(page); + + await page.getByRole('link', { name: 'Create Prompt' }).click(); + await expect(page).toHaveURL(/\/prompts\/new$/); + + await page.getByRole('textbox', { name: 'Prompt Name' }).fill(promptName); + await page.getByRole('textbox', { name: 'Prompt text input field' }).fill(promptText); + await page + .getByRole('textbox', { name: 'Optional: Enter a description to display for the prompt' }) + .fill(DESCRIPTION); + await page + .getByRole('textbox', { + name: 'Optional: Enter a command for the prompt or name will be used', + }) + .fill(COMMAND); + + const [createResponse] = await Promise.all([ + page.waitForResponse( + (response) => + response.request().method() === 'POST' && + new URL(response.url()).pathname === '/api/prompts' && + response.status() >= 200 && + response.status() < 300, + { timeout: 30000 }, + ), + page.getByRole('button', { name: 'Create Prompt' }).click(), + ]); + const createdPrompt = (await createResponse.json()) as { + group?: PromptGroup; + prompt?: Prompt; + }; + createdGroupId = createdPrompt.group?._id ?? createdPrompt.prompt?.groupId; + + const { group } = await waitForPersistedPrompt(page, promptName, promptText); + createdGroupId = group._id ?? createdGroupId; + expect(createdGroupId).toBeTruthy(); + await expect(page).toHaveURL(new RegExp(`/prompts/${createdGroupId}$`)); + await expect(page.getByRole('button', { name: `Edit: ${promptName}` })).toBeVisible(); + + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await selectMockEndpoint(page, MOCK_ENDPOINTS[0]); + await openPromptsPanel(page); + await ensureAutoSendPrompts(page); + + await page.getByLabel('Filter prompts by name').fill(promptName); + const promptCard = page.getByRole('button', { + name: new RegExp(`^${escapeRegExp(promptName)} prompt`), + }); + await expect(promptCard).toBeVisible({ timeout: 10000 }); + + const [response] = await Promise.all([ + page.waitForResponse(isAgentsStream, { timeout: 30000 }), + promptCard.click(), + ]); + expect(response.ok()).toBeTruthy(); + + await expect(page.getByTestId('messages-view').getByText(promptText)).toBeVisible(); + await expect(page.getByTestId('messages-view').getByText(replyText(label))).toBeVisible({ + timeout: 30000, + }); + } finally { + await cleanupPromptGroup(page, createdGroupId); + } + }); +});