mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-08-04 14:57:42 +00:00
✅ Add mock e2e coverage for agents, prompts, MCP, and chat flows
This commit is contained in:
teresama
Danny Avila
parent
7791fcfe39
commit
cec5b0d56f
11 changed files with 1063 additions and 29 deletions
|
|
@ -4,6 +4,16 @@
|
|||
version: 1.3.11
|
||||
cache: true
|
||||
|
||||
mcpServers:
|
||||
e2e-memory:
|
||||
type: stdio
|
||||
command: node
|
||||
args:
|
||||
- e2e/setup/fake-mcp-server.js
|
||||
title: E2E Memory
|
||||
description: Local MCP fixture used by mock end-to-end tests.
|
||||
timeout: 30000
|
||||
|
||||
endpoints:
|
||||
custom:
|
||||
- name: 'Mock Provider A'
|
||||
|
|
@ -15,6 +25,8 @@ endpoints:
|
|||
fetch: false
|
||||
titleConvo: false
|
||||
modelDisplayLabel: 'Mock Provider A'
|
||||
customParams:
|
||||
defaultParamsEndpoint: anthropic
|
||||
|
||||
- name: 'Mock Provider B'
|
||||
apiKey: 'e2e-mock-key-b'
|
||||
|
|
|
|||
53
e2e/setup/fake-mcp-server.js
Normal file
53
e2e/setup/fake-mcp-server.js
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
const { McpServer } = require('@modelcontextprotocol/sdk/server/mcp.js');
|
||||
const { StdioServerTransport } = require('@modelcontextprotocol/sdk/server/stdio.js');
|
||||
const z = require('zod/v4');
|
||||
|
||||
const server = new McpServer({
|
||||
name: 'e2e-memory',
|
||||
version: '1.0.0',
|
||||
});
|
||||
|
||||
server.registerTool(
|
||||
'remember_fact',
|
||||
{
|
||||
description: 'Stores a deterministic fact for LibreChat mock end-to-end tests.',
|
||||
inputSchema: {
|
||||
fact: z.string().optional(),
|
||||
},
|
||||
},
|
||||
async ({ fact = 'LibreChat MCP e2e fact' }) => ({
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `E2E MCP memory noted: ${fact}`,
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
'recall_fact',
|
||||
{
|
||||
description: 'Returns a deterministic fact for LibreChat mock end-to-end tests.',
|
||||
inputSchema: {},
|
||||
},
|
||||
async () => ({
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: 'E2E MCP memory recalled: LibreChat can persist MCP tools on agents.',
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
async function main() {
|
||||
await server.connect(new StdioServerTransport());
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error('[fake-mcp-server] failed to start', error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -23,6 +23,7 @@ const COUNTED_REPLY_MARKER = 'E2E_COUNTED_REPLY:';
|
|||
const SLOW_REPLY_MARKER = 'E2E_SLOW_REPLY:';
|
||||
const RESUME_ICON_REPLY_MARKER = 'E2E_RESUME_ICON_REPLY:';
|
||||
const FORCED_ERROR_MARKER = 'E2E_FORCED_ERROR:';
|
||||
const MARKDOWN_REPLY_MARKER = 'E2E_MARKDOWN_REPLY';
|
||||
const CREATE_FILE_AUTHORING_FINAL_TEXT = 'E2E file authoring complete';
|
||||
const EDIT_FILE_AUTHORING_FINAL_TEXT = 'E2E file edit complete';
|
||||
const MODEL_SPEC_SKILL_ASSERTION_FINAL_TEXT = 'E2E model spec skill assertion passed';
|
||||
|
|
@ -228,6 +229,24 @@ function providerFileAssertionResponses({ messages, text }) {
|
|||
}
|
||||
|
||||
function replyResponses(text) {
|
||||
if (text.includes(MARKDOWN_REPLY_MARKER)) {
|
||||
return {
|
||||
responses: [
|
||||
[
|
||||
'## E2E markdown heading',
|
||||
'',
|
||||
'**E2E bold text**',
|
||||
'',
|
||||
'- E2E list item',
|
||||
'',
|
||||
'```javascript',
|
||||
'const e2eSyntaxHighlight = "ok";',
|
||||
'```',
|
||||
].join('\n'),
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
const errorName = getMarkerValue(text, FORCED_ERROR_MARKER);
|
||||
if (errorName) {
|
||||
return {
|
||||
|
|
|
|||
130
e2e/specs/mock/agents.helpers.ts
Normal file
130
e2e/specs/mock/agents.helpers.ts
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
import { expect } from '@playwright/test';
|
||||
import type { Page } from '@playwright/test';
|
||||
import { MOCK_ENDPOINTS, NEW_CHAT_PATH, fetchJson, getAccessToken, requestJson } from './helpers';
|
||||
|
||||
export const AGENT_EDIT_PERMISSION = 2;
|
||||
|
||||
export type AgentSummary = {
|
||||
_id: string;
|
||||
id: string;
|
||||
name?: string;
|
||||
description?: string;
|
||||
provider: string;
|
||||
model: string;
|
||||
category?: string;
|
||||
};
|
||||
|
||||
export type ModelParameters = {
|
||||
maxContextTokens?: number;
|
||||
maxOutputTokens?: number;
|
||||
temperature?: number;
|
||||
topP?: number;
|
||||
topK?: number;
|
||||
thinkingBudget?: number;
|
||||
fileTokenLimit?: number;
|
||||
resendFiles?: boolean;
|
||||
promptCache?: boolean;
|
||||
thinking?: boolean;
|
||||
web_search?: boolean;
|
||||
};
|
||||
|
||||
export type AgentDetail = AgentSummary & {
|
||||
instructions?: string;
|
||||
model_parameters?: ModelParameters;
|
||||
tools?: string[];
|
||||
mcpServerNames?: string[];
|
||||
};
|
||||
|
||||
export const uniqueAgentName = (prefix: string) =>
|
||||
`${prefix} ${Date.now()}-${Math.floor(Math.random() * 1e4)}`;
|
||||
|
||||
export async function findAgent(
|
||||
page: Page,
|
||||
agentName: string,
|
||||
token: string,
|
||||
): Promise<AgentSummary | null> {
|
||||
const body = await fetchJson<{ data?: AgentSummary[] }>(
|
||||
page,
|
||||
`/api/agents?search=${encodeURIComponent(agentName)}&limit=10&requiredPermission=${AGENT_EDIT_PERMISSION}`,
|
||||
token,
|
||||
);
|
||||
return body.data?.find((agent) => agent.name === agentName) ?? null;
|
||||
}
|
||||
|
||||
export async function waitForPersistedAgent(
|
||||
page: Page,
|
||||
agentName: string,
|
||||
description: string,
|
||||
): Promise<AgentDetail> {
|
||||
const token = await getAccessToken(page);
|
||||
let latestAgent: AgentDetail | null = null;
|
||||
|
||||
for (let attempt = 0; attempt < 20; attempt++) {
|
||||
const agent = await findAgent(page, agentName, token);
|
||||
if (agent) {
|
||||
latestAgent = await fetchJson<AgentDetail>(
|
||||
page,
|
||||
`/api/agents/${encodeURIComponent(agent.id)}/expanded`,
|
||||
token,
|
||||
);
|
||||
if (latestAgent.description === description) {
|
||||
return latestAgent;
|
||||
}
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
}
|
||||
|
||||
expect(latestAgent, `Expected agent "${agentName}" to be persisted`).not.toBeNull();
|
||||
expect(latestAgent?.description).toBe(description);
|
||||
return latestAgent!;
|
||||
}
|
||||
|
||||
export async function openAgentBuilder(page: Page) {
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
|
||||
const form = page.getByRole('form', { name: 'Agent configuration form' });
|
||||
const builderVisible = await form
|
||||
.waitFor({ state: 'visible', timeout: 1000 })
|
||||
.then(() => true)
|
||||
.catch(() => false);
|
||||
if (!builderVisible) {
|
||||
const agentBuilderButton = page.getByRole('button', { name: 'Agent Builder' });
|
||||
await expect(agentBuilderButton).toBeVisible();
|
||||
if ((await agentBuilderButton.getAttribute('aria-pressed')) !== 'true') {
|
||||
await agentBuilderButton.click();
|
||||
}
|
||||
}
|
||||
await expect(form).toBeVisible();
|
||||
return form;
|
||||
}
|
||||
|
||||
export async function selectMockModel(page: Page, clickBackToBuilder = false) {
|
||||
const form = page.getByRole('form', { name: 'Agent configuration form' });
|
||||
|
||||
await form.locator('label[for="provider"] + button').click();
|
||||
await expect(form.getByText('Model Parameters', { exact: true })).toBeVisible();
|
||||
|
||||
await form.getByRole('combobox', { name: 'Provider' }).click();
|
||||
await page.getByRole('option', { name: MOCK_ENDPOINTS[0].label }).click();
|
||||
|
||||
await form.getByRole('combobox', { name: 'Model' }).click();
|
||||
await page.getByRole('option', { name: MOCK_ENDPOINTS[0].model, exact: true }).click();
|
||||
|
||||
if (clickBackToBuilder) {
|
||||
await form.getByRole('button', { name: 'Back to builder' }).click();
|
||||
await expect(page.getByRole('form', { name: 'Agent configuration form' })).toBeVisible();
|
||||
}
|
||||
}
|
||||
|
||||
export async function cleanupAgent(page: Page, agentId?: string) {
|
||||
if (!agentId) {
|
||||
return;
|
||||
}
|
||||
|
||||
const token = await getAccessToken(page);
|
||||
await requestJson<{ message?: string }>(page, {
|
||||
path: `/api/agents/${encodeURIComponent(agentId)}`,
|
||||
token,
|
||||
method: 'DELETE',
|
||||
});
|
||||
}
|
||||
195
e2e/specs/mock/agents.spec.ts
Normal file
195
e2e/specs/mock/agents.spec.ts
Normal file
|
|
@ -0,0 +1,195 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import type { Locator, Page } from '@playwright/test';
|
||||
import type { AgentDetail } from './agents.helpers';
|
||||
import {
|
||||
cleanupAgent,
|
||||
openAgentBuilder,
|
||||
selectMockModel,
|
||||
uniqueAgentName,
|
||||
waitForPersistedAgent,
|
||||
} from './agents.helpers';
|
||||
import { MOCK_ENDPOINTS, mockReply, sendMessage } from './helpers';
|
||||
|
||||
const DESCRIPTION = 'Use this agent to verify LibreChat agent creation in mock end-to-end tests.';
|
||||
const INSTRUCTIONS =
|
||||
'Reply through the mock e2e model and keep this agent available for UI persistence checks.';
|
||||
const MODEL_PARAMETERS = {
|
||||
maxContextTokens: 32000,
|
||||
maxOutputTokens: 4096,
|
||||
temperature: 0.25,
|
||||
topP: 0.8,
|
||||
topK: 12,
|
||||
resendFiles: false,
|
||||
promptCache: true,
|
||||
thinking: true,
|
||||
thinkingBudget: 2000,
|
||||
web_search: true,
|
||||
fileTokenLimit: 12000,
|
||||
};
|
||||
const PERSISTED_MODEL_PARAMETERS = {
|
||||
maxContextTokens: MODEL_PARAMETERS.maxContextTokens,
|
||||
maxOutputTokens: MODEL_PARAMETERS.maxOutputTokens,
|
||||
temperature: MODEL_PARAMETERS.temperature,
|
||||
topP: MODEL_PARAMETERS.topP,
|
||||
topK: MODEL_PARAMETERS.topK,
|
||||
resendFiles: MODEL_PARAMETERS.resendFiles,
|
||||
web_search: MODEL_PARAMETERS.web_search,
|
||||
fileTokenLimit: MODEL_PARAMETERS.fileTokenLimit,
|
||||
};
|
||||
|
||||
async function setSwitch(form: Locator, name: string, checked: boolean) {
|
||||
const control = form.getByRole('switch', { name, exact: true });
|
||||
await expect(control).toBeVisible();
|
||||
if ((await control.getAttribute('aria-checked')) !== String(checked)) {
|
||||
await control.click();
|
||||
}
|
||||
await expect(control).toHaveAttribute('aria-checked', String(checked));
|
||||
}
|
||||
|
||||
async function fillAnthropicStyleModelParameters(page: Page) {
|
||||
const form = page.getByRole('form', { name: 'Agent configuration form' });
|
||||
|
||||
await expect(form.getByLabel('Max Context Tokens')).toBeVisible();
|
||||
await expect(form.getByLabel('Max Output Tokens')).toBeVisible();
|
||||
await expect(form.getByText('Temperature', { exact: true })).toBeVisible();
|
||||
await expect(form.getByText('Top P', { exact: true })).toBeVisible();
|
||||
await expect(form.getByText('Top K', { exact: true })).toBeVisible();
|
||||
await expect(form.getByText('Effort', { exact: true })).toBeVisible();
|
||||
await expect(form.getByText('Thought Visibility', { exact: true })).toBeVisible();
|
||||
|
||||
await form
|
||||
.locator('#maxContextTokens-dynamic-input')
|
||||
.fill(`${MODEL_PARAMETERS.maxContextTokens}`);
|
||||
await form.locator('#maxOutputTokens-dynamic-input').fill(`${MODEL_PARAMETERS.maxOutputTokens}`);
|
||||
await form
|
||||
.locator('#temperature-dynamic-setting-input-number')
|
||||
.fill(`${MODEL_PARAMETERS.temperature}`);
|
||||
await form.locator('#topP-dynamic-setting-input-number').fill(`${MODEL_PARAMETERS.topP}`);
|
||||
await form.locator('#topK-dynamic-setting-input-number').fill(`${MODEL_PARAMETERS.topK}`);
|
||||
await form.locator('#thinkingBudget-dynamic-input').fill(`${MODEL_PARAMETERS.thinkingBudget}`);
|
||||
await form.locator('#fileTokenLimit-dynamic-input').fill(`${MODEL_PARAMETERS.fileTokenLimit}`);
|
||||
|
||||
await setSwitch(form, 'Resend Files', MODEL_PARAMETERS.resendFiles);
|
||||
await setSwitch(form, 'Use Prompt Caching', MODEL_PARAMETERS.promptCache);
|
||||
await setSwitch(form, 'Thinking', MODEL_PARAMETERS.thinking);
|
||||
await setSwitch(form, 'Web Search', MODEL_PARAMETERS.web_search);
|
||||
|
||||
// DynamicInput/DynamicSlider values use a 450ms debounced form update.
|
||||
await page.waitForTimeout(600);
|
||||
}
|
||||
|
||||
async function expectAnthropicStyleModelParameters(page: Page) {
|
||||
const form = page.getByRole('form', { name: 'Agent configuration form' });
|
||||
|
||||
await expect(form.locator('#maxContextTokens-dynamic-input')).toHaveValue(
|
||||
`${MODEL_PARAMETERS.maxContextTokens}`,
|
||||
);
|
||||
await expect(form.locator('#maxOutputTokens-dynamic-input')).toHaveValue(
|
||||
`${MODEL_PARAMETERS.maxOutputTokens}`,
|
||||
);
|
||||
await expect(form.locator('#temperature-dynamic-setting-input-number')).toHaveValue(
|
||||
`${MODEL_PARAMETERS.temperature}`,
|
||||
);
|
||||
await expect(form.locator('#topP-dynamic-setting-input-number')).toHaveValue(
|
||||
MODEL_PARAMETERS.topP.toFixed(2),
|
||||
);
|
||||
await expect(form.locator('#topK-dynamic-setting-input-number')).toHaveValue(
|
||||
`${MODEL_PARAMETERS.topK}`,
|
||||
);
|
||||
await expect(form.locator('#thinkingBudget-dynamic-input')).toHaveValue(
|
||||
`${MODEL_PARAMETERS.thinkingBudget}`,
|
||||
);
|
||||
await expect(form.locator('#fileTokenLimit-dynamic-input')).toHaveValue(
|
||||
`${MODEL_PARAMETERS.fileTokenLimit}`,
|
||||
);
|
||||
|
||||
await expect(form.getByRole('switch', { name: 'Resend Files', exact: true })).toHaveAttribute(
|
||||
'aria-checked',
|
||||
String(MODEL_PARAMETERS.resendFiles),
|
||||
);
|
||||
await expect(
|
||||
form.getByRole('switch', { name: 'Use Prompt Caching', exact: true }),
|
||||
).toHaveAttribute('aria-checked', String(MODEL_PARAMETERS.promptCache));
|
||||
await expect(form.getByRole('switch', { name: 'Thinking', exact: true })).toHaveAttribute(
|
||||
'aria-checked',
|
||||
String(MODEL_PARAMETERS.thinking),
|
||||
);
|
||||
await expect(form.getByRole('switch', { name: 'Web Search', exact: true })).toHaveAttribute(
|
||||
'aria-checked',
|
||||
String(MODEL_PARAMETERS.web_search),
|
||||
);
|
||||
}
|
||||
|
||||
test.describe('agent builder', () => {
|
||||
test('creates an agent, persists its configuration, and can chat with it', async ({ page }) => {
|
||||
test.setTimeout(120000);
|
||||
|
||||
const agentName = uniqueAgentName('E2E Agent');
|
||||
let createdAgentId: string | undefined;
|
||||
|
||||
try {
|
||||
let form = await openAgentBuilder(page);
|
||||
|
||||
await form.getByLabel('Agent name').fill(agentName);
|
||||
await form.getByLabel('Agent description').fill(DESCRIPTION);
|
||||
await form.getByLabel('Agent instructions').fill(INSTRUCTIONS);
|
||||
|
||||
await selectMockModel(page);
|
||||
await fillAnthropicStyleModelParameters(page);
|
||||
await page
|
||||
.getByRole('form', { name: 'Agent configuration form' })
|
||||
.getByRole('button', {
|
||||
name: 'Back to builder',
|
||||
})
|
||||
.click();
|
||||
form = page.getByRole('form', { name: 'Agent configuration form' });
|
||||
|
||||
const [createResponse] = await Promise.all([
|
||||
page.waitForResponse(
|
||||
(response) =>
|
||||
response.request().method() === 'POST' &&
|
||||
new URL(response.url()).pathname === '/api/agents' &&
|
||||
response.status() === 201,
|
||||
{ timeout: 30000 },
|
||||
),
|
||||
form.getByRole('button', { name: 'Create' }).click(),
|
||||
]);
|
||||
const createdAgent = (await createResponse.json()) as AgentDetail;
|
||||
createdAgentId = createdAgent.id;
|
||||
|
||||
await expect(page.getByText(`Successfully created ${agentName}`)).toBeVisible();
|
||||
|
||||
const persistedAgent = await waitForPersistedAgent(page, agentName, DESCRIPTION);
|
||||
expect(persistedAgent).toMatchObject({
|
||||
id: createdAgentId,
|
||||
name: agentName,
|
||||
description: DESCRIPTION,
|
||||
instructions: INSTRUCTIONS,
|
||||
provider: MOCK_ENDPOINTS[0].label,
|
||||
model: MOCK_ENDPOINTS[0].model,
|
||||
category: 'general',
|
||||
});
|
||||
expect(persistedAgent.model_parameters).toMatchObject(PERSISTED_MODEL_PARAMETERS);
|
||||
|
||||
form = await openAgentBuilder(page);
|
||||
await form.getByRole('combobox', { name: 'Agent', exact: true }).click();
|
||||
await page.getByRole('option', { name: agentName }).click();
|
||||
|
||||
await expect(form.getByLabel('Agent name')).toHaveValue(agentName);
|
||||
await expect(form.getByLabel('Agent description')).toHaveValue(DESCRIPTION);
|
||||
await expect(form.getByLabel('Agent instructions')).toHaveValue(INSTRUCTIONS);
|
||||
|
||||
await form.locator('label[for="provider"] + button').click();
|
||||
await expectAnthropicStyleModelParameters(page);
|
||||
await form.getByRole('button', { name: 'Back to builder' }).click();
|
||||
|
||||
await form.getByRole('button', { name: 'Select Agent' }).click();
|
||||
|
||||
const response = await sendMessage(page, `hello from ${agentName}`);
|
||||
expect(response.ok()).toBeTruthy();
|
||||
await expect(mockReply(page)).toBeVisible({ timeout: 30000 });
|
||||
} finally {
|
||||
await cleanupAgent(page, createdAgentId);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
@ -1,10 +1,12 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import { getPrimaryE2EUser } from '../../setup/users.mock';
|
||||
import { NEW_CHAT_PATH } from './helpers';
|
||||
|
||||
test.describe('app loads cleanly', () => {
|
||||
test('authenticated user lands on a rendered chat view without runtime errors', async ({
|
||||
page,
|
||||
}) => {
|
||||
const user = getPrimaryE2EUser();
|
||||
const pageErrors: string[] = [];
|
||||
page.on('pageerror', (error) => pageErrors.push(error.message));
|
||||
|
||||
|
|
@ -12,6 +14,7 @@ test.describe('app loads cleanly', () => {
|
|||
|
||||
await expect(page).toHaveURL(/\/c\/new$/);
|
||||
await expect(page.getByRole('main')).toBeVisible();
|
||||
await expect(page.getByRole('main')).toContainText(user.name);
|
||||
await expect(page.getByRole('textbox', { name: 'Message input' })).toBeVisible();
|
||||
await expect(page.getByRole('button', { name: 'Select a model' }).first()).toBeVisible();
|
||||
await expect(page.getByTestId('nav-user')).toBeVisible();
|
||||
|
|
|
|||
|
|
@ -1,13 +1,83 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import type { Page } from '@playwright/test';
|
||||
import {
|
||||
isAgentsStream,
|
||||
MOCK_ENDPOINTS,
|
||||
NEW_CHAT_PATH,
|
||||
messagesView,
|
||||
mockReply,
|
||||
selectMockEndpoint,
|
||||
sendMessage,
|
||||
} from './helpers';
|
||||
|
||||
type UploadFixture = {
|
||||
name: string;
|
||||
mimeType: string;
|
||||
buffer: Buffer;
|
||||
};
|
||||
|
||||
const pdfFixture: UploadFixture = {
|
||||
name: 'provider-context.pdf',
|
||||
mimeType: 'application/pdf',
|
||||
buffer: Buffer.from(
|
||||
`%PDF-1.4
|
||||
1 0 obj
|
||||
<< /Type /Catalog /Pages 2 0 R >>
|
||||
endobj
|
||||
2 0 obj
|
||||
<< /Type /Pages /Count 0 >>
|
||||
endobj
|
||||
trailer
|
||||
<< /Root 1 0 R >>
|
||||
%%EOF
|
||||
`,
|
||||
),
|
||||
};
|
||||
|
||||
const textFixture: UploadFixture = {
|
||||
name: 'provider-context.txt',
|
||||
mimeType: 'text/plain',
|
||||
buffer: Buffer.from('This text attachment should be available to the mock model.\n'),
|
||||
};
|
||||
|
||||
const imageFixture: UploadFixture = {
|
||||
name: 'provider-context.png',
|
||||
mimeType: 'image/png',
|
||||
buffer: Buffer.from(
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=',
|
||||
'base64',
|
||||
),
|
||||
};
|
||||
|
||||
const composer = (page: Page) => page.locator('form');
|
||||
|
||||
async function openProviderFileChooser(page: Page) {
|
||||
await page.getByRole('button', { name: 'Attach File Options' }).click();
|
||||
await expect(page.getByText('Upload to Provider')).toBeVisible();
|
||||
|
||||
const fileChooserPromise = page.waitForEvent('filechooser');
|
||||
await page.getByText('Upload to Provider').click();
|
||||
const fileChooser = await fileChooserPromise;
|
||||
expect(await fileChooser.element().getAttribute('type')).toBe('file');
|
||||
return fileChooser;
|
||||
}
|
||||
|
||||
async function uploadProviderFile(page: Page, fixture: UploadFixture) {
|
||||
const fileChooser = await openProviderFileChooser(page);
|
||||
const uploadResponsePromise = page.waitForResponse(
|
||||
(response) =>
|
||||
response.url().includes('/api/files') &&
|
||||
response.request().method() === 'POST' &&
|
||||
response.status() === 200,
|
||||
{ timeout: 30000 },
|
||||
);
|
||||
await fileChooser.setFiles(fixture);
|
||||
const uploadResponse = await uploadResponsePromise;
|
||||
expect(uploadResponse.ok()).toBeTruthy();
|
||||
await page.waitForTimeout(350);
|
||||
return uploadResponse;
|
||||
}
|
||||
|
||||
test.describe('core chat loop', () => {
|
||||
test('streams a response, saves the conversation, and persists across reload', async ({
|
||||
page,
|
||||
|
|
@ -22,6 +92,11 @@ test.describe('core chat loop', () => {
|
|||
expect(response.ok()).toBeTruthy();
|
||||
|
||||
await expect(page.getByText(userMessage)).toBeVisible();
|
||||
const userMessageTurn = messagesView(page)
|
||||
.locator('.message-render')
|
||||
.filter({ hasText: userMessage });
|
||||
await expect(userMessageTurn.locator('.user-turn')).toBeVisible();
|
||||
await expect(userMessageTurn.locator('.agent-turn')).toHaveCount(0);
|
||||
await expect(mockReply(page)).toBeVisible();
|
||||
|
||||
await expect(page).toHaveURL(/\/c\/[0-9a-fA-F-]{36}$/);
|
||||
|
|
@ -36,6 +111,48 @@ test.describe('core chat loop', () => {
|
|||
await expect(page.getByTestId('convo-item').first()).toBeVisible();
|
||||
});
|
||||
|
||||
test('keeps send disabled until the composer has message text', async ({ page }) => {
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
|
||||
|
||||
const input = page.getByRole('textbox', { name: 'Message input' });
|
||||
const sendButton = page.getByTestId('send-button');
|
||||
|
||||
await expect(sendButton).toBeDisabled();
|
||||
await input.fill('ready to send');
|
||||
await expect(sendButton).toBeEnabled();
|
||||
await input.fill(' ');
|
||||
await expect(sendButton).toBeDisabled();
|
||||
});
|
||||
|
||||
test('renders assistant markdown and syntax-highlighted code blocks', async ({ page }) => {
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
|
||||
|
||||
const response = await sendMessage(page, 'E2E_MARKDOWN_REPLY');
|
||||
expect(response.ok()).toBeTruthy();
|
||||
|
||||
const assistantMessage = messagesView(page)
|
||||
.locator('.message-render')
|
||||
.filter({ hasText: 'E2E markdown heading' })
|
||||
.last();
|
||||
await expect(assistantMessage.locator('.agent-turn')).toBeVisible();
|
||||
await expect(
|
||||
assistantMessage.getByRole('heading', { name: 'E2E markdown heading' }),
|
||||
).toBeVisible();
|
||||
await expect(
|
||||
assistantMessage.locator('strong').filter({ hasText: 'E2E bold text' }),
|
||||
).toBeVisible();
|
||||
await expect(
|
||||
assistantMessage.getByRole('listitem').filter({ hasText: 'E2E list item' }),
|
||||
).toBeVisible();
|
||||
|
||||
const codeBlock = assistantMessage.locator('code').filter({ hasText: 'e2eSyntaxHighlight' });
|
||||
await expect(codeBlock).toBeVisible();
|
||||
await expect(codeBlock).toHaveClass(/hljs/);
|
||||
await expect(codeBlock).toHaveClass(/language-javascript/);
|
||||
});
|
||||
|
||||
test('can switch back to the previous branch after regenerating an earlier response', async ({
|
||||
page,
|
||||
}) => {
|
||||
|
|
@ -54,10 +171,7 @@ test.describe('core chat loop', () => {
|
|||
expect(response.ok()).toBeTruthy();
|
||||
await expect(page.getByText(followUpMessage)).toBeVisible();
|
||||
|
||||
const firstAssistantMessage = page
|
||||
.getByTestId('messages-view')
|
||||
.locator('.message-render')
|
||||
.nth(1);
|
||||
const firstAssistantMessage = messagesView(page).locator('.message-render').nth(1);
|
||||
await firstAssistantMessage.hover();
|
||||
const regenerateButton = firstAssistantMessage.locator('button[title="Regenerate"]').last();
|
||||
await expect(regenerateButton).toBeVisible();
|
||||
|
|
@ -79,31 +193,19 @@ test.describe('core chat loop', () => {
|
|||
}) => {
|
||||
test.setTimeout(90000);
|
||||
|
||||
const filename = 'provider-upload.csv';
|
||||
const csvFixture: UploadFixture = {
|
||||
name: 'provider-upload.csv',
|
||||
mimeType: 'text/csv',
|
||||
buffer: Buffer.from('name,value\nalpha,1\n'),
|
||||
};
|
||||
const filename = csvFixture.name;
|
||||
const assertionText = `E2E_ASSERT_PROVIDER_FILE:${filename}`;
|
||||
const fileChip = page.getByTestId('messages-view').getByRole('button', { name: filename });
|
||||
const fileChip = messagesView(page).getByRole('button', { name: filename });
|
||||
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
|
||||
|
||||
await page.getByRole('button', { name: 'Attach File Options' }).click();
|
||||
const fileChooserPromise = page.waitForEvent('filechooser');
|
||||
await page.getByText('Upload to Provider').click();
|
||||
const fileChooser = await fileChooserPromise;
|
||||
|
||||
const uploadResponsePromise = page.waitForResponse(
|
||||
(response) =>
|
||||
response.url().includes('/api/files') &&
|
||||
response.request().method() === 'POST' &&
|
||||
response.status() === 200,
|
||||
{ timeout: 30000 },
|
||||
);
|
||||
await fileChooser.setFiles({
|
||||
name: filename,
|
||||
mimeType: 'text/csv',
|
||||
buffer: Buffer.from('name,value\nalpha,1\n'),
|
||||
});
|
||||
await uploadResponsePromise;
|
||||
await uploadProviderFile(page, csvFixture);
|
||||
|
||||
await expect(page.getByRole('button', { name: filename })).toBeVisible();
|
||||
|
||||
|
|
@ -119,9 +221,7 @@ test.describe('core chat loop', () => {
|
|||
expect(response.ok()).toBeTruthy();
|
||||
|
||||
await expect(
|
||||
page
|
||||
.getByTestId('messages-view')
|
||||
.getByText(`E2E provider file assertion passed: ${filename}`),
|
||||
messagesView(page).getByText(`E2E provider file assertion passed: ${filename}`),
|
||||
).toBeVisible();
|
||||
await expect(fileChip).toBeVisible();
|
||||
|
||||
|
|
@ -131,4 +231,46 @@ test.describe('core chat loop', () => {
|
|||
await expect(page).toHaveURL(conversationUrl);
|
||||
await expect(fileChip).toBeVisible();
|
||||
});
|
||||
|
||||
test('supports attaching, removing, and sending provider files from the composer', async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(90000);
|
||||
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
|
||||
|
||||
await uploadProviderFile(page, pdfFixture);
|
||||
const pdfChip = composer(page).getByRole('button', { name: pdfFixture.name });
|
||||
await expect(pdfChip).toBeVisible();
|
||||
|
||||
await composer(page).getByRole('button', { name: 'Remove file' }).click();
|
||||
await expect(pdfChip).toHaveCount(0);
|
||||
|
||||
await uploadProviderFile(page, imageFixture);
|
||||
await expect(
|
||||
composer(page).getByRole('button', { name: 'View Preview image in full size' }),
|
||||
).toBeVisible();
|
||||
|
||||
await uploadProviderFile(page, textFixture);
|
||||
const textChip = composer(page).getByRole('button', { name: textFixture.name });
|
||||
await expect(textChip).toBeVisible();
|
||||
|
||||
const assertionText = `E2E_ASSERT_PROVIDER_FILE:${textFixture.name}`;
|
||||
const input = page.getByRole('textbox', { name: 'Message input' });
|
||||
await input.click();
|
||||
await input.fill(assertionText);
|
||||
await expect(page.getByTestId('send-button')).toBeEnabled();
|
||||
|
||||
const [response] = await Promise.all([
|
||||
page.waitForResponse(isAgentsStream, { timeout: 30000 }),
|
||||
page.getByTestId('send-button').click(),
|
||||
]);
|
||||
expect(response.ok()).toBeTruthy();
|
||||
|
||||
await expect(
|
||||
messagesView(page).getByText(`E2E provider file assertion passed: ${textFixture.name}`),
|
||||
).toBeVisible();
|
||||
await expect(messagesView(page).getByRole('button', { name: textFixture.name })).toBeVisible();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
120
e2e/specs/mock/conversation-management.spec.ts
Normal file
120
e2e/specs/mock/conversation-management.spec.ts
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import type { Locator, Page } from '@playwright/test';
|
||||
import {
|
||||
MOCK_ENDPOINTS,
|
||||
NEW_CHAT_PATH,
|
||||
messagesView,
|
||||
replyPrompt,
|
||||
replyText,
|
||||
selectMockEndpoint,
|
||||
sendMessage,
|
||||
} from './helpers';
|
||||
|
||||
const firstConversation = (page: Page) => page.getByTestId('convo-item').first();
|
||||
|
||||
function uniqueLabel(prefix: string) {
|
||||
return `${prefix}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
}
|
||||
|
||||
async function openMockChat(page: Page) {
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
|
||||
}
|
||||
|
||||
async function sendAndExpectReply(page: Page, label: string) {
|
||||
const prompt = replyPrompt(label);
|
||||
const reply = replyText(label);
|
||||
const response = await sendMessage(page, prompt);
|
||||
expect(response.ok()).toBeTruthy();
|
||||
await expect(messagesView(page).getByText(prompt)).toBeVisible();
|
||||
await expect(messagesView(page).getByText(reply)).toBeVisible();
|
||||
return { prompt, reply };
|
||||
}
|
||||
|
||||
async function openConversationMenu(conversation: Locator) {
|
||||
await conversation.hover();
|
||||
await conversation.getByRole('button', { name: 'Conversation Menu Options' }).click();
|
||||
}
|
||||
|
||||
async function renameConversation(page: Page, conversation: Locator, title: string) {
|
||||
await openConversationMenu(conversation);
|
||||
await page.getByRole('menuitem', { name: 'Rename' }).click();
|
||||
const titleInput = conversation.getByRole('textbox', { name: 'New Conversation Title' });
|
||||
await expect(titleInput).toBeVisible();
|
||||
await titleInput.fill(title);
|
||||
await conversation.getByRole('button', { name: 'Save' }).click();
|
||||
await expect(conversation).toContainText(title);
|
||||
}
|
||||
|
||||
test.describe('conversation management', () => {
|
||||
test('loads a past sidebar conversation with its message history', async ({ page }) => {
|
||||
const firstLabel = uniqueLabel('sidebar-history-first');
|
||||
const secondLabel = uniqueLabel('sidebar-history-second');
|
||||
|
||||
await openMockChat(page);
|
||||
const firstTurn = await sendAndExpectReply(page, firstLabel);
|
||||
const secondTurn = await sendAndExpectReply(page, secondLabel);
|
||||
const conversationUrl = page.url();
|
||||
|
||||
await expect(page).toHaveURL(/\/c\/[0-9a-fA-F-]{36}$/);
|
||||
await expect(firstConversation(page)).toBeVisible();
|
||||
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
await expect(page).toHaveURL(/\/c\/new$/);
|
||||
await expect(messagesView(page).getByText(firstTurn.prompt)).toHaveCount(0);
|
||||
await expect(messagesView(page).getByText(secondTurn.prompt)).toHaveCount(0);
|
||||
|
||||
await firstConversation(page).click();
|
||||
await expect(page).toHaveURL(conversationUrl);
|
||||
await expect(messagesView(page).getByText(firstTurn.prompt)).toBeVisible();
|
||||
await expect(messagesView(page).getByText(firstTurn.reply)).toBeVisible();
|
||||
await expect(messagesView(page).getByText(secondTurn.prompt)).toBeVisible();
|
||||
await expect(messagesView(page).getByText(secondTurn.reply)).toBeVisible();
|
||||
});
|
||||
|
||||
test('renames a conversation from the sidebar', async ({ page }) => {
|
||||
const label = uniqueLabel('sidebar-rename');
|
||||
const renamedTitle = `Renamed ${label}`;
|
||||
|
||||
await openMockChat(page);
|
||||
await sendAndExpectReply(page, label);
|
||||
|
||||
await renameConversation(page, firstConversation(page), renamedTitle);
|
||||
await page.reload({ timeout: 10000 });
|
||||
await expect(page.getByTestId('convo-item').filter({ hasText: renamedTitle })).toBeVisible();
|
||||
});
|
||||
|
||||
test('deletes a conversation from the sidebar and blocks direct URL access', async ({ page }) => {
|
||||
const label = uniqueLabel('sidebar-delete');
|
||||
const renamedTitle = `Delete ${label}`;
|
||||
|
||||
await openMockChat(page);
|
||||
const turn = await sendAndExpectReply(page, label);
|
||||
const conversationUrl = page.url();
|
||||
|
||||
const conversation = firstConversation(page);
|
||||
await renameConversation(page, conversation, renamedTitle);
|
||||
await openConversationMenu(conversation);
|
||||
await page.getByRole('menuitem', { name: 'Delete' }).click();
|
||||
|
||||
const dialog = page.getByRole('dialog', { name: 'Delete chat?' });
|
||||
await expect(dialog).toBeVisible();
|
||||
const [deleteResponse] = await Promise.all([
|
||||
page.waitForResponse(
|
||||
(response) =>
|
||||
response.request().method() === 'DELETE' && response.url().includes('/api/convos'),
|
||||
{ timeout: 30000 },
|
||||
),
|
||||
dialog.getByRole('button', { name: 'Delete' }).click(),
|
||||
]);
|
||||
expect(deleteResponse.ok()).toBeTruthy();
|
||||
|
||||
await expect(page).toHaveURL(/\/c\/new$/);
|
||||
await expect(page.getByTestId('convo-item').filter({ hasText: renamedTitle })).toHaveCount(0);
|
||||
|
||||
await page.goto(conversationUrl, { timeout: 10000 });
|
||||
await expect(page.getByRole('textbox', { name: 'Message input' })).toBeVisible();
|
||||
await expect(messagesView(page).getByText(turn.prompt)).toHaveCount(0);
|
||||
await expect(messagesView(page).getByText(turn.reply)).toHaveCount(0);
|
||||
});
|
||||
});
|
||||
|
|
@ -36,7 +36,7 @@ export function isAgentGenerationStart(response: Response) {
|
|||
const modelSelectorTrigger = (page: Page) =>
|
||||
page.getByRole('button', { name: 'Select a model' }).first();
|
||||
|
||||
const escapeRegExp = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
export const escapeRegExp = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
|
||||
/** Open the model selector, choose an endpoint, then its model (committed on the model click). */
|
||||
export async function selectMockEndpoint(page: Page, endpoint: MockEndpoint) {
|
||||
|
|
@ -70,9 +70,16 @@ export async function enableSkills(page: Page) {
|
|||
await expect(page.getByRole('button', { name: 'Skills' })).toBeVisible();
|
||||
}
|
||||
|
||||
/** The conversation messages container. */
|
||||
export const messagesView = (page: Page) => page.getByTestId('messages-view');
|
||||
|
||||
/** Build the mock-model reply trigger and its expected rendered text for a label. */
|
||||
export const replyPrompt = (label: string) => `E2E_REPLY:${label}`;
|
||||
export const replyText = (label: string) => `E2E reply ${label}`;
|
||||
|
||||
/** The mock reply as rendered in the conversation, scoped to the messages view. */
|
||||
export function mockReply(page: Page) {
|
||||
return page.getByTestId('messages-view').getByText(new RegExp(MOCK_REPLY_TEXT, 'i'));
|
||||
return messagesView(page).getByText(new RegExp(MOCK_REPLY_TEXT, 'i'));
|
||||
}
|
||||
|
||||
/** Type a message, send it, and wait for the streamed `/api/agents` response. */
|
||||
|
|
|
|||
150
e2e/specs/mock/mcp.spec.ts
Normal file
150
e2e/specs/mock/mcp.spec.ts
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import type { Locator, Page } from '@playwright/test';
|
||||
import type { AgentDetail } from './agents.helpers';
|
||||
import {
|
||||
cleanupAgent,
|
||||
openAgentBuilder,
|
||||
selectMockModel,
|
||||
uniqueAgentName,
|
||||
waitForPersistedAgent,
|
||||
} from './agents.helpers';
|
||||
import { MOCK_ENDPOINTS, fetchJson, getAccessToken, mockReply, sendMessage } from './helpers';
|
||||
|
||||
const MCP_SERVER_NAME = 'e2e-memory';
|
||||
const MCP_SERVER_TITLE = 'E2E Memory';
|
||||
const MCP_TOOL_NAME = 'remember_fact';
|
||||
const MCP_SERVER_TOOL_ID = `sys__server__sys_mcp_${MCP_SERVER_NAME}`;
|
||||
const MCP_TOOL_ID = `${MCP_TOOL_NAME}_mcp_${MCP_SERVER_NAME}`;
|
||||
const DESCRIPTION = 'Use this agent to verify LibreChat MCP tool selection in mock e2e tests.';
|
||||
const INSTRUCTIONS =
|
||||
'Keep the selected MCP server tools available while replying through the mock e2e model.';
|
||||
|
||||
type MCPToolsResponse = {
|
||||
servers?: Record<
|
||||
string,
|
||||
{
|
||||
tools?: Array<{
|
||||
name: string;
|
||||
pluginKey: string;
|
||||
description?: string;
|
||||
}>;
|
||||
}
|
||||
>;
|
||||
};
|
||||
|
||||
async function waitForMCPTools(page: Page) {
|
||||
const token = await getAccessToken(page);
|
||||
let latestTools: MCPToolsResponse | null = null;
|
||||
|
||||
for (let attempt = 0; attempt < 20; attempt++) {
|
||||
latestTools = await fetchJson<MCPToolsResponse>(page, '/api/mcp/tools', token);
|
||||
const serverTools = latestTools.servers?.[MCP_SERVER_NAME]?.tools ?? [];
|
||||
if (serverTools.some((tool) => tool.pluginKey === MCP_TOOL_ID)) {
|
||||
return serverTools;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
}
|
||||
|
||||
expect(
|
||||
latestTools?.servers?.[MCP_SERVER_NAME]?.tools,
|
||||
`Expected ${MCP_SERVER_NAME} to expose ${MCP_TOOL_ID}`,
|
||||
).toEqual(expect.arrayContaining([expect.objectContaining({ pluginKey: MCP_TOOL_ID })]));
|
||||
return latestTools!.servers![MCP_SERVER_NAME].tools!;
|
||||
}
|
||||
|
||||
async function addMCPServerTools(page: Page, form: Locator) {
|
||||
await expect(form.getByText('MCP Servers', { exact: true })).toBeVisible();
|
||||
await form.getByRole('button', { name: 'Add MCP Server Tools' }).click();
|
||||
|
||||
const dialogTitle = page.getByText('MCP Server Tools', { exact: true });
|
||||
await expect(dialogTitle).toBeVisible();
|
||||
await expect(page.getByText(MCP_SERVER_TITLE, { exact: true })).toBeVisible();
|
||||
|
||||
await page.getByRole('button', { name: `Add ${MCP_SERVER_TITLE}` }).click();
|
||||
await expect(page.getByRole('button', { name: `Remove ${MCP_SERVER_TITLE}` })).toBeVisible({
|
||||
timeout: 30000,
|
||||
});
|
||||
|
||||
await page.getByRole('button', { name: 'Close dialog' }).click();
|
||||
await expect(dialogTitle).toBeHidden();
|
||||
|
||||
await expect(form.getByText(MCP_SERVER_NAME, { exact: true })).toBeVisible();
|
||||
await form.getByText(MCP_SERVER_NAME, { exact: true }).click();
|
||||
await expect(form.getByLabel(MCP_TOOL_NAME)).toBeVisible();
|
||||
}
|
||||
|
||||
async function expectSelectedMCPServerTools(form: Locator) {
|
||||
await expect(form.getByText(MCP_SERVER_NAME, { exact: true })).toBeVisible();
|
||||
await form.getByText(MCP_SERVER_NAME, { exact: true }).click();
|
||||
await expect(form.getByLabel(MCP_TOOL_NAME)).toBeVisible();
|
||||
}
|
||||
|
||||
test.describe('agent builder MCP tools', () => {
|
||||
test('creates an agent with MCP server tools and persists the selection', async ({ page }) => {
|
||||
test.setTimeout(120000);
|
||||
|
||||
const agentName = uniqueAgentName('E2E MCP Agent');
|
||||
let createdAgentId: string | undefined;
|
||||
|
||||
try {
|
||||
const form = await openAgentBuilder(page);
|
||||
await waitForMCPTools(page);
|
||||
|
||||
await form.getByLabel('Agent name').fill(agentName);
|
||||
await form.getByLabel('Agent description').fill(DESCRIPTION);
|
||||
await form.getByLabel('Agent instructions').fill(INSTRUCTIONS);
|
||||
await selectMockModel(page, true);
|
||||
|
||||
await addMCPServerTools(page, form);
|
||||
|
||||
const [createResponse] = await Promise.all([
|
||||
page.waitForResponse(
|
||||
(response) =>
|
||||
response.request().method() === 'POST' &&
|
||||
new URL(response.url()).pathname === '/api/agents' &&
|
||||
response.status() === 201,
|
||||
{ timeout: 30000 },
|
||||
),
|
||||
form.getByRole('button', { name: 'Create' }).click(),
|
||||
]);
|
||||
const createdAgent = (await createResponse.json()) as AgentDetail;
|
||||
createdAgentId = createdAgent.id;
|
||||
|
||||
await expect(
|
||||
page.getByText(`Successfully created ${agentName}`, { exact: true }),
|
||||
).toBeVisible();
|
||||
|
||||
const persistedAgent = await waitForPersistedAgent(page, agentName, DESCRIPTION);
|
||||
expect(persistedAgent).toMatchObject({
|
||||
id: createdAgentId,
|
||||
name: agentName,
|
||||
description: DESCRIPTION,
|
||||
instructions: INSTRUCTIONS,
|
||||
provider: MOCK_ENDPOINTS[0].label,
|
||||
model: MOCK_ENDPOINTS[0].model,
|
||||
category: 'general',
|
||||
});
|
||||
expect(persistedAgent.tools).toEqual(
|
||||
expect.arrayContaining([MCP_SERVER_TOOL_ID, MCP_TOOL_ID]),
|
||||
);
|
||||
expect(persistedAgent.mcpServerNames).toEqual(expect.arrayContaining([MCP_SERVER_NAME]));
|
||||
|
||||
const reopenedForm = await openAgentBuilder(page);
|
||||
await reopenedForm.getByRole('combobox', { name: 'Agent', exact: true }).click();
|
||||
await page.getByRole('option', { name: agentName }).click();
|
||||
|
||||
await expect(reopenedForm.getByLabel('Agent name')).toHaveValue(agentName);
|
||||
await expect(reopenedForm.getByLabel('Agent description')).toHaveValue(DESCRIPTION);
|
||||
await expect(reopenedForm.getByLabel('Agent instructions')).toHaveValue(INSTRUCTIONS);
|
||||
await expectSelectedMCPServerTools(reopenedForm);
|
||||
|
||||
await reopenedForm.getByRole('button', { name: 'Select Agent' }).click();
|
||||
|
||||
const response = await sendMessage(page, `hello from ${agentName}`);
|
||||
expect(response.ok()).toBeTruthy();
|
||||
await expect(mockReply(page)).toBeVisible({ timeout: 30000 });
|
||||
} finally {
|
||||
await cleanupAgent(page, createdAgentId);
|
||||
}
|
||||
});
|
||||
});
|
||||
203
e2e/specs/mock/prompts.spec.ts
Normal file
203
e2e/specs/mock/prompts.spec.ts
Normal file
|
|
@ -0,0 +1,203 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import type { Page } from '@playwright/test';
|
||||
import {
|
||||
isAgentsStream,
|
||||
MOCK_ENDPOINTS,
|
||||
NEW_CHAT_PATH,
|
||||
escapeRegExp,
|
||||
fetchJson,
|
||||
getAccessToken,
|
||||
replyPrompt,
|
||||
replyText,
|
||||
requestJson,
|
||||
selectMockEndpoint,
|
||||
} from './helpers';
|
||||
|
||||
const DESCRIPTION = 'Use this prompt to verify LibreChat prompt creation in mock e2e tests.';
|
||||
const COMMAND = 'e2e-prompt';
|
||||
|
||||
type Prompt = {
|
||||
_id?: string;
|
||||
groupId: string;
|
||||
prompt: string;
|
||||
type: 'text' | 'chat';
|
||||
};
|
||||
|
||||
type PromptGroup = {
|
||||
_id?: string;
|
||||
name: string;
|
||||
oneliner?: string;
|
||||
command?: string;
|
||||
productionPrompt?: Pick<Prompt, 'prompt'> | null;
|
||||
};
|
||||
|
||||
type PromptGroupListResponse = {
|
||||
promptGroups?: PromptGroup[];
|
||||
};
|
||||
|
||||
const uniquePromptName = () => `E2E Prompt ${Date.now()}-${Math.floor(Math.random() * 1e4)}`;
|
||||
|
||||
async function findPromptGroup(
|
||||
page: Page,
|
||||
promptName: string,
|
||||
token: string,
|
||||
): Promise<PromptGroup | null> {
|
||||
const body = await fetchJson<PromptGroupListResponse>(
|
||||
page,
|
||||
`/api/prompts/groups?name=${encodeURIComponent(promptName)}&limit=10`,
|
||||
token,
|
||||
);
|
||||
return body.promptGroups?.find((group) => group.name === promptName) ?? null;
|
||||
}
|
||||
|
||||
async function waitForPersistedPrompt(
|
||||
page: Page,
|
||||
promptName: string,
|
||||
expectedPrompt: string,
|
||||
): Promise<{ group: PromptGroup; prompts: Prompt[] }> {
|
||||
const token = await getAccessToken(page);
|
||||
let latestGroup: PromptGroup | null = null;
|
||||
let latestPrompts: Prompt[] = [];
|
||||
|
||||
for (let attempt = 0; attempt < 20; attempt++) {
|
||||
const group = await findPromptGroup(page, promptName, token);
|
||||
if (group?._id) {
|
||||
latestGroup = await fetchJson<PromptGroup>(
|
||||
page,
|
||||
`/api/prompts/groups/${encodeURIComponent(group._id)}`,
|
||||
token,
|
||||
);
|
||||
latestPrompts = await fetchJson<Prompt[]>(
|
||||
page,
|
||||
`/api/prompts?groupId=${encodeURIComponent(group._id)}`,
|
||||
token,
|
||||
);
|
||||
|
||||
const hasExpectedPrompt = latestPrompts.some((prompt) => prompt.prompt === expectedPrompt);
|
||||
if (
|
||||
latestGroup.oneliner === DESCRIPTION &&
|
||||
latestGroup.command === COMMAND &&
|
||||
latestGroup.productionPrompt?.prompt === expectedPrompt &&
|
||||
hasExpectedPrompt
|
||||
) {
|
||||
return { group: latestGroup, prompts: latestPrompts };
|
||||
}
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
}
|
||||
|
||||
expect(latestGroup, `Expected prompt group "${promptName}" to be persisted`).not.toBeNull();
|
||||
expect(latestGroup?.oneliner).toBe(DESCRIPTION);
|
||||
expect(latestGroup?.command).toBe(COMMAND);
|
||||
expect(latestGroup?.productionPrompt?.prompt).toBe(expectedPrompt);
|
||||
expect(latestPrompts.some((prompt) => prompt.prompt === expectedPrompt)).toBe(true);
|
||||
return { group: latestGroup!, prompts: latestPrompts };
|
||||
}
|
||||
|
||||
async function cleanupPromptGroup(page: Page, groupId?: string) {
|
||||
if (!groupId) {
|
||||
return;
|
||||
}
|
||||
|
||||
const token = await getAccessToken(page);
|
||||
await requestJson<{ message?: string }>(page, {
|
||||
path: `/api/prompts/groups/${encodeURIComponent(groupId)}`,
|
||||
token,
|
||||
method: 'DELETE',
|
||||
});
|
||||
}
|
||||
|
||||
async function openPromptsPanel(page: Page) {
|
||||
const promptsButton = page.getByRole('button', { name: 'Prompts', exact: true });
|
||||
await expect(promptsButton).toBeVisible();
|
||||
if ((await promptsButton.getAttribute('aria-pressed')) !== 'true') {
|
||||
await promptsButton.click();
|
||||
}
|
||||
await expect(page.getByRole('search')).toBeVisible();
|
||||
}
|
||||
|
||||
async function ensureAutoSendPrompts(page: Page) {
|
||||
const autoSend = page.getByRole('button', { name: 'Send prompts on select' });
|
||||
await expect(autoSend).toBeVisible();
|
||||
if ((await autoSend.getAttribute('aria-pressed')) !== 'true') {
|
||||
await autoSend.click();
|
||||
}
|
||||
await expect(autoSend).toHaveAttribute('aria-pressed', 'true');
|
||||
}
|
||||
|
||||
test.describe('prompt manager', () => {
|
||||
test('creates a prompt and can send it from chat', async ({ page }) => {
|
||||
test.setTimeout(120000);
|
||||
|
||||
const promptName = uniquePromptName();
|
||||
const label = promptName.toLowerCase().replace(/[^a-z0-9]+/g, '-');
|
||||
const promptText = replyPrompt(label);
|
||||
let createdGroupId: string | undefined;
|
||||
|
||||
try {
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
await openPromptsPanel(page);
|
||||
|
||||
await page.getByRole('link', { name: 'Create Prompt' }).click();
|
||||
await expect(page).toHaveURL(/\/prompts\/new$/);
|
||||
|
||||
await page.getByRole('textbox', { name: 'Prompt Name' }).fill(promptName);
|
||||
await page.getByRole('textbox', { name: 'Prompt text input field' }).fill(promptText);
|
||||
await page
|
||||
.getByRole('textbox', { name: 'Optional: Enter a description to display for the prompt' })
|
||||
.fill(DESCRIPTION);
|
||||
await page
|
||||
.getByRole('textbox', {
|
||||
name: 'Optional: Enter a command for the prompt or name will be used',
|
||||
})
|
||||
.fill(COMMAND);
|
||||
|
||||
const [createResponse] = await Promise.all([
|
||||
page.waitForResponse(
|
||||
(response) =>
|
||||
response.request().method() === 'POST' &&
|
||||
new URL(response.url()).pathname === '/api/prompts' &&
|
||||
response.status() >= 200 &&
|
||||
response.status() < 300,
|
||||
{ timeout: 30000 },
|
||||
),
|
||||
page.getByRole('button', { name: 'Create Prompt' }).click(),
|
||||
]);
|
||||
const createdPrompt = (await createResponse.json()) as {
|
||||
group?: PromptGroup;
|
||||
prompt?: Prompt;
|
||||
};
|
||||
createdGroupId = createdPrompt.group?._id ?? createdPrompt.prompt?.groupId;
|
||||
|
||||
const { group } = await waitForPersistedPrompt(page, promptName, promptText);
|
||||
createdGroupId = group._id ?? createdGroupId;
|
||||
expect(createdGroupId).toBeTruthy();
|
||||
await expect(page).toHaveURL(new RegExp(`/prompts/${createdGroupId}$`));
|
||||
await expect(page.getByRole('button', { name: `Edit: ${promptName}` })).toBeVisible();
|
||||
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
|
||||
await openPromptsPanel(page);
|
||||
await ensureAutoSendPrompts(page);
|
||||
|
||||
await page.getByLabel('Filter prompts by name').fill(promptName);
|
||||
const promptCard = page.getByRole('button', {
|
||||
name: new RegExp(`^${escapeRegExp(promptName)} prompt`),
|
||||
});
|
||||
await expect(promptCard).toBeVisible({ timeout: 10000 });
|
||||
|
||||
const [response] = await Promise.all([
|
||||
page.waitForResponse(isAgentsStream, { timeout: 30000 }),
|
||||
promptCard.click(),
|
||||
]);
|
||||
expect(response.ok()).toBeTruthy();
|
||||
|
||||
await expect(page.getByTestId('messages-view').getByText(promptText)).toBeVisible();
|
||||
await expect(page.getByTestId('messages-view').getByText(replyText(label))).toBeVisible({
|
||||
timeout: 30000,
|
||||
});
|
||||
} finally {
|
||||
await cleanupPromptGroup(page, createdGroupId);
|
||||
}
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue