diff --git a/e2e/config/librechat.e2e.yaml b/e2e/config/librechat.e2e.yaml index 3a8e287f94..c5833814d9 100644 --- a/e2e/config/librechat.e2e.yaml +++ b/e2e/config/librechat.e2e.yaml @@ -107,6 +107,26 @@ endpoints: titleConvo: false modelDisplayLabel: 'Mock Provider D' + # Activity labels are enabled ONLY here. They get a dedicated endpoint + # because a label collapses its tool group (a labeled block auto-collapses + # even at one call), which hides the tool cards other specs assert on — + # enabling this on a shared endpoint broke steering.spec.ts. Provider D is + # the unlabeled control. The label call is the one request that leaves the + # process (e2e/setup/fake-label-server.js serves the baseURL below), and + # `activityModel` differs from the chat model so a spec can prove the label + # ran on the configured model rather than the agent's. + - name: 'Mock Provider E' + apiKey: 'e2e-mock-key-e' + baseURL: 'http://127.0.0.1:8889/v1' + models: + default: + - 'mock-model-e' + fetch: false + titleConvo: false + modelDisplayLabel: 'Mock Provider E' + activityLabel: true + activityModel: 'mock-label-model' + modelSpecs: prioritize: true # Enforcement would reject sends from the non-spec paths addedEndpoints @@ -117,6 +137,7 @@ modelSpecs: addedEndpoints: - 'Mock Provider C' - 'Mock Provider D' + - 'Mock Provider E' - 'agents' list: - name: 'e2e-mock-provider-a' diff --git a/e2e/playwright.config.mock.ts b/e2e/playwright.config.mock.ts index 74aac8dbd0..721df606ce 100644 --- a/e2e/playwright.config.mock.ts +++ b/e2e/playwright.config.mock.ts @@ -8,6 +8,9 @@ const serverPath = path.resolve(rootPath, 'e2e/setup/start-server.js'); const mcpHttpServerPath = path.resolve(rootPath, 'e2e/setup/fake-mcp-http-server.js'); /** Must match the `e2e-http` server URL in e2e/config/librechat.e2e.yaml. */ const MCP_HTTP_PORT = process.env.E2E_MCP_HTTP_PORT || '8765'; +const labelServerPath = path.resolve(rootPath, 'e2e/setup/fake-label-server.js'); +/** Must match the custom endpoints' `baseURL` in e2e/config/librechat.e2e.yaml. */ +const LABEL_PORT = process.env.E2E_LABEL_PORT || '8889'; const fakeModelHookPath = path.resolve(rootPath, 'e2e/setup/fake-model.js'); const configTemplatePath = path.resolve(rootPath, 'e2e/config/librechat.e2e.yaml'); const configPath = path.resolve(rootPath, 'e2e/.generated/librechat.e2e.yaml'); @@ -45,9 +48,13 @@ const preservedCredentialEnvKeys = new Set([ ]); /** - * The custom endpoints in the template point at an unreachable baseURL; the fake - * model injected via `LIBRECHAT_TEST_RUN_HOOK` overrides the run before any - * request is made, so no real (or mock HTTP) provider is contacted. + * The custom endpoints in the template point their `baseURL` at the local fake + * label server; the fake model injected via `LIBRECHAT_TEST_RUN_HOOK` overrides + * the GRAPH before any request is made, so no real provider is contacted. + * + * Activity labels are the one exception: `run.generateActivityLabel()` bypasses + * the graph override and calls the endpoint's resolved client options, so that + * request does go out over HTTP — to `fake-label-server.js` on 127.0.0.1. */ function writeRuntimeMockConfig() { const template = fs.readFileSync(configTemplatePath, 'utf8'); @@ -146,5 +153,15 @@ export default defineConfig({ timeout: 60_000, reuseExistingServer: false, }, + { + // Serves the activity-label model call (the custom endpoints' baseURL). + command: `node ${labelServerPath}`, + cwd: rootPath, + env: { ...process.env, E2E_LABEL_PORT: LABEL_PORT }, + url: `http://127.0.0.1:${LABEL_PORT}/`, + stdout: 'pipe', + timeout: 60_000, + reuseExistingServer: false, + }, ], }); diff --git a/e2e/setup/fake-label-server.js b/e2e/setup/fake-label-server.js new file mode 100644 index 0000000000..ad2e590b27 --- /dev/null +++ b/e2e/setup/fake-label-server.js @@ -0,0 +1,180 @@ +/** + * OpenAI-compatible HTTP fixture for activity-label e2e tests. + * + * Activity labels are the one model call in a mock run that is NOT served by + * `e2e/setup/fake-model.js`: that hook swaps the GRAPH's model via + * `run.Graph.overrideTestModel(...)`, while the label call goes out through + * `run.generateActivityLabel()` against client options resolved from the + * endpoint config. Those options carry the template's `baseURL` + * (http://127.0.0.1:8889/v1), so a real server on that port serves label + * calls — and only label calls — with no production seam. Every mock endpoint + * sets `titleConvo: false`, so nothing else lands here. + * + * Beyond returning a label it RECORDS each request, which is what lets a spec + * assert the prompt contract (that the register and the tool OUTPUTS actually + * reached the model) rather than just that some text rendered. + */ +const http = require('http'); + +const PORT = Number(process.env.E2E_LABEL_PORT) || 8889; + +/** Recorded label requests, newest last. */ +const requests = []; +/** Test-controlled response behavior; `reset` restores these defaults. */ +const DEFAULT_BEHAVIOR = { mode: 'ok', label: null, delayMs: 0 }; +let behavior = { ...DEFAULT_BEHAVIOR }; +let labelCount = 0; + +function readBody(req) { + return new Promise((resolve) => { + let raw = ''; + req.on('data', (chunk) => { + raw += chunk; + }); + req.on('end', () => { + try { + resolve(raw ? JSON.parse(raw) : {}); + } catch { + resolve({}); + } + }); + }); +} + +function sendJson(res, status, payload) { + const body = JSON.stringify(payload); + res.writeHead(status, { + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(body), + }); + res.end(body); +} + +function messageText(content) { + if (typeof content === 'string') { + return content; + } + if (!Array.isArray(content)) { + return ''; + } + return content.map((part) => (typeof part === 'string' ? part : (part?.text ?? ''))).join('\n'); +} + +/** Flattened prompt text so specs can assert on the register and tool outputs. */ +function flattenPrompt(messages) { + return (messages ?? []).map((message) => messageText(message?.content)).join('\n\n'); +} + +/** Non-streaming OpenAI chat completion. */ +function completionPayload(model, label) { + return { + id: `chatcmpl-e2e-${labelCount}`, + object: 'chat.completion', + created: 0, + model: model ?? 'mock-label-model', + choices: [ + { + index: 0, + message: { role: 'assistant', content: label }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 42, completion_tokens: 7, total_tokens: 49 }, + }; +} + +/** + * SSE form of the same completion. The label call inherits the endpoint's + * client options, which may leave streaming on, so both shapes are served. + */ +function sendStream(res, model, label) { + res.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + }); + const base = { + id: `chatcmpl-e2e-${labelCount}`, + object: 'chat.completion.chunk', + created: 0, + model, + }; + res.write( + `data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: { role: 'assistant', content: label }, finish_reason: null }] })}\n\n`, + ); + res.write( + `data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], usage: { prompt_tokens: 42, completion_tokens: 7, total_tokens: 49 } })}\n\n`, + ); + res.write('data: [DONE]\n\n'); + res.end(); +} + +const server = http.createServer(async (req, res) => { + const url = new URL(req.url, `http://127.0.0.1:${PORT}`); + + /** Playwright's webServer readiness probe. */ + if (req.method === 'GET' && url.pathname === '/') { + sendJson(res, 200, { ok: true, service: 'fake-label-server' }); + return; + } + + if (req.method === 'GET' && url.pathname === '/__e2e/requests') { + sendJson(res, 200, { count: requests.length, requests }); + return; + } + + /** Specs reset between cases so counts and prompts stay per-test. */ + if (req.method === 'POST' && url.pathname === '/__e2e/reset') { + requests.length = 0; + labelCount = 0; + behavior = { ...DEFAULT_BEHAVIOR }; + sendJson(res, 200, { ok: true }); + return; + } + + if (req.method === 'POST' && url.pathname === '/__e2e/behavior') { + const body = await readBody(req); + behavior = { ...DEFAULT_BEHAVIOR, ...body }; + sendJson(res, 200, { ok: true, behavior }); + return; + } + + if (req.method === 'POST' && url.pathname === '/v1/chat/completions') { + const body = await readBody(req); + labelCount += 1; + const prompt = flattenPrompt(body.messages); + requests.push({ + model: body.model, + stream: body.stream === true, + prompt, + messages: body.messages ?? [], + }); + + if (behavior.delayMs > 0) { + await new Promise((resolve) => setTimeout(resolve, behavior.delayMs)); + } + + /** Generation failure: the run must finish cleanly with no header. */ + if (behavior.mode === 'error') { + sendJson(res, 500, { error: { message: 'E2E forced label failure' } }); + return; + } + + /** Whitespace-only output must fill null, leaving the block unlabeled. */ + const label = + behavior.mode === 'blank' ? ' ' : (behavior.label ?? `E2E activity label ${labelCount}`); + + if (body.stream === true) { + sendStream(res, body.model, label); + return; + } + sendJson(res, 200, completionPayload(body.model, label)); + return; + } + + sendJson(res, 404, { error: { message: `Unhandled ${req.method} ${url.pathname}` } }); +}); + +server.listen(PORT, '127.0.0.1', () => { + console.log(`[e2e] fake label server listening on http://127.0.0.1:${PORT}`); +}); diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.js index d432e687d0..a04be9e714 100644 --- a/e2e/setup/fake-model.js +++ b/e2e/setup/fake-model.js @@ -29,6 +29,7 @@ const COUNTED_REPLY_MARKER = 'E2E_COUNTED_REPLY:'; const SLOW_REPLY_MARKER = 'E2E_SLOW_REPLY:'; const SLOW_COUNTED_REPLY_MARKER = 'E2E_SLOW_COUNTED_REPLY:'; const STEER_TOOL_REPLY_MARKER = 'E2E_STEER_TOOL_REPLY:'; +const ACTIVITY_REPLY_MARKER = 'E2E_ACTIVITY_REPLY:'; const RESUME_ICON_REPLY_MARKER = 'E2E_RESUME_ICON_REPLY:'; const FORCED_ERROR_MARKER = 'E2E_FORCED_ERROR:'; const MARKDOWN_REPLY_MARKER = 'E2E_MARKDOWN_REPLY'; @@ -43,6 +44,7 @@ const PROVIDER_FILE_ASSERTION_FINAL_TEXT = 'E2E provider file assertion passed'; const AGENT_CONTEXT_ASSERTION_FINAL_TEXT = 'E2E agent context assertion passed'; const QUOTE_ASSERTION_FINAL_TEXT = 'E2E quote assertion passed'; const STEER_TOOL_FINAL_TEXT = 'E2E steer tool reply done'; +const ACTIVITY_FINAL_TEXT = 'E2E activity reply done'; const STEER_TOOL_NAME_PREFIX = 'remember_fact'; const SLOW_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_SLOW_CHUNK_DELAY_MS) || 35; const SLOW_REPLY_CHUNKS = 160; @@ -818,6 +820,40 @@ function steerToolReplyResponses(label, toolNames) { }; } +/** + * Two-turn run with a real tool boundary for the activity-label e2e: turn 1 + * emits TWO parallel `remember_fact` calls (one `PostToolBatch` -> one label), + * turn 2 streams the final text. The args are distinct and the MCP fixture + * echoes them back prefixed, so a spec can tell an OUTPUT ("E2E MCP memory + * noted: ...") from an INPUT in the recorded label prompt — which is the whole + * point of labeling after the batch rather than before it. + */ +function activityReplyResponses(label, toolNames) { + const toolName = Array.from(toolNames).find((name) => name.startsWith(STEER_TOOL_NAME_PREFIX)); + if (!toolName) { + return { + responses: [`E2E activity reply unavailable: no ${STEER_TOOL_NAME_PREFIX} tool advertised.`], + }; + } + return { + responses: ['', `${ACTIVITY_FINAL_TEXT} ${label}`], + toolCalls: [ + { + id: `call_e2e_activity_alpha_${label}`, + name: toolName, + args: { fact: `activity alpha ${label}` }, + type: 'tool_call', + }, + { + id: `call_e2e_activity_beta_${label}`, + name: toolName, + args: { fact: `activity beta ${label}` }, + type: 'tool_call', + }, + ], + }; +} + function findLastToolMessageText(messages, requiredToken) { for (let index = (messages ?? []).length - 1; index >= 0; index--) { const message = messages[index]; @@ -935,6 +971,11 @@ function resolveResponses({ graph, messages, text, toolNames }) { return steerToolReplyResponses(steerToolLabel, toolNames); } + const activityLabel = getMarkerValue(text, ACTIVITY_REPLY_MARKER); + if (activityLabel) { + return activityReplyResponses(activityLabel, toolNames); + } + if (text.includes(ASSERT_AGENT_CONTEXT_MARKER)) { return { responses: [MOCK_REPLY], diff --git a/e2e/specs/mock/activity-labels.spec.ts b/e2e/specs/mock/activity-labels.spec.ts new file mode 100644 index 0000000000..d139762cb9 --- /dev/null +++ b/e2e/specs/mock/activity-labels.spec.ts @@ -0,0 +1,222 @@ +import { expect, test } from '@playwright/test'; +import type { APIRequestContext, Page } from '@playwright/test'; +import { NEW_CHAT_PATH, messagesView, selectMockEndpoint, sendMessage } from './helpers'; + +/** The only endpoint with `activityLabel` in e2e/config/librechat.e2e.yaml. It + * is dedicated to this spec: a label auto-collapses its tool group, hiding the + * tool cards other specs assert on. Both are non-spec `addedEndpoints`, the + * path the ephemeral MCP dropdown rides (mirroring steering.spec.ts) — a + * spec-backed endpoint would not surface the selector at all. */ +const LABELED_ENDPOINT = { label: 'Mock Provider E', model: 'mock-model-e' }; +/** Same path, no `activityLabel`: the control proving the config gates it. */ +const UNLABELED_ENDPOINT = { label: 'Mock Provider D', model: 'mock-model-d' }; +/** Distinct from the chat model, so a label request proves `activityModel` won. */ +const LABEL_MODEL = 'mock-label-model'; +const MCP_SERVER_TITLE = 'E2E Memory'; +const LABEL_SERVER = `http://127.0.0.1:${process.env.E2E_LABEL_PORT || '8889'}`; + +type LabelRequest = { model?: string; stream: boolean; prompt: string }; + +const uniqueLabel = (prefix: string) => + `${prefix}-${Date.now()}-${Math.floor(Math.random() * 1e4)}`; + +async function resetLabelServer(request: APIRequestContext) { + const response = await request.post(`${LABEL_SERVER}/__e2e/reset`); + expect(response.ok()).toBeTruthy(); +} + +async function setLabelBehavior( + request: APIRequestContext, + behavior: { mode?: 'ok' | 'blank' | 'error'; label?: string; delayMs?: number }, +) { + const response = await request.post(`${LABEL_SERVER}/__e2e/behavior`, { data: behavior }); + expect(response.ok()).toBeTruthy(); +} + +async function getLabelRequests(request: APIRequestContext): Promise { + const response = await request.get(`${LABEL_SERVER}/__e2e/requests`); + expect(response.ok()).toBeTruthy(); + return (await response.json()).requests as LabelRequest[]; +} + +/** + * Label requests carrying THIS test's token, which reaches the server inside + * the recorded tool arguments. Counting every request instead would be racy: + * a 5xx label response is retried by the provider client, and a retry can land + * after the next test has already reset the server. + */ +async function getLabelRequestsFor( + request: APIRequestContext, + token: string, +): Promise { + return (await getLabelRequests(request)).filter((entry) => entry.prompt.includes(token)); +} + +/** Select the MCP server whose `remember_fact` tool creates the batch boundary. */ +async function selectEphemeralMCP(page: Page) { + await page.getByRole('button', { name: 'MCP Servers', exact: true }).click(); + const serverItem = page.getByRole('menuitemcheckbox', { name: new RegExp(MCP_SERVER_TITLE) }); + await expect(serverItem).toBeVisible(); + await serverItem.click(); + await expect(serverItem).toHaveAttribute('aria-checked', 'true'); + await page.keyboard.press('Escape'); + await expect(page.getByRole('button', { name: new RegExp(MCP_SERVER_TITLE) })).toBeVisible(); +} + +/** Run one labeled turn: two parallel tool calls => exactly one PostToolBatch. */ +async function runLabeledTurn(page: Page, label: string) { + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await selectMockEndpoint(page, LABELED_ENDPOINT); + await selectEphemeralMCP(page); + const run = await sendMessage(page, `E2E_ACTIVITY_REPLY:${label}`); + expect(run.ok()).toBeTruthy(); + await expect(messagesView(page).getByText(`E2E activity reply done ${label}`)).toBeVisible({ + timeout: 60000, + }); +} + +test.describe('activity labels', () => { + test.beforeEach(async ({ request }) => { + await resetLabelServer(request); + }); + + /** + * The header is the feature: once a label lands it REPLACES the generic + * "Used N tools" verb above the same tool cards. + */ + test('renders the generated label as the tool-group header', async ({ page, request }) => { + test.setTimeout(120000); + const label = uniqueLabel('activity'); + await setLabelBehavior(request, { label: 'Stored two facts in memory' }); + + await runLabeledTurn(page, label); + + await expect( + messagesView(page).getByRole('button', { name: 'Stored two facts in memory' }), + ).toBeVisible({ timeout: 30000 }); + await expect(messagesView(page).getByRole('button', { name: 'Used 2 tools' })).toHaveCount(0); + }); + + /** + * Regression for the bug that made real output "abysmal": the wiring passed a + * prompt ONLY when `activityPrompt` was configured, so a default install ran + * the SDK's own generic prompt and this repo's register never reached the + * model. Asserting on the request the model actually received is the only way + * to catch that — rendered text looks identical either way. + * + * Also pins the two things that make the header worth a row: it runs on the + * configured `activityModel`, and it sees the tool OUTPUTS (only available + * because the hook fires AFTER the batch), not just the arguments. + */ + test('sends the register, the tool outputs, and the configured model', async ({ + page, + request, + }) => { + test.setTimeout(120000); + const label = uniqueLabel('prompt'); + + await runLabeledTurn(page, label); + + await expect + .poll(async () => (await getLabelRequestsFor(request, label)).length, { timeout: 30000 }) + .toBe(1); + const [labelRequest] = await getLabelRequestsFor(request, label); + + /** `activityModel` beat the agent's own model. */ + expect(labelRequest.model).toBe(LABEL_MODEL); + + /** This repo's register reached the model, not the SDK's built-in prompt. */ + expect(labelRequest.prompt).toMatch(/never name the tools/i); + expect(labelRequest.prompt).toMatch(/outcome, not the attempt/i); + /** Deliberately NOT asserted: the "do not restate these" entry framing + * lives in `buildPrompt`, which only the direct fallback path uses. The + * SDK path builds the entry list with its own `buildActivityLabelPrompt`, + * so the two paths agree on the register (above) but not on that framing. + * Asserting it here would encode a divergence the SDK owns. */ + + /** Tool OUTPUTS, not just inputs — the reason this runs post-batch. */ + expect(labelRequest.prompt).toContain(`E2E MCP memory noted: activity alpha ${label}`); + expect(labelRequest.prompt).toContain(`E2E MCP memory noted: activity beta ${label}`); + }); + + /** + * A whitespace-only label must fill null. There is deliberately no templated + * stand-in ("ran 2 tools" only restates the cards), so the block renders + * exactly as it would without the feature. + */ + test('leaves the generic header when the model returns a blank label', async ({ + page, + request, + }) => { + test.setTimeout(120000); + const label = uniqueLabel('blank'); + await setLabelBehavior(request, { mode: 'blank' }); + + await runLabeledTurn(page, label); + + await expect(messagesView(page).getByRole('button', { name: 'Used 2 tools' })).toBeVisible({ + timeout: 30000, + }); + }); + + /** Label generation is best-effort: a failing label must not fail the run. */ + test('completes the run cleanly when label generation errors', async ({ page, request }) => { + test.setTimeout(120000); + const label = uniqueLabel('failure'); + await setLabelBehavior(request, { mode: 'error' }); + + await runLabeledTurn(page, label); + + /** The turn still finished (asserted in runLabeledTurn) and the block kept + * its generic header rather than rendering an empty row. */ + await expect(messagesView(page).getByRole('button', { name: 'Used 2 tools' })).toBeVisible({ + timeout: 30000, + }); + /** At least one attempt was made and failed; the client may retry a 5xx, + * so the exact count is not part of the contract. */ + expect((await getLabelRequestsFor(request, label)).length).toBeGreaterThanOrEqual(1); + }); + + /** `activityLabel` is per-endpoint: an endpoint without it must not call out. */ + test('makes no label request on an endpoint without activityLabel', async ({ page, request }) => { + test.setTimeout(120000); + const label = uniqueLabel('disabled'); + + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await selectMockEndpoint(page, UNLABELED_ENDPOINT); + await selectEphemeralMCP(page); + const run = await sendMessage(page, `E2E_ACTIVITY_REPLY:${label}`); + expect(run.ok()).toBeTruthy(); + await expect(messagesView(page).getByText(`E2E activity reply done ${label}`)).toBeVisible({ + timeout: 60000, + }); + + await expect(messagesView(page).getByRole('button', { name: 'Used 2 tools' })).toBeVisible(); + expect(await getLabelRequestsFor(request, label)).toHaveLength(0); + }); + + /** + * The label is a persisted content part at a claimed index, not a live-only + * decoration: it must survive a reload at the same position. + */ + test('persists the label across a page reload', async ({ page, request }) => { + test.setTimeout(120000); + const label = uniqueLabel('persist'); + await setLabelBehavior(request, { label: 'Recorded both facts for later' }); + + await runLabeledTurn(page, label); + const header = messagesView(page).getByRole('button', { + name: 'Recorded both facts for later', + }); + await expect(header).toBeVisible({ timeout: 30000 }); + + await expect(page).toHaveURL(/\/c\/[0-9a-fA-F-]{36}$/, { timeout: 15000 }); + await page.reload(); + + await expect( + messagesView(page).getByRole('button', { name: 'Recorded both facts for later' }), + ).toBeVisible({ timeout: 30000 }); + /** Reload replays persisted content; it must not trigger a new generation. */ + expect(await getLabelRequestsFor(request, label)).toHaveLength(1); + }); +});