test: Add Activity-Label e2e Coverage with a Recording Label Server

Activity labels are the one model call a mock run does not already fake:
fake-model.js swaps the GRAPH model via overrideTestModel, while
run.generateActivityLabel() calls the endpoint resolved client options
over HTTP. The custom endpoints already point baseURL at 127.0.0.1:8889,
so serving that port exercises the real path with no production seam.

fake-label-server.js answers it in both JSON and SSE form, records each
prompt, and can inject blank/error responses. Recording is what lets the
spec assert the CONTRACT rather than the rendering: that this repo
register and the tool OUTPUTS actually reach the model. That is the bug
class that produced unusable labels before, and rendered text looks
identical whether or not the instruction arrived.

Labels get a dedicated endpoint (Mock Provider E). A labeled block
auto-collapses even at one tool call, which hides the tool cards other
specs assert on -- enabling this on a shared endpoint broke
steering.spec.ts. Provider D is the unlabeled control.

Request-count assertions are scoped to a per-test token: a 5xx label
response is retried by the provider client, and a retry can land after
the next test has reset the server.
This commit is contained in:
Danny Avila 2026-07-24 19:58:57 -04:00
parent 89a8e16bae
commit 52099d87f7
5 changed files with 484 additions and 3 deletions

View file

@ -107,6 +107,26 @@ endpoints:
titleConvo: false
modelDisplayLabel: 'Mock Provider D'
# Activity labels are enabled ONLY here. They get a dedicated endpoint
# because a label collapses its tool group (a labeled block auto-collapses
# even at one call), which hides the tool cards other specs assert on —
# enabling this on a shared endpoint broke steering.spec.ts. Provider D is
# the unlabeled control. The label call is the one request that leaves the
# process (e2e/setup/fake-label-server.js serves the baseURL below), and
# `activityModel` differs from the chat model so a spec can prove the label
# ran on the configured model rather than the agent's.
- name: 'Mock Provider E'
apiKey: 'e2e-mock-key-e'
baseURL: 'http://127.0.0.1:8889/v1'
models:
default:
- 'mock-model-e'
fetch: false
titleConvo: false
modelDisplayLabel: 'Mock Provider E'
activityLabel: true
activityModel: 'mock-label-model'
modelSpecs:
prioritize: true
# Enforcement would reject sends from the non-spec paths addedEndpoints
@ -117,6 +137,7 @@ modelSpecs:
addedEndpoints:
- 'Mock Provider C'
- 'Mock Provider D'
- 'Mock Provider E'
- 'agents'
list:
- name: 'e2e-mock-provider-a'

View file

@ -8,6 +8,9 @@ const serverPath = path.resolve(rootPath, 'e2e/setup/start-server.js');
const mcpHttpServerPath = path.resolve(rootPath, 'e2e/setup/fake-mcp-http-server.js');
/** Must match the `e2e-http` server URL in e2e/config/librechat.e2e.yaml. */
const MCP_HTTP_PORT = process.env.E2E_MCP_HTTP_PORT || '8765';
const labelServerPath = path.resolve(rootPath, 'e2e/setup/fake-label-server.js');
/** Must match the custom endpoints' `baseURL` in e2e/config/librechat.e2e.yaml. */
const LABEL_PORT = process.env.E2E_LABEL_PORT || '8889';
const fakeModelHookPath = path.resolve(rootPath, 'e2e/setup/fake-model.js');
const configTemplatePath = path.resolve(rootPath, 'e2e/config/librechat.e2e.yaml');
const configPath = path.resolve(rootPath, 'e2e/.generated/librechat.e2e.yaml');
@ -45,9 +48,13 @@ const preservedCredentialEnvKeys = new Set([
]);
/**
* The custom endpoints in the template point at an unreachable baseURL; the fake
* model injected via `LIBRECHAT_TEST_RUN_HOOK` overrides the run before any
* request is made, so no real (or mock HTTP) provider is contacted.
* The custom endpoints in the template point their `baseURL` at the local fake
* label server; the fake model injected via `LIBRECHAT_TEST_RUN_HOOK` overrides
* the GRAPH before any request is made, so no real provider is contacted.
*
* Activity labels are the one exception: `run.generateActivityLabel()` bypasses
* the graph override and calls the endpoint's resolved client options, so that
* request does go out over HTTP to `fake-label-server.js` on 127.0.0.1.
*/
function writeRuntimeMockConfig() {
const template = fs.readFileSync(configTemplatePath, 'utf8');
@ -146,5 +153,15 @@ export default defineConfig({
timeout: 60_000,
reuseExistingServer: false,
},
{
// Serves the activity-label model call (the custom endpoints' baseURL).
command: `node ${labelServerPath}`,
cwd: rootPath,
env: { ...process.env, E2E_LABEL_PORT: LABEL_PORT },
url: `http://127.0.0.1:${LABEL_PORT}/`,
stdout: 'pipe',
timeout: 60_000,
reuseExistingServer: false,
},
],
});

View file

@ -0,0 +1,180 @@
/**
* OpenAI-compatible HTTP fixture for activity-label e2e tests.
*
* Activity labels are the one model call in a mock run that is NOT served by
* `e2e/setup/fake-model.js`: that hook swaps the GRAPH's model via
* `run.Graph.overrideTestModel(...)`, while the label call goes out through
* `run.generateActivityLabel()` against client options resolved from the
* endpoint config. Those options carry the template's `baseURL`
* (http://127.0.0.1:8889/v1), so a real server on that port serves label
* calls and only label calls with no production seam. Every mock endpoint
* sets `titleConvo: false`, so nothing else lands here.
*
* Beyond returning a label it RECORDS each request, which is what lets a spec
* assert the prompt contract (that the register and the tool OUTPUTS actually
* reached the model) rather than just that some text rendered.
*/
const http = require('http');
const PORT = Number(process.env.E2E_LABEL_PORT) || 8889;
/** Recorded label requests, newest last. */
const requests = [];
/** Test-controlled response behavior; `reset` restores these defaults. */
const DEFAULT_BEHAVIOR = { mode: 'ok', label: null, delayMs: 0 };
let behavior = { ...DEFAULT_BEHAVIOR };
let labelCount = 0;
function readBody(req) {
return new Promise((resolve) => {
let raw = '';
req.on('data', (chunk) => {
raw += chunk;
});
req.on('end', () => {
try {
resolve(raw ? JSON.parse(raw) : {});
} catch {
resolve({});
}
});
});
}
function sendJson(res, status, payload) {
const body = JSON.stringify(payload);
res.writeHead(status, {
'Content-Type': 'application/json',
'Content-Length': Buffer.byteLength(body),
});
res.end(body);
}
function messageText(content) {
if (typeof content === 'string') {
return content;
}
if (!Array.isArray(content)) {
return '';
}
return content.map((part) => (typeof part === 'string' ? part : (part?.text ?? ''))).join('\n');
}
/** Flattened prompt text so specs can assert on the register and tool outputs. */
function flattenPrompt(messages) {
return (messages ?? []).map((message) => messageText(message?.content)).join('\n\n');
}
/** Non-streaming OpenAI chat completion. */
function completionPayload(model, label) {
return {
id: `chatcmpl-e2e-${labelCount}`,
object: 'chat.completion',
created: 0,
model: model ?? 'mock-label-model',
choices: [
{
index: 0,
message: { role: 'assistant', content: label },
finish_reason: 'stop',
},
],
usage: { prompt_tokens: 42, completion_tokens: 7, total_tokens: 49 },
};
}
/**
* SSE form of the same completion. The label call inherits the endpoint's
* client options, which may leave streaming on, so both shapes are served.
*/
function sendStream(res, model, label) {
res.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
Connection: 'keep-alive',
});
const base = {
id: `chatcmpl-e2e-${labelCount}`,
object: 'chat.completion.chunk',
created: 0,
model,
};
res.write(
`data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: { role: 'assistant', content: label }, finish_reason: null }] })}\n\n`,
);
res.write(
`data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], usage: { prompt_tokens: 42, completion_tokens: 7, total_tokens: 49 } })}\n\n`,
);
res.write('data: [DONE]\n\n');
res.end();
}
const server = http.createServer(async (req, res) => {
const url = new URL(req.url, `http://127.0.0.1:${PORT}`);
/** Playwright's webServer readiness probe. */
if (req.method === 'GET' && url.pathname === '/') {
sendJson(res, 200, { ok: true, service: 'fake-label-server' });
return;
}
if (req.method === 'GET' && url.pathname === '/__e2e/requests') {
sendJson(res, 200, { count: requests.length, requests });
return;
}
/** Specs reset between cases so counts and prompts stay per-test. */
if (req.method === 'POST' && url.pathname === '/__e2e/reset') {
requests.length = 0;
labelCount = 0;
behavior = { ...DEFAULT_BEHAVIOR };
sendJson(res, 200, { ok: true });
return;
}
if (req.method === 'POST' && url.pathname === '/__e2e/behavior') {
const body = await readBody(req);
behavior = { ...DEFAULT_BEHAVIOR, ...body };
sendJson(res, 200, { ok: true, behavior });
return;
}
if (req.method === 'POST' && url.pathname === '/v1/chat/completions') {
const body = await readBody(req);
labelCount += 1;
const prompt = flattenPrompt(body.messages);
requests.push({
model: body.model,
stream: body.stream === true,
prompt,
messages: body.messages ?? [],
});
if (behavior.delayMs > 0) {
await new Promise((resolve) => setTimeout(resolve, behavior.delayMs));
}
/** Generation failure: the run must finish cleanly with no header. */
if (behavior.mode === 'error') {
sendJson(res, 500, { error: { message: 'E2E forced label failure' } });
return;
}
/** Whitespace-only output must fill null, leaving the block unlabeled. */
const label =
behavior.mode === 'blank' ? ' ' : (behavior.label ?? `E2E activity label ${labelCount}`);
if (body.stream === true) {
sendStream(res, body.model, label);
return;
}
sendJson(res, 200, completionPayload(body.model, label));
return;
}
sendJson(res, 404, { error: { message: `Unhandled ${req.method} ${url.pathname}` } });
});
server.listen(PORT, '127.0.0.1', () => {
console.log(`[e2e] fake label server listening on http://127.0.0.1:${PORT}`);
});

View file

@ -29,6 +29,7 @@ const COUNTED_REPLY_MARKER = 'E2E_COUNTED_REPLY:';
const SLOW_REPLY_MARKER = 'E2E_SLOW_REPLY:';
const SLOW_COUNTED_REPLY_MARKER = 'E2E_SLOW_COUNTED_REPLY:';
const STEER_TOOL_REPLY_MARKER = 'E2E_STEER_TOOL_REPLY:';
const ACTIVITY_REPLY_MARKER = 'E2E_ACTIVITY_REPLY:';
const RESUME_ICON_REPLY_MARKER = 'E2E_RESUME_ICON_REPLY:';
const FORCED_ERROR_MARKER = 'E2E_FORCED_ERROR:';
const MARKDOWN_REPLY_MARKER = 'E2E_MARKDOWN_REPLY';
@ -43,6 +44,7 @@ const PROVIDER_FILE_ASSERTION_FINAL_TEXT = 'E2E provider file assertion passed';
const AGENT_CONTEXT_ASSERTION_FINAL_TEXT = 'E2E agent context assertion passed';
const QUOTE_ASSERTION_FINAL_TEXT = 'E2E quote assertion passed';
const STEER_TOOL_FINAL_TEXT = 'E2E steer tool reply done';
const ACTIVITY_FINAL_TEXT = 'E2E activity reply done';
const STEER_TOOL_NAME_PREFIX = 'remember_fact';
const SLOW_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_SLOW_CHUNK_DELAY_MS) || 35;
const SLOW_REPLY_CHUNKS = 160;
@ -818,6 +820,40 @@ function steerToolReplyResponses(label, toolNames) {
};
}
/**
* Two-turn run with a real tool boundary for the activity-label e2e: turn 1
* emits TWO parallel `remember_fact` calls (one `PostToolBatch` -> one label),
* turn 2 streams the final text. The args are distinct and the MCP fixture
* echoes them back prefixed, so a spec can tell an OUTPUT ("E2E MCP memory
* noted: ...") from an INPUT in the recorded label prompt which is the whole
* point of labeling after the batch rather than before it.
*/
function activityReplyResponses(label, toolNames) {
const toolName = Array.from(toolNames).find((name) => name.startsWith(STEER_TOOL_NAME_PREFIX));
if (!toolName) {
return {
responses: [`E2E activity reply unavailable: no ${STEER_TOOL_NAME_PREFIX} tool advertised.`],
};
}
return {
responses: ['', `${ACTIVITY_FINAL_TEXT} ${label}`],
toolCalls: [
{
id: `call_e2e_activity_alpha_${label}`,
name: toolName,
args: { fact: `activity alpha ${label}` },
type: 'tool_call',
},
{
id: `call_e2e_activity_beta_${label}`,
name: toolName,
args: { fact: `activity beta ${label}` },
type: 'tool_call',
},
],
};
}
function findLastToolMessageText(messages, requiredToken) {
for (let index = (messages ?? []).length - 1; index >= 0; index--) {
const message = messages[index];
@ -935,6 +971,11 @@ function resolveResponses({ graph, messages, text, toolNames }) {
return steerToolReplyResponses(steerToolLabel, toolNames);
}
const activityLabel = getMarkerValue(text, ACTIVITY_REPLY_MARKER);
if (activityLabel) {
return activityReplyResponses(activityLabel, toolNames);
}
if (text.includes(ASSERT_AGENT_CONTEXT_MARKER)) {
return {
responses: [MOCK_REPLY],

View file

@ -0,0 +1,222 @@
import { expect, test } from '@playwright/test';
import type { APIRequestContext, Page } from '@playwright/test';
import { NEW_CHAT_PATH, messagesView, selectMockEndpoint, sendMessage } from './helpers';
/** The only endpoint with `activityLabel` in e2e/config/librechat.e2e.yaml. It
* is dedicated to this spec: a label auto-collapses its tool group, hiding the
* tool cards other specs assert on. Both are non-spec `addedEndpoints`, the
* path the ephemeral MCP dropdown rides (mirroring steering.spec.ts) a
* spec-backed endpoint would not surface the selector at all. */
const LABELED_ENDPOINT = { label: 'Mock Provider E', model: 'mock-model-e' };
/** Same path, no `activityLabel`: the control proving the config gates it. */
const UNLABELED_ENDPOINT = { label: 'Mock Provider D', model: 'mock-model-d' };
/** Distinct from the chat model, so a label request proves `activityModel` won. */
const LABEL_MODEL = 'mock-label-model';
const MCP_SERVER_TITLE = 'E2E Memory';
const LABEL_SERVER = `http://127.0.0.1:${process.env.E2E_LABEL_PORT || '8889'}`;
type LabelRequest = { model?: string; stream: boolean; prompt: string };
const uniqueLabel = (prefix: string) =>
`${prefix}-${Date.now()}-${Math.floor(Math.random() * 1e4)}`;
async function resetLabelServer(request: APIRequestContext) {
const response = await request.post(`${LABEL_SERVER}/__e2e/reset`);
expect(response.ok()).toBeTruthy();
}
async function setLabelBehavior(
request: APIRequestContext,
behavior: { mode?: 'ok' | 'blank' | 'error'; label?: string; delayMs?: number },
) {
const response = await request.post(`${LABEL_SERVER}/__e2e/behavior`, { data: behavior });
expect(response.ok()).toBeTruthy();
}
async function getLabelRequests(request: APIRequestContext): Promise<LabelRequest[]> {
const response = await request.get(`${LABEL_SERVER}/__e2e/requests`);
expect(response.ok()).toBeTruthy();
return (await response.json()).requests as LabelRequest[];
}
/**
* Label requests carrying THIS test's token, which reaches the server inside
* the recorded tool arguments. Counting every request instead would be racy:
* a 5xx label response is retried by the provider client, and a retry can land
* after the next test has already reset the server.
*/
async function getLabelRequestsFor(
request: APIRequestContext,
token: string,
): Promise<LabelRequest[]> {
return (await getLabelRequests(request)).filter((entry) => entry.prompt.includes(token));
}
/** Select the MCP server whose `remember_fact` tool creates the batch boundary. */
async function selectEphemeralMCP(page: Page) {
await page.getByRole('button', { name: 'MCP Servers', exact: true }).click();
const serverItem = page.getByRole('menuitemcheckbox', { name: new RegExp(MCP_SERVER_TITLE) });
await expect(serverItem).toBeVisible();
await serverItem.click();
await expect(serverItem).toHaveAttribute('aria-checked', 'true');
await page.keyboard.press('Escape');
await expect(page.getByRole('button', { name: new RegExp(MCP_SERVER_TITLE) })).toBeVisible();
}
/** Run one labeled turn: two parallel tool calls => exactly one PostToolBatch. */
async function runLabeledTurn(page: Page, label: string) {
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
await selectMockEndpoint(page, LABELED_ENDPOINT);
await selectEphemeralMCP(page);
const run = await sendMessage(page, `E2E_ACTIVITY_REPLY:${label}`);
expect(run.ok()).toBeTruthy();
await expect(messagesView(page).getByText(`E2E activity reply done ${label}`)).toBeVisible({
timeout: 60000,
});
}
test.describe('activity labels', () => {
test.beforeEach(async ({ request }) => {
await resetLabelServer(request);
});
/**
* The header is the feature: once a label lands it REPLACES the generic
* "Used N tools" verb above the same tool cards.
*/
test('renders the generated label as the tool-group header', async ({ page, request }) => {
test.setTimeout(120000);
const label = uniqueLabel('activity');
await setLabelBehavior(request, { label: 'Stored two facts in memory' });
await runLabeledTurn(page, label);
await expect(
messagesView(page).getByRole('button', { name: 'Stored two facts in memory' }),
).toBeVisible({ timeout: 30000 });
await expect(messagesView(page).getByRole('button', { name: 'Used 2 tools' })).toHaveCount(0);
});
/**
* Regression for the bug that made real output "abysmal": the wiring passed a
* prompt ONLY when `activityPrompt` was configured, so a default install ran
* the SDK's own generic prompt and this repo's register never reached the
* model. Asserting on the request the model actually received is the only way
* to catch that rendered text looks identical either way.
*
* Also pins the two things that make the header worth a row: it runs on the
* configured `activityModel`, and it sees the tool OUTPUTS (only available
* because the hook fires AFTER the batch), not just the arguments.
*/
test('sends the register, the tool outputs, and the configured model', async ({
page,
request,
}) => {
test.setTimeout(120000);
const label = uniqueLabel('prompt');
await runLabeledTurn(page, label);
await expect
.poll(async () => (await getLabelRequestsFor(request, label)).length, { timeout: 30000 })
.toBe(1);
const [labelRequest] = await getLabelRequestsFor(request, label);
/** `activityModel` beat the agent's own model. */
expect(labelRequest.model).toBe(LABEL_MODEL);
/** This repo's register reached the model, not the SDK's built-in prompt. */
expect(labelRequest.prompt).toMatch(/never name the tools/i);
expect(labelRequest.prompt).toMatch(/outcome, not the attempt/i);
/** Deliberately NOT asserted: the "do not restate these" entry framing
* lives in `buildPrompt`, which only the direct fallback path uses. The
* SDK path builds the entry list with its own `buildActivityLabelPrompt`,
* so the two paths agree on the register (above) but not on that framing.
* Asserting it here would encode a divergence the SDK owns. */
/** Tool OUTPUTS, not just inputs — the reason this runs post-batch. */
expect(labelRequest.prompt).toContain(`E2E MCP memory noted: activity alpha ${label}`);
expect(labelRequest.prompt).toContain(`E2E MCP memory noted: activity beta ${label}`);
});
/**
* A whitespace-only label must fill null. There is deliberately no templated
* stand-in ("ran 2 tools" only restates the cards), so the block renders
* exactly as it would without the feature.
*/
test('leaves the generic header when the model returns a blank label', async ({
page,
request,
}) => {
test.setTimeout(120000);
const label = uniqueLabel('blank');
await setLabelBehavior(request, { mode: 'blank' });
await runLabeledTurn(page, label);
await expect(messagesView(page).getByRole('button', { name: 'Used 2 tools' })).toBeVisible({
timeout: 30000,
});
});
/** Label generation is best-effort: a failing label must not fail the run. */
test('completes the run cleanly when label generation errors', async ({ page, request }) => {
test.setTimeout(120000);
const label = uniqueLabel('failure');
await setLabelBehavior(request, { mode: 'error' });
await runLabeledTurn(page, label);
/** The turn still finished (asserted in runLabeledTurn) and the block kept
* its generic header rather than rendering an empty row. */
await expect(messagesView(page).getByRole('button', { name: 'Used 2 tools' })).toBeVisible({
timeout: 30000,
});
/** At least one attempt was made and failed; the client may retry a 5xx,
* so the exact count is not part of the contract. */
expect((await getLabelRequestsFor(request, label)).length).toBeGreaterThanOrEqual(1);
});
/** `activityLabel` is per-endpoint: an endpoint without it must not call out. */
test('makes no label request on an endpoint without activityLabel', async ({ page, request }) => {
test.setTimeout(120000);
const label = uniqueLabel('disabled');
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
await selectMockEndpoint(page, UNLABELED_ENDPOINT);
await selectEphemeralMCP(page);
const run = await sendMessage(page, `E2E_ACTIVITY_REPLY:${label}`);
expect(run.ok()).toBeTruthy();
await expect(messagesView(page).getByText(`E2E activity reply done ${label}`)).toBeVisible({
timeout: 60000,
});
await expect(messagesView(page).getByRole('button', { name: 'Used 2 tools' })).toBeVisible();
expect(await getLabelRequestsFor(request, label)).toHaveLength(0);
});
/**
* The label is a persisted content part at a claimed index, not a live-only
* decoration: it must survive a reload at the same position.
*/
test('persists the label across a page reload', async ({ page, request }) => {
test.setTimeout(120000);
const label = uniqueLabel('persist');
await setLabelBehavior(request, { label: 'Recorded both facts for later' });
await runLabeledTurn(page, label);
const header = messagesView(page).getByRole('button', {
name: 'Recorded both facts for later',
});
await expect(header).toBeVisible({ timeout: 30000 });
await expect(page).toHaveURL(/\/c\/[0-9a-fA-F-]{36}$/, { timeout: 15000 });
await page.reload();
await expect(
messagesView(page).getByRole('button', { name: 'Recorded both facts for later' }),
).toBeVisible({ timeout: 30000 });
/** Reload replays persisted content; it must not trigger a new generation. */
expect(await getLabelRequestsFor(request, label)).toHaveLength(1);
});
});