mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-08-04 06:52:47 +00:00
test(e2e): cover escalation of waiting messages through the real seal
Three mock-harness tests on E2E_SLOW_REPLY, a 160-chunk stream with no tool boundary, so an in-thread steer part can ONLY come from a genuine mid-stream seal — which makes each test a behavioral proof rather than a UI check: - Queued row escalation: the ZapOff button turns a waiting queued message into a preempt-armed steer (202 echoes preempt: true) that seals and injects, where the sibling steering.spec test proves the unescalated path waits for run end instead. - Bubble in-place arm: an ordinary steer (202 with no preempt echo) waits as a bubble, POST /chat/steer/arm answers armed: true, the bubble relabels in place (same single bubble, same text, escalation no longer offered on reopen), and the armed steer seals mid-stream. - Always-interrupt toggle: flipped from a waiting row's overflow menu, plain Enter now produces a preempt: true steer that seals in the SAME run, and the menu offers the way back. An afterEach clears the localStorage preference so a mid-test failure cannot leak preempt-by-default into the rest of the serial suite. All three verified locally through the full harness (real backend, mock LLM, seeded DB): 3 passed in 27s.
This commit is contained in:
parent
964897a3b3
commit
199bd77d12
1 changed files with 225 additions and 0 deletions
225
e2e/specs/mock/steering-escalation.spec.ts
Normal file
225
e2e/specs/mock/steering-escalation.spec.ts
Normal file
|
|
@ -0,0 +1,225 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import type { Page, Response } from '@playwright/test';
|
||||
import {
|
||||
MOCK_ENDPOINTS,
|
||||
NEW_CHAT_PATH,
|
||||
messagesView,
|
||||
replyPrompt,
|
||||
replyText,
|
||||
selectMockEndpoint,
|
||||
sendMessage,
|
||||
} from './helpers';
|
||||
|
||||
/** Last chunk streamed by the fake model's slow replies (160 chunks, 0-indexed). */
|
||||
const SLOW_REPLY_LAST_CHUNK = 'chunk-159';
|
||||
|
||||
const uniqueLabel = (prefix: string) =>
|
||||
`${prefix}-${Date.now()}-${Math.floor(Math.random() * 1e4)}`;
|
||||
|
||||
const messageInput = (page: Page) => page.getByRole('textbox', { name: 'Message input' });
|
||||
const duringRunSendButton = (page: Page) => page.getByTestId('during-run-send-button');
|
||||
const queuedRows = (page: Page) => page.getByTestId('queued-message-row');
|
||||
const messageTurns = (page: Page) => messagesView(page).locator('.message-render');
|
||||
const inFlightSteers = (page: Page) => page.getByTestId('in-flight-steer');
|
||||
const appliedSteerParts = (page: Page) => messagesView(page).getByTestId('steer-part');
|
||||
|
||||
function isSteerRequest(response: Response) {
|
||||
return (
|
||||
response.request().method() === 'POST' &&
|
||||
new URL(response.url()).pathname === '/api/agents/chat/steer'
|
||||
);
|
||||
}
|
||||
|
||||
function isArmRequest(response: Response) {
|
||||
return (
|
||||
response.request().method() === 'POST' &&
|
||||
new URL(response.url()).pathname === '/api/agents/chat/steer/arm'
|
||||
);
|
||||
}
|
||||
|
||||
/** Establish a real conversation with a fast first turn so during-run actions
|
||||
* target a persisted conversation id instead of racing new-convo creation. */
|
||||
async function establishConversation(page: Page, label: string) {
|
||||
const setup = await sendMessage(page, replyPrompt(label));
|
||||
expect(setup.ok()).toBeTruthy();
|
||||
await expect(messagesView(page).getByText(replyText(label))).toBeVisible({ timeout: 30000 });
|
||||
await expect(page).toHaveURL(/\/c\/[0-9a-fA-F-]{36}$/, { timeout: 15000 });
|
||||
}
|
||||
|
||||
/** Fill the composer mid-run: the during-run send button must take the
|
||||
* send/stop slot (it becomes the form submit target for Enter). */
|
||||
async function typeDuringRun(page: Page, text: string) {
|
||||
const input = messageInput(page);
|
||||
await input.click();
|
||||
await input.fill(text);
|
||||
await expect(duringRunSendButton(page)).toBeVisible({ timeout: 5000 });
|
||||
}
|
||||
|
||||
/**
|
||||
* Escalation of WAITING messages (PR: interrupt-steer escalation controls).
|
||||
* `E2E_SLOW_REPLY` streams pure text with no tool boundary, so nothing here
|
||||
* can inject the ordinary way — an in-thread steer part can only come from a
|
||||
* mid-stream seal, which makes it the behavioral proof that escalation armed
|
||||
* a real interrupt rather than relabelling a chip.
|
||||
*/
|
||||
test.describe('escalating waiting messages to an interrupt', () => {
|
||||
/** `steerInterruptsByDefault` is a localStorage preference; the toggle test
|
||||
* flips it, and a mid-test failure must not leak preempt-by-default into
|
||||
* the rest of the serial suite. */
|
||||
test.afterEach(async ({ page }) => {
|
||||
await page.evaluate(() => window.localStorage.removeItem('steerInterruptsByDefault'));
|
||||
});
|
||||
|
||||
test('queued row escalates as an interrupt: the message seals mid-stream instead of waiting for run end', async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(150000);
|
||||
const label = uniqueLabel('queue-escalate');
|
||||
const queueText = `Escalated queued message ${label}`;
|
||||
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
|
||||
await establishConversation(page, `queue-escalate-setup-${label}`);
|
||||
|
||||
const run = await sendMessage(page, `E2E_SLOW_REPLY:${label}`);
|
||||
expect(run.ok()).toBeTruthy();
|
||||
await expect(messagesView(page).getByText('chunk-010')).toBeVisible({ timeout: 15000 });
|
||||
|
||||
// Queue the message (Ctrl/Cmd+Enter routes to the non-default action).
|
||||
await typeDuringRun(page, queueText);
|
||||
await messageInput(page).press('ControlOrMeta+Enter');
|
||||
const row = queuedRows(page).filter({ hasText: queueText });
|
||||
await expect(row).toBeVisible({ timeout: 10000 });
|
||||
|
||||
// Escalate it: the row's ZapOff button submits the queued text as an
|
||||
// interrupt steer (a preempt-armed POST /chat/steer).
|
||||
const [steerResponse] = await Promise.all([
|
||||
page.waitForResponse(isSteerRequest, { timeout: 15000 }),
|
||||
row.getByTestId('queued-interrupt-now').click(),
|
||||
]);
|
||||
expect(steerResponse.status()).toBe(202);
|
||||
expect(((await steerResponse.json()) as { preempt?: boolean }).preempt).toBe(true);
|
||||
await expect(row).toHaveCount(0, { timeout: 10000 });
|
||||
|
||||
// Injected in-thread with no tool boundary available — only a mid-stream
|
||||
// seal can put a steer part here. Without escalation this message would
|
||||
// have waited for run end and auto-sent as its own follow-up turn.
|
||||
await expect(appliedSteerParts(page).filter({ hasText: queueText })).toHaveCount(1, {
|
||||
timeout: 90000,
|
||||
});
|
||||
await expect(inFlightSteers(page)).toHaveCount(0);
|
||||
|
||||
// Sealed, not run to completion, and the pre-seal text survives.
|
||||
await expect(messagesView(page).getByText(SLOW_REPLY_LAST_CHUNK)).toHaveCount(0);
|
||||
await expect(messagesView(page).getByText('chunk-010')).toBeVisible();
|
||||
|
||||
// Stayed INSIDE the response: no auto-sent follow-up pair.
|
||||
await expect(messageTurns(page)).toHaveCount(4);
|
||||
await expect(queuedRows(page)).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('waiting steer bubble arms in place via POST /chat/steer/arm and seals mid-stream', async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(150000);
|
||||
const label = uniqueLabel('bubble-arm');
|
||||
const steerText = `Armed waiting steer ${label}`;
|
||||
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
|
||||
await establishConversation(page, `bubble-arm-setup-${label}`);
|
||||
|
||||
const run = await sendMessage(page, `E2E_SLOW_REPLY:${label}`);
|
||||
expect(run.ok()).toBeTruthy();
|
||||
await expect(messagesView(page).getByText('chunk-010')).toBeVisible({ timeout: 15000 });
|
||||
|
||||
// An ORDINARY steer (plain Enter, preference off): with no tool boundary
|
||||
// in this stream it stays acknowledged-and-waiting as a bubble.
|
||||
await typeDuringRun(page, steerText);
|
||||
const [steerResponse] = await Promise.all([
|
||||
page.waitForResponse(isSteerRequest, { timeout: 15000 }),
|
||||
messageInput(page).press('Enter'),
|
||||
]);
|
||||
expect(steerResponse.status()).toBe(202);
|
||||
expect(((await steerResponse.json()) as { preempt?: boolean }).preempt).toBeFalsy();
|
||||
const bubble = inFlightSteers(page).filter({ hasText: steerText });
|
||||
await expect(bubble).toBeVisible({ timeout: 10000 });
|
||||
|
||||
// Escalate from the bubble's overflow menu: ONE atomic in-place arm.
|
||||
await bubble.getByRole('button', { name: 'More options' }).click();
|
||||
const [armResponse] = await Promise.all([
|
||||
page.waitForResponse(isArmRequest, { timeout: 15000 }),
|
||||
page.getByRole('menuitem', { name: 'Interrupt & steer now' }).click(),
|
||||
]);
|
||||
expect(armResponse.status()).toBe(200);
|
||||
expect(((await armResponse.json()) as { armed?: boolean }).armed).toBe(true);
|
||||
|
||||
// Relabelled IN PLACE: still exactly one bubble with the same text, and
|
||||
// an interrupting steer no longer offers escalation on reopen.
|
||||
await expect(inFlightSteers(page)).toHaveCount(1);
|
||||
await bubble.getByRole('button', { name: 'More options' }).click();
|
||||
await expect(page.getByRole('menuitem', { name: 'Cancel steering message' })).toBeVisible({
|
||||
timeout: 5000,
|
||||
});
|
||||
await expect(page.getByRole('menuitem', { name: 'Interrupt & steer now' })).toHaveCount(0);
|
||||
await page.keyboard.press('Escape');
|
||||
|
||||
// The armed steer seals mid-stream and injects with no tool boundary.
|
||||
await expect(appliedSteerParts(page).filter({ hasText: steerText })).toHaveCount(1, {
|
||||
timeout: 90000,
|
||||
});
|
||||
await expect(inFlightSteers(page)).toHaveCount(0);
|
||||
await expect(messagesView(page).getByText(SLOW_REPLY_LAST_CHUNK)).toHaveCount(0);
|
||||
await expect(messagesView(page).getByText('chunk-010')).toBeVisible();
|
||||
await expect(messageTurns(page)).toHaveCount(4);
|
||||
});
|
||||
|
||||
test('always-interrupt toggle in a waiting row menu makes plain Enter preempt', async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(150000);
|
||||
const label = uniqueLabel('toggle');
|
||||
const queueText = `Queued while toggling ${label}`;
|
||||
const steerText = `Enter now interrupts ${label}`;
|
||||
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
|
||||
await establishConversation(page, `toggle-setup-${label}`);
|
||||
|
||||
const run = await sendMessage(page, `E2E_SLOW_REPLY:${label}`);
|
||||
expect(run.ok()).toBeTruthy();
|
||||
await expect(messagesView(page).getByText('chunk-010')).toBeVisible({ timeout: 15000 });
|
||||
|
||||
// A queued row hosts the overflow menu carrying the preference toggle.
|
||||
await typeDuringRun(page, queueText);
|
||||
await messageInput(page).press('ControlOrMeta+Enter');
|
||||
const row = queuedRows(page).filter({ hasText: queueText });
|
||||
await expect(row).toBeVisible({ timeout: 10000 });
|
||||
|
||||
await row.getByRole('button', { name: 'More options' }).click();
|
||||
await page.getByRole('menuitem', { name: 'Always interrupt instead' }).click();
|
||||
|
||||
// The toggle is live for the SAME run: plain Enter now routes the default
|
||||
// steer through the preempt path (the 202 carries the armed flag).
|
||||
await typeDuringRun(page, steerText);
|
||||
const [steerResponse] = await Promise.all([
|
||||
page.waitForResponse(isSteerRequest, { timeout: 15000 }),
|
||||
messageInput(page).press('Enter'),
|
||||
]);
|
||||
expect(steerResponse.status()).toBe(202);
|
||||
expect(((await steerResponse.json()) as { preempt?: boolean }).preempt).toBe(true);
|
||||
|
||||
// And the seal proves it end to end: injected with no boundary available.
|
||||
await expect(appliedSteerParts(page).filter({ hasText: steerText })).toHaveCount(1, {
|
||||
timeout: 90000,
|
||||
});
|
||||
await expect(messagesView(page).getByText(SLOW_REPLY_LAST_CHUNK)).toHaveCount(0);
|
||||
|
||||
// The menu now offers the way back (label flipped), on the queued row
|
||||
// that is still waiting for run end.
|
||||
await row.getByRole('button', { name: 'More options' }).click();
|
||||
await expect(page.getByRole('menuitem', { name: 'Wait for tool steps instead' })).toBeVisible({
|
||||
timeout: 5000,
|
||||
});
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue