🎛️ test: Guard Multi-Steer Injection Across Tool Boundaries (#14498)

* 🎛️ test: Guard Multi-Steer Injection Across Tool Boundaries

* 🎛️ test: Enforce ACK Overlap and Ordered Steer Echo Assertions
This commit is contained in:
Danny Avila 2026-07-28 22:19:11 -04:00 committed by GitHub
parent 4f5808d9ae
commit aa357a8e17
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 426 additions and 10 deletions

View file

@ -29,6 +29,8 @@ const COUNTED_REPLY_MARKER = 'E2E_COUNTED_REPLY:';
const SLOW_REPLY_MARKER = 'E2E_SLOW_REPLY:';
const SLOW_COUNTED_REPLY_MARKER = 'E2E_SLOW_COUNTED_REPLY:';
const STEER_TOOL_REPLY_MARKER = 'E2E_STEER_TOOL_REPLY:';
const STEER_SPLIT_REPLY_MARKER = 'E2E_STEER_SPLIT_REPLY:';
const STEER_LATE_REPLY_MARKER = 'E2E_STEER_LATE_REPLY:';
const RESUME_ICON_REPLY_MARKER = 'E2E_RESUME_ICON_REPLY:';
const FORCED_ERROR_MARKER = 'E2E_FORCED_ERROR:';
const MARKDOWN_REPLY_MARKER = 'E2E_MARKDOWN_REPLY';
@ -49,6 +51,8 @@ const PROVIDER_FILE_ASSERTION_FINAL_TEXT = 'E2E provider file assertion passed';
const AGENT_CONTEXT_ASSERTION_FINAL_TEXT = 'E2E agent context assertion passed';
const QUOTE_ASSERTION_FINAL_TEXT = 'E2E quote assertion passed';
const STEER_TOOL_FINAL_TEXT = 'E2E steer tool reply done';
const STEER_SPLIT_FINAL_TEXT = 'E2E steer split reply done';
const STEER_LATE_FINAL_TEXT = 'E2E steer late reply done';
const STEER_TOOL_NAME_PREFIX = 'remember_fact';
const SLOW_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_SLOW_CHUNK_DELAY_MS) || 35;
const SLOW_REPLY_CHUNKS = 160;
@ -859,21 +863,139 @@ function steerToolReplyResponses(label, toolNames) {
],
};
}
const chunks = Array.from(
let invocation = 0;
return {
responses: [''],
sleep: SLOW_CHUNK_DELAY_MS,
resolveInvocation: async (messages) => {
invocation += 1;
if (invocation === 1) {
return {
response: `E2E steer tool preamble ${label} ${slowChunkPayload()}`,
toolCalls: [
{
id: `call_e2e_steer_${label}`,
name: toolName,
args: { fact: `steer boundary ${label}` },
type: 'tool_call',
},
],
};
}
return { response: `${STEER_TOOL_FINAL_TEXT} ${label} ${steerEchoSuffix(messages)}` };
},
};
}
/**
* Model-visible injection proof: echoes every steer-injected user message the
* model actually received (`additional_kwargs.source === 'steer'`, stamped by
* the SDK's `convertInjectedMessages`), so specs can assert the words reached
* the model rather than only that the UI rendered a part.
*/
function steerEchoSuffix(messages) {
const steerTexts = (messages ?? [])
.filter((message) => message?.additional_kwargs?.source === 'steer')
.map((message) => getContentText(message.content));
return `[steers-seen=${steerTexts.length}] ${steerTexts.join(' | ')}`.trim();
}
/** Slow word-chunk payload shared by the steer scenarios. */
function slowChunkPayload() {
return Array.from(
{ length: SLOW_REPLY_CHUNKS },
(_, index) => `chunk-${String(index).padStart(3, '0')}`,
).join(' ');
}
/**
* Three-turn run with TWO tool boundaries for the split-steer e2e: turn 1
* streams a slow preamble then calls the MCP tool (boundary A), turn 2 streams
* a slow middle segment then calls it again (boundary B), turn 3 streams the
* final text. Lets a test land one steer before each boundary.
*/
function steerSplitReplyResponses(label, toolNames) {
const toolName = Array.from(toolNames).find((name) => name.startsWith(STEER_TOOL_NAME_PREFIX));
if (!toolName) {
return {
responses: [
`E2E steer split reply unavailable: no ${STEER_TOOL_NAME_PREFIX} tool advertised.`,
],
};
}
let invocation = 0;
return {
responses: [`E2E steer tool preamble ${label} ${chunks}`, `${STEER_TOOL_FINAL_TEXT} ${label}`],
responses: [''],
sleep: SLOW_CHUNK_DELAY_MS,
toolCalls: [
{
id: `call_e2e_steer_${label}`,
name: toolName,
args: { fact: `steer boundary ${label}` },
type: 'tool_call',
},
],
resolveInvocation: async (messages) => {
invocation += 1;
if (invocation === 1) {
return {
response: `E2E steer split preamble ${label} ${slowChunkPayload()}`,
toolCalls: [
{
id: `call_e2e_steer_split_a_${label}`,
name: toolName,
args: { fact: `steer split boundary A ${label}` },
type: 'tool_call',
},
],
};
}
if (invocation === 2) {
return {
response: `E2E steer split middle ${label} ${slowChunkPayload()}`,
toolCalls: [
{
id: `call_e2e_steer_split_b_${label}`,
name: toolName,
args: { fact: `steer split boundary B ${label}` },
type: 'tool_call',
},
],
};
}
return { response: `${STEER_SPLIT_FINAL_TEXT} ${label} ${steerEchoSuffix(messages)}` };
},
};
}
/**
* Two-turn run whose FINAL segment streams slowly: turn 1 streams a slow
* preamble then calls the MCP tool (the only boundary), turn 2 streams a slow
* final text. Lets a test submit a steer AFTER the last boundary no drain
* point remains, so the terminal path must convert it to a queued follow-up.
*/
function steerLateReplyResponses(label, toolNames) {
const toolName = Array.from(toolNames).find((name) => name.startsWith(STEER_TOOL_NAME_PREFIX));
if (!toolName) {
return {
responses: [
`E2E steer late reply unavailable: no ${STEER_TOOL_NAME_PREFIX} tool advertised.`,
],
};
}
let invocation = 0;
return {
responses: [''],
sleep: SLOW_CHUNK_DELAY_MS,
resolveInvocation: async () => {
invocation += 1;
if (invocation === 1) {
return {
response: `E2E steer late preamble ${label} ${slowChunkPayload()}`,
toolCalls: [
{
id: `call_e2e_steer_late_${label}`,
name: toolName,
args: { fact: `steer late boundary ${label}` },
type: 'tool_call',
},
],
};
}
return { response: `${STEER_LATE_FINAL_TEXT} ${label} ${slowChunkPayload()}` };
},
};
}
@ -1436,6 +1558,16 @@ function resolveResponses({ graph, messages, text, toolNames }) {
return steerToolReplyResponses(steerToolLabel, toolNames);
}
const steerSplitLabel = getMarkerValue(text, STEER_SPLIT_REPLY_MARKER);
if (steerSplitLabel) {
return steerSplitReplyResponses(steerSplitLabel, toolNames);
}
const steerLateLabel = getMarkerValue(text, STEER_LATE_REPLY_MARKER);
if (steerLateLabel) {
return steerLateReplyResponses(steerLateLabel, toolNames);
}
if (text.includes(ASSERT_AGENT_CONTEXT_MARKER)) {
return {
responses: [MOCK_REPLY],

View file

@ -119,6 +119,11 @@ test.describe('mid-run steering and queuing', () => {
await expect(messagesView(page).getByText(`E2E steer tool reply done ${label}`)).toBeVisible({
timeout: 60000,
});
// Ordered content proof, not just a count: the echo carries the exact
// injected words in message order.
await expect(messagesView(page).getByText(`[steers-seen=1] ${steerText}`)).toBeVisible({
timeout: 30000,
});
// The steer stays INSIDE the response after run end — a user message at
// its injection point, not a queued follow-up turn (4 turns: the setup
@ -129,6 +134,285 @@ test.describe('mid-run steering and queuing', () => {
await expect(queuedRows(page)).toHaveCount(0);
});
/**
* Two steers submitted in quick succession must BOTH inject at the next
* tool-batch boundary: the drain is an atomic take-all, the hook returns one
* injected message per item, and the host applies one content part per item.
* Regression: only one of two waiting steers went through.
*/
test('steers twice in succession: both waiting bubbles inject at the same tool boundary', async ({
page,
}) => {
test.setTimeout(150000);
const label = uniqueLabel('steer2');
const firstSteer = `First steer ${label}`;
const secondSteer = `Second steer ${label}`;
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
await selectMockEndpoint(page, PROVIDER_C);
await selectEphemeralMCP(page);
await establishConversation(page, `steer2-setup-${label}`);
const run = await sendMessage(page, `E2E_STEER_TOOL_REPLY:${label}`);
expect(run.ok()).toBeTruthy();
await typeDuringRun(page, firstSteer);
const [firstResponse] = await Promise.all([
page.waitForResponse(isSteerRequest, { timeout: 15000 }),
messageInput(page).press('Enter'),
]);
expect(firstResponse.status()).toBe(202);
await typeDuringRun(page, secondSteer);
const [secondResponse] = await Promise.all([
page.waitForResponse(isSteerRequest, { timeout: 15000 }),
messageInput(page).press('Enter'),
]);
expect(secondResponse.status()).toBe(202);
// Both steers wait as anchored bubbles — nothing injected yet.
await expect(inFlightSteers(page).filter({ hasText: firstSteer })).toHaveCount(1, {
timeout: 10000,
});
await expect(inFlightSteers(page).filter({ hasText: secondSteer })).toHaveCount(1, {
timeout: 10000,
});
// At the boundary, BOTH inject as in-thread parts, in submission order.
await expect(appliedSteerParts(page).filter({ hasText: firstSteer })).toHaveCount(1, {
timeout: 60000,
});
await expect(appliedSteerParts(page).filter({ hasText: secondSteer })).toHaveCount(1, {
timeout: 60000,
});
await expect(inFlightSteers(page)).toHaveCount(0);
await expect(messagesView(page).getByText(`E2E steer tool reply done ${label}`)).toBeVisible({
timeout: 60000,
});
// Model-visible proof: the fake model echoes the steer-injected user
// messages it actually received on the post-boundary turn — both unique
// texts, in submission order, so duplicated or swapped words fail here.
await expect(
messagesView(page).getByText(`[steers-seen=2] ${firstSteer} | ${secondSteer}`),
).toBeVisible({ timeout: 30000 });
// Both survive run end inside the response — no queued follow-ups, no
// extra turns (setup pair + this pair).
await expect(messageTurns(page)).toHaveCount(4);
await expect(appliedSteerParts(page)).toHaveCount(2);
await expect(queuedRows(page)).toHaveCount(0);
});
/**
* Human-cadence variant: the second steer is submitted while the FIRST
* steer's 202 is still pending, so the two POSTs (and their chip upserts)
* genuinely overlap in flight. The overlap is enforced, not raced: a route
* intercept forwards the first POST to the server (the enqueue happens in
* submission order) but holds its 202 from the client until the second POST
* has been observed. Both must still inject.
*/
test('steers twice rapidly (second sent before the first ACK): both inject', async ({ page }) => {
test.setTimeout(150000);
const label = uniqueLabel('steerrapid');
const firstSteer = `Rapid first steer ${label}`;
const secondSteer = `Rapid second steer ${label}`;
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
await selectMockEndpoint(page, PROVIDER_C);
await selectEphemeralMCP(page);
await establishConversation(page, `steerrapid-setup-${label}`);
const run = await sendMessage(page, `E2E_STEER_TOOL_REPLY:${label}`);
expect(run.ok()).toBeTruthy();
let releaseFirstAck!: () => void;
const secondPosted = new Promise<void>((resolve) => (releaseFirstAck = resolve));
let steersSeen = 0;
let overlapProven = false;
await page.route('**/api/agents/chat/steer', async (route) => {
const ordinal = ++steersSeen;
if (ordinal === 2) {
releaseFirstAck();
}
// Forward to the real server first: the enqueue lands in submission
// order; only the CLIENT-side 202 delivery is held back.
const response = await route.fetch();
if (ordinal === 1) {
// Bounded so a broken second submit fails on assertions, not a hang.
await Promise.race([
secondPosted.then(() => {
overlapProven = true;
}),
new Promise<void>((resolve) => setTimeout(resolve, 10000)),
]);
}
await route.fulfill({ response });
});
const steerResponseFor = (text: string) =>
page.waitForResponse(
(response) =>
isSteerRequest(response) && response.request().postData()?.includes(text) === true,
{ timeout: 15000 },
);
const responses: Promise<Response>[] = [steerResponseFor(firstSteer)];
await typeDuringRun(page, firstSteer);
await messageInput(page).press('Enter');
responses.push(steerResponseFor(secondSteer));
await typeDuringRun(page, secondSteer);
await messageInput(page).press('Enter');
const [firstResponse, secondResponse] = await Promise.all(responses);
expect(firstResponse.status()).toBe(202);
expect(secondResponse.status()).toBe(202);
// The hold proves the overlap actually happened: the second POST was
// observed while the first 202 was still parked in the intercept.
expect(steersSeen).toBe(2);
expect(overlapProven).toBe(true);
await page.unroute('**/api/agents/chat/steer');
await expect(appliedSteerParts(page).filter({ hasText: firstSteer })).toHaveCount(1, {
timeout: 60000,
});
await expect(appliedSteerParts(page).filter({ hasText: secondSteer })).toHaveCount(1, {
timeout: 60000,
});
await expect(inFlightSteers(page)).toHaveCount(0);
await expect(messagesView(page).getByText(`E2E steer tool reply done ${label}`)).toBeVisible({
timeout: 60000,
});
await expect(
messagesView(page).getByText(`[steers-seen=2] ${firstSteer} | ${secondSteer}`),
).toBeVisible({ timeout: 30000 });
await expect(messageTurns(page)).toHaveCount(4);
await expect(appliedSteerParts(page)).toHaveCount(2);
await expect(queuedRows(page)).toHaveCount(0);
});
/**
* Two steers split across DIFFERENT tool boundaries: the first drains at
* boundary A, the second is submitted while the next segment streams and
* must drain at boundary B. Regression guard for the succession case where
* a boundary falls between the two submissions.
*/
test('steers split across two tool boundaries: each injects at its own boundary', async ({
page,
}) => {
test.setTimeout(180000);
const label = uniqueLabel('steersplit');
const firstSteer = `Boundary A steer ${label}`;
const secondSteer = `Boundary B steer ${label}`;
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
await selectMockEndpoint(page, PROVIDER_C);
await selectEphemeralMCP(page);
await establishConversation(page, `steersplit-setup-${label}`);
const run = await sendMessage(page, `E2E_STEER_SPLIT_REPLY:${label}`);
expect(run.ok()).toBeTruthy();
// First steer lands during the turn-1 preamble.
await typeDuringRun(page, firstSteer);
const [firstResponse] = await Promise.all([
page.waitForResponse(isSteerRequest, { timeout: 15000 }),
messageInput(page).press('Enter'),
]);
expect(firstResponse.status()).toBe(202);
// Boundary A injects it while turn 2 is still ahead.
await expect(appliedSteerParts(page).filter({ hasText: firstSteer })).toHaveCount(1, {
timeout: 60000,
});
// Second steer lands during the turn-2 middle segment.
await typeDuringRun(page, secondSteer);
const [secondResponse] = await Promise.all([
page.waitForResponse(isSteerRequest, { timeout: 15000 }),
messageInput(page).press('Enter'),
]);
expect(secondResponse.status()).toBe(202);
await expect(inFlightSteers(page).filter({ hasText: secondSteer })).toHaveCount(1, {
timeout: 10000,
});
// Boundary B injects the second steer too.
await expect(appliedSteerParts(page).filter({ hasText: secondSteer })).toHaveCount(1, {
timeout: 60000,
});
await expect(inFlightSteers(page)).toHaveCount(0);
await expect(messagesView(page).getByText(`E2E steer split reply done ${label}`)).toBeVisible({
timeout: 60000,
});
// The post-boundary-B turn must have BOTH injected steers in its context,
// as the exact words in submission order.
await expect(
messagesView(page).getByText(`[steers-seen=2] ${firstSteer} | ${secondSteer}`),
).toBeVisible({ timeout: 30000 });
await expect(messageTurns(page)).toHaveCount(4);
await expect(appliedSteerParts(page)).toHaveCount(2);
await expect(queuedRows(page)).toHaveCount(0);
});
/**
* A steer submitted AFTER the run's last tool boundary can never inject:
* the terminal drain reports it on the final event and the client must
* convert it to a queued follow-up and auto-send it as the next turn
* the user's words go through either way, never silently dropped.
*/
test('steer after the last tool boundary converts to a queued follow-up and auto-sends', async ({
page,
}) => {
test.setTimeout(180000);
const label = uniqueLabel('steerlate');
const firstSteer = `Injected steer ${label}`;
const lateSteer = `Late steer ${label}`;
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
await selectMockEndpoint(page, PROVIDER_C);
await selectEphemeralMCP(page);
await establishConversation(page, `steerlate-setup-${label}`);
const run = await sendMessage(page, `E2E_STEER_LATE_REPLY:${label}`);
expect(run.ok()).toBeTruthy();
// First steer lands during the preamble and injects at the only boundary.
await typeDuringRun(page, firstSteer);
const [firstResponse] = await Promise.all([
page.waitForResponse(isSteerRequest, { timeout: 15000 }),
messageInput(page).press('Enter'),
]);
expect(firstResponse.status()).toBe(202);
await expect(appliedSteerParts(page).filter({ hasText: firstSteer })).toHaveCount(1, {
timeout: 60000,
});
// The final segment is streaming now (its lead text is already visible) —
// this steer arrives after the last boundary.
await expect(messagesView(page).getByText(`E2E steer late reply done ${label}`)).toBeVisible({
timeout: 60000,
});
await typeDuringRun(page, lateSteer);
const [lateResponse] = await Promise.all([
page.waitForResponse(isSteerRequest, { timeout: 15000 }),
messageInput(page).press('Enter'),
]);
expect(lateResponse.status()).toBe(202);
// Never injected — converted to a queued follow-up at run end and
// auto-sent as the next user turn (6 turns: setup pair, this pair,
// auto-sent follow-up pair).
await expect(messageTurns(page)).toHaveCount(6, { timeout: 90000 });
const followupTurn = messageTurns(page).nth(4);
await expect(followupTurn).toContainText(lateSteer);
await expect(followupTurn.locator('.user-turn')).toBeVisible();
await expect(messageTurns(page).nth(5)).toContainText(MOCK_REPLY_TEXT, { timeout: 30000 });
await expect(appliedSteerParts(page)).toHaveCount(1);
await expect(inFlightSteers(page)).toHaveCount(0);
await expect(queuedRows(page)).toHaveCount(0);
});
test('queues with Cmd/Ctrl+Enter during a run and auto-sends after clean completion', async ({
page,
}) => {