test: cover tool approval workflows end to end

This commit is contained in:
Danny Avila 2026-07-24 20:50:10 -04:00
parent f3159f9891
commit 90ac3d25a5
12 changed files with 1064 additions and 5 deletions

View file

@ -281,6 +281,9 @@ export function useResumeSubmit() {
const approvalMutation = useSubmitToolApprovalMutation();
const askMutation = useSubmitAskAnswerMutation();
const { getDecisions, isReady, setStatus } = useApprovalContext();
/** React state cannot lock a second click in the same browser task. Keep a
* synchronous action-id guard alongside the rendered submission status. */
const submittingToolActionIdsRef = useRef(new Set<string>());
/** Ask status lives in Recoil so it works from the composer (outside the
* provider); tool-approval status stays on the context. */
const { setAskStatus } = useAskSubmitStatus();
@ -311,12 +314,23 @@ export function useResumeSubmit() {
if (!fields || decisions.length === 0 || !isReady(actionId)) {
return;
}
if (submittingToolActionIdsRef.current.has(actionId)) {
return;
}
submittingToolActionIdsRef.current.add(actionId);
setStatus(actionId, 'submitting');
approvalMutation.mutate(
{ ...fields, actionId, decisions },
{
onSuccess: () => setStatus(actionId, 'submitted'),
onError: (error) => setStatus(actionId, isExpiredError(error) ? 'expired' : 'error'),
onError: (error) => {
const expired = isExpiredError(error);
if (!expired) {
// Network/validation failures are retryable; a 409 is terminal.
submittingToolActionIdsRef.current.delete(actionId);
}
setStatus(actionId, expired ? 'expired' : 'error');
},
},
);
},

View file

@ -174,7 +174,11 @@ export default function ToolApproval({
}
return (
<div className="my-2 flex w-full flex-col gap-2 rounded-lg border border-border-light bg-surface-secondary p-3">
<div
className="my-2 flex w-full flex-col gap-2 rounded-lg border border-border-light bg-surface-secondary p-3"
data-testid="tool-approval"
data-tool-call-id={toolCallId}
>
{description != null && description.length > 0 && (
<p className="text-sm text-text-secondary">{description}</p>
)}

View file

@ -110,6 +110,17 @@ export default function ToolCallGroup({
const count = parts.length;
const toolMetadata = useMemo(() => parts.map((p) => getToolMeta(p.part)), [parts]);
const hasPendingApproval = useMemo(
() =>
parts.some(({ part }) => {
if (part.type !== ContentTypes.TOOL_CALL) {
return false;
}
const toolCall = part[ContentTypes.TOOL_CALL] as Agents.ToolCall | undefined;
return toolCall?.approval != null && (toolCall.output?.length ?? 0) === 0;
}),
[parts],
);
const allCompleted = useMemo(
() => toolMetadata.every((m) => m?.hasOutput === true),
[toolMetadata],
@ -223,13 +234,16 @@ export default function ToolCallGroup({
if (event.target !== event.currentTarget) {
return;
}
if (isExpanded) {
if (isExpanded || hasPendingApproval) {
return;
}
// Approval controls own unsent local form state. Keep unresolved cards
// mounted (the collapsed panel is inert/hidden) so collapsing a batch
// cannot erase decisions the reviewer already made.
setShouldRenderBody(false);
notifyLayoutChange();
},
[isExpanded, notifyLayoutChange],
[hasPendingApproval, isExpanded, notifyLayoutChange],
);
/** Category-aware header verb: subagents and questions read as their own

View file

@ -0,0 +1,76 @@
import React from 'react';
import { RecoilRoot } from 'recoil';
import { act, renderHook } from '@testing-library/react';
import { ChatContext } from '~/Providers/ChatContext';
import ApprovalProvider, { useApprovalContext, useResumeSubmit } from '../ApprovalContext';
const mockApprovalMutate = jest.fn();
const mockAskMutate = jest.fn();
jest.mock('~/data-provider', () => ({
useSubmitToolApprovalMutation: () => ({ mutate: mockApprovalMutate }),
useSubmitAskAnswerMutation: () => ({ mutate: mockAskMutate }),
}));
jest.mock('~/store/agents', () => ({
useGetEphemeralAgent: () => () => undefined,
}));
const chatContextValue = {
conversation: {
conversationId: 'conversation-1',
endpoint: 'agents',
agent_id: 'agent-1',
},
} as unknown as React.ContextType<typeof ChatContext>;
function wrapper({ children }: { children: React.ReactNode }) {
return (
<RecoilRoot>
<ChatContext.Provider value={chatContextValue}>
<ApprovalProvider>{children}</ApprovalProvider>
</ChatContext.Provider>
</RecoilRoot>
);
}
describe('useResumeSubmit', () => {
beforeEach(() => {
jest.clearAllMocks();
});
test('synchronously deduplicates tool approval submissions and unlocks a retryable error', () => {
const { result } = renderHook(
() => ({
approval: useApprovalContext(),
resume: useResumeSubmit(),
}),
{ wrapper },
);
act(() => {
result.current.approval.registerToolCall('action-1', 'call-1');
result.current.approval.setDecision('action-1', 'call-1', {
tool_call_id: 'call-1',
decision: 'approve',
});
});
act(() => {
result.current.resume.submitToolApproval('action-1');
result.current.resume.submitToolApproval('action-1');
});
expect(mockApprovalMutate).toHaveBeenCalledTimes(1);
const firstOptions = mockApprovalMutate.mock.calls[0][1] as {
onError: (error: unknown) => void;
};
act(() => firstOptions.onError(new Error('temporary failure')));
act(() => {
result.current.resume.submitToolApproval('action-1');
result.current.resume.submitToolApproval('action-1');
});
expect(mockApprovalMutate).toHaveBeenCalledTimes(2);
});
});

View file

@ -83,6 +83,21 @@ const makePart = (
},
}) as unknown as TMessageContentParts;
const makePendingApprovalPart = (id: string): TMessageContentParts =>
({
type: ContentTypes.TOOL_CALL,
[ContentTypes.TOOL_CALL]: {
id,
name: 'approval_probe',
args: {},
output: '',
approval: {
actionId: 'action-1',
allowed_decisions: ['approve', 'reject'],
},
},
}) as unknown as TMessageContentParts;
const imageAttachment: TAttachment = {
filename: 'foo.png',
filepath: '/files/foo.png',
@ -211,6 +226,33 @@ describe('ToolCallGroup image hoisting', () => {
expect(screen.queryByTestId('inner-0')).not.toBeInTheDocument();
});
it('keeps unresolved approval bodies mounted while the group is collapsed', () => {
const approvalParts = [
{ part: makePendingApprovalPart('t1'), idx: 0 },
{ part: makePendingApprovalPart('t2'), idx: 1 },
];
renderGroup({
...baseProps,
parts: approvalParts,
renderPart: (_p: TMessageContentParts, idx: number) => (
<div data-testid={`approval-${idx}`} key={idx}>
{'approval'}
</div>
),
});
const button = screen.getByRole('button', { name: 'Used 2 tools' });
const collapsible = button.nextElementSibling as HTMLElement;
expect(screen.getByTestId('approval-0')).toBeInTheDocument();
fireEvent.click(button);
fireEvent.transitionEnd(collapsible);
expect(button).toHaveAttribute('aria-expanded', 'false');
expect(screen.getByTestId('approval-0')).toBeInTheDocument();
expect(screen.getByTestId('approval-1')).toBeInTheDocument();
});
it('reconciles layout after the group collapses from an expanded state', async () => {
renderGroup(baseProps);

View file

@ -71,6 +71,37 @@ describe('applyPendingAction — tool_approval', () => {
});
});
it('replaces displayed tool args with the matching action request arguments', () => {
const originalArgs = { query: 'original model args' };
const rewrittenArgs = { query: 'rewritten by policy hook' };
const message = msg({ content: [toolCallPart('tc1', { args: originalArgs })] });
const action = toolApprovalAction({
payload: {
type: 'tool_approval',
action_requests: [
{
name: 'search',
arguments: rewrittenArgs,
tool_call_id: 'tc1',
description: 'Review rewritten search',
},
],
review_configs: [
{
action_name: 'search',
tool_call_id: 'tc1',
allowed_decisions: ['approve', 'reject', 'edit', 'respond'],
},
],
},
});
const result = applyPendingAction(message, action);
expect(getToolCall(result.content?.[0] as TMessageContentParts)?.args).toEqual(rewrittenArgs);
expect(getToolCall(message.content?.[0] as TMessageContentParts)?.args).toEqual(originalArgs);
});
it('leaves a completed tool call (with output) untouched and returns the same message reference', () => {
const message = msg({ content: [toolCallPart('tc1', { output: 'already ran' })] });
const result = applyPendingAction(message, toolApprovalAction());

View file

@ -95,6 +95,10 @@ function tagApprovalOnPart(
const reviewConfig = reviewByToolCallId.get(toolCallId);
nextToolCall = {
...nextToolCall,
// A PreToolUse hook may replace the model's original args before asking.
// The interrupt payload is authoritative so the reviewer sees, edits, and
// approves the same arguments the resumed tool will actually execute.
args: request.arguments,
approval: {
actionId,
allowed_decisions: reviewConfig?.allowed_decisions ?? [],

View file

@ -60,6 +60,18 @@ endpoints:
- chain
- ocr
- run_in_background
# Keep the shared mock profile non-interactive except for the dedicated
# approval probe. This exercises real HITL pause/resume without wedging the
# existing file-authoring, steering, background-tool, or MCP specs.
toolApproval:
enabled: true
mode: bypass
ask:
- approval_probe_mcp_e2e-memory
reason: E2E approval required before running {tool}.
hooks:
- module: e2e/setup/tool-approval-hook.js
matcher: ^approval_probe_mcp_e2e-memory$
custom:
- name: 'Mock Provider A'
apiKey: 'e2e-mock-key-a'

View file

@ -66,6 +66,26 @@ server.registerTool(
},
);
server.registerTool(
'approval_probe',
{
description:
'Echoes reviewed input so LibreChat mock end-to-end tests can verify tool approval decisions.',
inputSchema: {
value: z.string(),
review: z.string().optional(),
},
},
async ({ value }) => ({
content: [
{
type: 'text',
text: `E2E approval probe executed: ${value}`,
},
],
}),
);
async function main() {
await server.connect(new StdioServerTransport());
}

View file

@ -34,6 +34,10 @@ const FORCED_ERROR_MARKER = 'E2E_FORCED_ERROR:';
const MARKDOWN_REPLY_MARKER = 'E2E_MARKDOWN_REPLY';
const BACKGROUND_DISPATCH_MARKER = 'E2E_BACKGROUND_DISPATCH:';
const BACKGROUND_COLLECT_MARKER = 'E2E_BACKGROUND_COLLECT:';
const TOOL_APPROVAL_MARKER = 'E2E_TOOL_APPROVAL:';
const TOOL_APPROVAL_BATCH_MARKER = 'E2E_TOOL_APPROVAL_BATCH:';
const TOOL_APPROVAL_RESTRICTED_MARKER = 'E2E_TOOL_APPROVAL_RESTRICTED:';
const TOOL_APPROVAL_REWRITE_MARKER = 'E2E_TOOL_APPROVAL_REWRITE:';
const CREATE_FILE_AUTHORING_FINAL_TEXT = 'E2E file authoring complete';
const EDIT_FILE_AUTHORING_FINAL_TEXT = 'E2E file edit complete';
const SKILL_ASSERTION_FINAL_TEXT = 'E2E skill assertion passed';
@ -56,6 +60,8 @@ const CREATE_SKILL_TOOL_CALL_ID = 'call_e2e_create_skill';
const EDIT_SKILL_TOOL_CALL_ID = 'call_e2e_edit_skill';
const BACKGROUND_TOOL_NAME = 'slow_echo_mcp_e2e-memory';
const CHECK_BACKGROUND_TASK_TOOL_NAME = 'check_background_task';
const APPROVAL_TOOL_NAME = 'approval_probe_mcp_e2e-memory';
const APPROVAL_TOOL_CALL_PREFIX = 'call_e2e_approval_';
const BACKGROUND_DISPATCH_TOOL_CALL_ID = 'call_e2e_background_dispatch';
const BACKGROUND_COLLECT_TOOL_CALL_ID = 'call_e2e_background_collect';
const MODEL_SPEC_ACCESSIBLE_SKILL = 'e2e-model-spec-allowed';
@ -832,6 +838,92 @@ function findLastToolMessageText(messages, requiredToken) {
return '';
}
function approvalToolResponses(label, toolNames, review) {
if (!toolNames.has(APPROVAL_TOOL_NAME)) {
return {
responses: [`E2E approval unavailable: ${APPROVAL_TOOL_NAME} was not advertised.`],
};
}
return {
responses: ['', ''],
toolCalls: [
{
id: `${APPROVAL_TOOL_CALL_PREFIX}${label}`,
name: APPROVAL_TOOL_NAME,
args: {
value: `original-${label}`,
...(review ? { review } : {}),
},
type: 'tool_call',
},
],
};
}
function batchApprovalToolResponses(label, toolNames) {
if (!toolNames.has(APPROVAL_TOOL_NAME)) {
return {
responses: [`E2E approval unavailable: ${APPROVAL_TOOL_NAME} was not advertised.`],
};
}
return {
responses: ['', ''],
toolCalls: [
{
id: `${APPROVAL_TOOL_CALL_PREFIX}${label}_first`,
name: APPROVAL_TOOL_NAME,
args: { value: `first-${label}` },
type: 'tool_call',
},
{
id: `${APPROVAL_TOOL_CALL_PREFIX}${label}_second`,
name: APPROVAL_TOOL_NAME,
args: { value: `second-${label}` },
type: 'tool_call',
},
],
};
}
/**
* Resume rebuilds the fake model without the original prompt in `context.messages`.
* Detect the checkpoint-restored approval tool messages on every model instance
* so the continuation can report the real approve/reject/edit/respond outcome.
*/
function approvalOutcomeResponses(messages) {
let latestHumanIndex = -1;
for (let index = 0; index < (messages ?? []).length; index++) {
const type = messageType(messages[index]);
if (type === 'human' || type === 'user') {
latestHumanIndex = index;
}
}
const outcomeMessages = (messages ?? [])
.slice(latestHumanIndex + 1)
.filter(
(message) =>
messageType(message) === 'tool' &&
typeof message?.tool_call_id === 'string' &&
message.tool_call_id.startsWith(APPROVAL_TOOL_CALL_PREFIX),
);
const isBatch = outcomeMessages.some(
(message) =>
message.tool_call_id.endsWith('_first') || message.tool_call_id.endsWith('_second'),
);
if (isBatch && outcomeMessages.length < 2) {
return null;
}
const outcomes = outcomeMessages.map((message) => getContentText(message.content));
if (outcomes.length === 0) {
return null;
}
return { responses: [`E2E approval outcomes: ${outcomes.join(' | ')}`] };
}
/**
* Turn 1 of the background e2e: emit the MCP tool call with the injected
* `run_in_background: true` arg, then (second model invocation, after the
@ -925,6 +1017,26 @@ function backgroundCollectResponses(messages, toolNames) {
}
function resolveResponses({ graph, messages, text, toolNames }) {
const batchApprovalLabel = getMarkerValue(text, TOOL_APPROVAL_BATCH_MARKER);
if (batchApprovalLabel) {
return batchApprovalToolResponses(batchApprovalLabel, toolNames);
}
const restrictedApprovalLabel = getMarkerValue(text, TOOL_APPROVAL_RESTRICTED_MARKER);
if (restrictedApprovalLabel) {
return approvalToolResponses(restrictedApprovalLabel, toolNames, 'restricted');
}
const rewrittenApprovalLabel = getMarkerValue(text, TOOL_APPROVAL_REWRITE_MARKER);
if (rewrittenApprovalLabel) {
return approvalToolResponses(rewrittenApprovalLabel, toolNames, 'rewrite');
}
const approvalLabel = getMarkerValue(text, TOOL_APPROVAL_MARKER);
if (approvalLabel) {
return approvalToolResponses(approvalLabel, toolNames);
}
const reply = replyResponses(text);
if (reply) {
return reply;
@ -1055,5 +1167,13 @@ module.exports = function fakeModelHook(run, context) {
text,
toolNames,
});
overrideModel({ graph, responses, sleep, toolCalls, thrownError, resolveOnStream });
overrideModel({
graph,
responses,
sleep,
toolCalls,
thrownError,
resolveOnStream: (streamMessages) =>
approvalOutcomeResponses(streamMessages) ?? resolveOnStream?.(streamMessages) ?? null,
});
};

View file

@ -0,0 +1,29 @@
/**
* Dynamic approval-policy fixture for the mock Playwright suite.
*
* The `review` argument selects behavior that cannot be expressed by the static
* ask list: a restricted decision set, or an authoritative argument rewrite.
*/
module.exports = () => () => async (input) => {
if (input.toolInput.review === 'restricted') {
return {
decision: 'ask',
reason: 'E2E approval offers approve or reject only.',
allowedDecisions: ['approve', 'reject'],
};
}
if (input.toolInput.review === 'rewrite') {
const originalValue =
typeof input.toolInput.value === 'string' ? input.toolInput.value : 'original-missing';
return {
decision: 'ask',
reason: 'E2E approval reviews rewritten arguments.',
updatedInput: {
value: originalValue.replace(/^original-/, 'rewritten-'),
},
};
}
return {};
};

View file

@ -0,0 +1,693 @@
import { expect, test } from '@playwright/test';
import type { Locator, Page, Request, Route } from '@playwright/test';
import type { AgentDetail } from './agents.helpers';
import { cleanupAgent, openAgentBuilder, uniqueAgentName } from './agents.helpers';
import {
MOCK_ENDPOINTS,
NEW_CHAT_PATH,
fetchJson,
getAccessToken,
messagesView,
requestJson,
sendMessage,
} from './helpers';
const MCP_SERVER_NAME = 'e2e-memory';
const MCP_SERVER_TOOL_ID = `sys__server__sys_mcp_${MCP_SERVER_NAME}`;
const APPROVAL_TOOL_NAME = 'approval_probe';
const APPROVAL_TOOL_ID = `${APPROVAL_TOOL_NAME}_mcp_${MCP_SERVER_NAME}`;
const APPROVAL_PROMPT_MARKER = 'E2E_TOOL_APPROVAL:';
const BATCH_APPROVAL_PROMPT_MARKER = 'E2E_TOOL_APPROVAL_BATCH:';
const RESTRICTED_APPROVAL_PROMPT_MARKER = 'E2E_TOOL_APPROVAL_RESTRICTED:';
const REWRITTEN_APPROVAL_PROMPT_MARKER = 'E2E_TOOL_APPROVAL_REWRITE:';
const APPROVAL_REASON = `E2E approval required before running ${APPROVAL_TOOL_ID}.`;
const APPROVAL_ERROR = 'Something went wrong submitting your decision. Please try again.';
const APPROVAL_EXPIRED = 'This request expired or was already handled.';
const DESCRIPTION = 'Verifies human approval behavior for MCP tool calls in mock E2E tests.';
const uniqueLabel = () => `${Date.now()}-${Math.floor(Math.random() * 1e4)}`;
type MCPToolsResponse = {
servers?: Record<string, { tools?: Array<{ pluginKey: string }> }>;
};
type ApprovalResumeBody = {
actionId?: string;
agent_id?: string;
conversationId?: string;
endpoint?: string;
decisions?: Array<{
tool_call_id?: string;
decision?: string;
reason?: string;
responseText?: string;
editedArguments?: Record<string, unknown>;
}>;
};
type ApprovalResumeResponse = {
conversationId?: string;
status?: string;
streamId?: string;
};
const approvalCards = (page: Page) => messagesView(page).getByTestId('tool-approval');
const approvalCard = (page: Page, toolCallId: string) =>
messagesView(page).locator(`[data-testid="tool-approval"][data-tool-call-id="${toolCallId}"]`);
function isResumeRequest(request: Request) {
return (
request.method() === 'POST' && new URL(request.url()).pathname === '/api/agents/chat/resume'
);
}
async function waitForApprovalTool(page: Page) {
const token = await getAccessToken(page);
let latestTools: MCPToolsResponse | null = null;
for (let attempt = 0; attempt < 20; attempt++) {
latestTools = await fetchJson<MCPToolsResponse>(page, '/api/mcp/tools', token);
const tools = latestTools.servers?.[MCP_SERVER_NAME]?.tools ?? [];
if (tools.some((tool) => tool.pluginKey === APPROVAL_TOOL_ID)) {
return;
}
await page.waitForTimeout(500);
}
expect(
latestTools?.servers?.[MCP_SERVER_NAME]?.tools,
`Expected ${MCP_SERVER_NAME} to expose ${APPROVAL_TOOL_ID}`,
).toEqual(expect.arrayContaining([expect.objectContaining({ pluginKey: APPROVAL_TOOL_ID })]));
}
async function createAndSelectApprovalAgent(page: Page): Promise<string> {
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
await waitForApprovalTool(page);
const token = await getAccessToken(page);
const agentName = uniqueAgentName('E2E Tool Approval Agent');
const agent = await requestJson<AgentDetail>(page, {
path: '/api/agents',
token,
method: 'POST',
body: {
name: agentName,
description: DESCRIPTION,
instructions: 'Use the requested approval probe tools and report their results.',
provider: MOCK_ENDPOINTS[0].label,
model: MOCK_ENDPOINTS[0].model,
tools: [MCP_SERVER_TOOL_ID, APPROVAL_TOOL_ID],
},
});
const form = await openAgentBuilder(page);
await form.getByRole('combobox', { name: 'Agent', exact: true }).click();
await page.getByRole('option', { name: agentName }).click();
await expect(form.getByLabel('Agent name')).toHaveValue(agentName);
await form.getByRole('button', { name: 'Select Agent' }).click();
return agent.id;
}
async function startApproval(
page: Page,
label: string,
marker = APPROVAL_PROMPT_MARKER,
expectedReason = APPROVAL_REASON,
): Promise<Locator> {
const response = await sendMessage(page, `${marker}${label}`);
expect(response.ok()).toBeTruthy();
await expect(page).toHaveURL(/\/c\/(?!new)/, { timeout: 15000 });
const card = approvalCards(page).first();
await expect(card).toBeVisible({ timeout: 30000 });
await expect(card).toContainText(expectedReason);
return card;
}
async function submitAndCapture(page: Page, submit: Locator) {
const [request, response] = await Promise.all([
page.waitForRequest(isResumeRequest),
page.waitForResponse(
(candidate) => isResumeRequest(candidate.request()) && candidate.status() === 200,
),
submit.click(),
]);
return {
body: request.postDataJSON() as ApprovalResumeBody,
response,
};
}
test.describe('tool approvals', () => {
test('approves a paused tool with its original arguments', async ({ page }) => {
test.setTimeout(120000);
const label = uniqueLabel();
let agentId: string | undefined;
try {
agentId = await createAndSelectApprovalAgent(page);
const card = await startApproval(page, label);
await expect(card.getByRole('button', { name: 'Approve' })).toBeVisible();
await expect(card.getByRole('button', { name: 'Reject' })).toBeVisible();
await expect(card.getByRole('button', { name: 'Edit' })).toBeVisible();
await expect(card.getByRole('button', { name: 'Respond' })).toBeVisible();
const submit = card.getByRole('button', { name: 'Submit' });
await expect(submit).toBeDisabled();
await card.getByRole('button', { name: 'Approve' }).click();
await expect(submit).toBeEnabled();
const conversationId = new URL(page.url()).pathname.replace('/c/', '');
const { body, response } = await submitAndCapture(page, submit);
expect(body.actionId).toBeTruthy();
expect(body.agent_id).toBe(agentId);
expect(body.conversationId).toBe(conversationId);
expect(body.endpoint).toBe('agents');
expect(body.decisions).toEqual([
expect.objectContaining({
decision: 'approve',
tool_call_id: `call_e2e_approval_${label}`,
}),
]);
await expect(response.json() as Promise<ApprovalResumeResponse>).resolves.toEqual(
expect.objectContaining({
conversationId,
status: 'resuming',
streamId: conversationId,
}),
);
await expect(
messagesView(page).getByText(`E2E approval probe executed: original-${label}`),
).toBeVisible({ timeout: 30000 });
await expect(approvalCards(page)).toHaveCount(0);
} finally {
await cleanupAgent(page, agentId);
}
});
test('rejects with an optional reason without executing the tool', async ({ page }) => {
test.setTimeout(120000);
const label = uniqueLabel();
const reason = `do not run ${label}`;
let agentId: string | undefined;
try {
agentId = await createAndSelectApprovalAgent(page);
const card = await startApproval(page, label);
const submit = card.getByRole('button', { name: 'Submit' });
await card.getByRole('button', { name: 'Reject' }).click();
await card.getByRole('textbox', { name: 'Reject' }).fill(` ${reason} `);
await expect(submit).toBeEnabled();
const { body } = await submitAndCapture(page, submit);
expect(body.decisions).toEqual([
expect.objectContaining({
decision: 'reject',
reason,
tool_call_id: `call_e2e_approval_${label}`,
}),
]);
await expect(messagesView(page).getByText(`Blocked: ${reason}`)).toBeVisible({
timeout: 30000,
});
await expect(
messagesView(page).getByText(`E2E approval probe executed: original-${label}`),
).toHaveCount(0);
await expect(approvalCards(page)).toHaveCount(0);
} finally {
await cleanupAgent(page, agentId);
}
});
test('requires edited arguments to be a JSON object and executes only the edit', async ({
page,
}) => {
test.setTimeout(120000);
const label = uniqueLabel();
const editedValue = `edited-${label}`;
let agentId: string | undefined;
try {
agentId = await createAndSelectApprovalAgent(page);
const card = await startApproval(page, label);
const submit = card.getByRole('button', { name: 'Submit' });
await card.getByRole('button', { name: 'Edit' }).click();
const editor = card.getByRole('textbox', { name: 'Edit' });
await expect(editor).toHaveValue(new RegExp(`original-${label}`));
for (const invalid of ['{', 'null', '[]', '"text"']) {
await editor.fill(invalid);
await expect(card.getByText('Invalid JSON')).toBeVisible();
await expect(submit).toBeDisabled();
}
await editor.fill(JSON.stringify({ value: editedValue }));
await expect(card.getByText('Invalid JSON')).toHaveCount(0);
await expect(submit).toBeEnabled();
const { body } = await submitAndCapture(page, submit);
expect(body.decisions).toEqual([
expect.objectContaining({
decision: 'edit',
editedArguments: { value: editedValue },
tool_call_id: `call_e2e_approval_${label}`,
}),
]);
await expect(
messagesView(page).getByText(`E2E approval probe executed: ${editedValue}`),
).toBeVisible({ timeout: 30000 });
await expect(
messagesView(page).getByText(`E2E approval probe executed: original-${label}`),
).toHaveCount(0);
} finally {
await cleanupAgent(page, agentId);
}
});
test('requires a nonblank substitute response and skips tool execution', async ({ page }) => {
test.setTimeout(120000);
const label = uniqueLabel();
const responseText = `manual result ${label}`;
let agentId: string | undefined;
try {
agentId = await createAndSelectApprovalAgent(page);
const card = await startApproval(page, label);
const submit = card.getByRole('button', { name: 'Submit' });
await card.getByRole('button', { name: 'Respond' }).click();
const responseInput = card.getByRole('textbox', { name: 'Respond' });
await responseInput.fill(' ');
await expect(submit).toBeDisabled();
await responseInput.fill(` ${responseText} `);
await expect(submit).toBeEnabled();
const { body } = await submitAndCapture(page, submit);
expect(body.decisions).toEqual([
expect.objectContaining({
decision: 'respond',
responseText,
tool_call_id: `call_e2e_approval_${label}`,
}),
]);
await expect(messagesView(page).getByText(responseText)).toBeVisible({ timeout: 30000 });
await expect(
messagesView(page).getByText(`E2E approval probe executed: original-${label}`),
).toHaveCount(0);
} finally {
await cleanupAgent(page, agentId);
}
});
test('honors a hook-restricted decision set', async ({ page }) => {
test.setTimeout(120000);
const label = uniqueLabel();
let agentId: string | undefined;
try {
agentId = await createAndSelectApprovalAgent(page);
const card = await startApproval(
page,
label,
RESTRICTED_APPROVAL_PROMPT_MARKER,
APPROVAL_REASON,
);
await expect(card.getByRole('button', { name: 'Approve' })).toBeVisible();
await expect(card.getByRole('button', { name: 'Reject' })).toBeVisible();
await expect(card.getByRole('button', { name: 'Edit' })).toHaveCount(0);
await expect(card.getByRole('button', { name: 'Respond' })).toHaveCount(0);
const submit = card.getByRole('button', { name: 'Submit' });
await card.getByRole('button', { name: 'Approve' }).click();
const { body } = await submitAndCapture(page, submit);
expect(body.decisions).toEqual([
expect.objectContaining({
decision: 'approve',
tool_call_id: `call_e2e_approval_${label}`,
}),
]);
await expect(
messagesView(page).getByText(`E2E approval probe executed: original-${label}`),
).toBeVisible({ timeout: 30000 });
} finally {
await cleanupAgent(page, agentId);
}
});
test('reviews and approves the authoritative hook-rewritten arguments', async ({ page }) => {
test.setTimeout(120000);
const label = uniqueLabel();
let agentId: string | undefined;
try {
agentId = await createAndSelectApprovalAgent(page);
const card = await startApproval(
page,
label,
REWRITTEN_APPROVAL_PROMPT_MARKER,
APPROVAL_REASON,
);
await card.getByRole('button', { name: 'Edit' }).click();
const editor = card.getByRole('textbox', { name: 'Edit' });
await expect(editor).toHaveValue(new RegExp(`rewritten-${label}`));
await expect(editor).not.toHaveValue(new RegExp(`original-${label}`));
await card.getByRole('button', { name: 'Edit' }).click();
const submit = card.getByRole('button', { name: 'Submit' });
await card.getByRole('button', { name: 'Approve' }).click();
const { body } = await submitAndCapture(page, submit);
expect(body.decisions).toEqual([
expect.objectContaining({
decision: 'approve',
tool_call_id: `call_e2e_approval_${label}`,
}),
]);
await expect(
messagesView(page).getByText(`E2E approval probe executed: rewritten-${label}`),
).toBeVisible({ timeout: 30000 });
await expect(
messagesView(page).getByText(`E2E approval probe executed: original-${label}`),
).toHaveCount(0);
} finally {
await cleanupAgent(page, agentId);
}
});
test('submits a mixed batch once and preserves decisions through collapse', async ({ page }) => {
test.setTimeout(120000);
const label = uniqueLabel();
const firstCallId = `call_e2e_approval_${label}_first`;
const secondCallId = `call_e2e_approval_${label}_second`;
const responseText = `manual batch result ${label}`;
let agentId: string | undefined;
try {
agentId = await createAndSelectApprovalAgent(page);
await startApproval(page, label, BATCH_APPROVAL_PROMPT_MARKER);
await expect(approvalCards(page)).toHaveCount(2);
const firstCard = approvalCard(page, firstCallId);
const secondCard = approvalCard(page, secondCallId);
const submit = messagesView(page).getByRole('button', {
name: 'Submit 2 decisions',
exact: true,
});
await secondCard.getByRole('button', { name: 'Respond' }).click();
await secondCard.getByRole('textbox', { name: 'Respond' }).fill(responseText);
await expect(submit).toBeDisabled();
await firstCard.getByRole('button', { name: 'Approve' }).click();
await expect(submit).toBeEnabled();
const groupToggle = messagesView(page).getByRole('button', {
name: 'Used 2 tools',
exact: true,
});
await groupToggle.click();
await expect(groupToggle).toHaveAttribute('aria-expanded', 'false');
await page.waitForTimeout(400);
await groupToggle.click();
await expect(groupToggle).toHaveAttribute('aria-expanded', 'true');
const reopenedFirstCard = approvalCard(page, firstCallId);
const reopenedSecondCard = approvalCard(page, secondCallId);
await expect(reopenedFirstCard.getByRole('button', { name: 'Approve' })).toHaveAttribute(
'aria-pressed',
'true',
);
await expect(reopenedSecondCard.getByRole('button', { name: 'Respond' })).toHaveAttribute(
'aria-pressed',
'true',
);
await expect(reopenedSecondCard.getByRole('textbox', { name: 'Respond' })).toHaveValue(
responseText,
);
await expect(submit).toBeEnabled();
const { body } = await submitAndCapture(page, submit);
expect(body.decisions).toHaveLength(2);
expect(body.decisions).toEqual(
expect.arrayContaining([
expect.objectContaining({
decision: 'approve',
tool_call_id: firstCallId,
}),
expect.objectContaining({
decision: 'respond',
responseText,
tool_call_id: secondCallId,
}),
]),
);
await expect(
messagesView(page).getByText(`E2E approval probe executed: first-${label}`),
).toBeVisible({ timeout: 30000 });
await expect(messagesView(page).getByText(responseText)).toBeVisible({ timeout: 30000 });
await expect(
messagesView(page).getByText(`E2E approval probe executed: second-${label}`),
).toHaveCount(0);
await expect(approvalCards(page)).toHaveCount(0);
} finally {
await cleanupAgent(page, agentId);
}
});
test('rehydrates a paused approval and its completed result across reloads', async ({ page }) => {
test.setTimeout(120000);
const label = uniqueLabel();
const toolCallId = `call_e2e_approval_${label}`;
const executedText = `E2E approval probe executed: original-${label}`;
let agentId: string | undefined;
try {
agentId = await createAndSelectApprovalAgent(page);
await startApproval(page, label);
const conversationPath = new URL(page.url()).pathname;
await page.reload({ waitUntil: 'domcontentloaded' });
await expect.poll(() => new URL(page.url()).pathname).toBe(conversationPath);
const rehydratedCard = approvalCard(page, toolCallId);
await expect(rehydratedCard).toBeVisible({ timeout: 30000 });
await expect(rehydratedCard).toContainText(APPROVAL_REASON);
await page.goto(NEW_CHAT_PATH, { waitUntil: 'domcontentloaded' });
await expect(approvalCards(page)).toHaveCount(0);
await page.goto(conversationPath, { waitUntil: 'domcontentloaded' });
const navigatedCard = approvalCard(page, toolCallId);
await expect(navigatedCard).toBeVisible({ timeout: 30000 });
await expect(navigatedCard).toContainText(APPROVAL_REASON);
await navigatedCard.getByRole('button', { name: 'Approve' }).click();
await submitAndCapture(page, navigatedCard.getByRole('button', { name: 'Submit' }));
await expect(messagesView(page).getByText(executedText)).toBeVisible({ timeout: 30000 });
await expect(approvalCards(page)).toHaveCount(0);
await page.reload({ waitUntil: 'domcontentloaded' });
await expect.poll(() => new URL(page.url()).pathname).toBe(conversationPath);
await expect(messagesView(page).getByText(executedText)).toBeVisible({ timeout: 30000 });
await expect(approvalCards(page)).toHaveCount(0);
} finally {
await cleanupAgent(page, agentId);
}
});
test('sends only one resume request for two synchronous submit clicks', async ({ page }) => {
test.setTimeout(120000);
const label = uniqueLabel();
const executedText = `E2E approval probe executed: original-${label}`;
let agentId: string | undefined;
let releaseResume = () => undefined;
let resumeHandler: ((route: Route) => Promise<void>) | undefined;
try {
agentId = await createAndSelectApprovalAgent(page);
const card = await startApproval(page, label);
const submit = card.getByRole('button', { name: 'Submit' });
await card.getByRole('button', { name: 'Approve' }).click();
await expect(submit).toBeEnabled();
let resumeRequests = 0;
const resumeGate = new Promise<void>((resolve) => {
releaseResume = resolve;
});
resumeHandler = async (route) => {
resumeRequests++;
if (resumeRequests === 1) {
await resumeGate;
await route.continue();
return;
}
await route.fulfill({
status: 409,
contentType: 'application/json',
body: JSON.stringify({ message: 'duplicate resume request' }),
});
};
await page.route('**/api/agents/chat/resume', resumeHandler);
await submit.evaluate((button: HTMLButtonElement) => {
button.click();
button.click();
});
await page.waitForTimeout(250);
expect(resumeRequests).toBe(1);
await expect(card.getByRole('button', { name: 'Submitting' })).toBeDisabled();
await expect(card.getByRole('button', { name: 'Approve' })).toBeDisabled();
releaseResume();
await expect(messagesView(page).getByText(executedText)).toBeVisible({ timeout: 30000 });
await expect(approvalCards(page)).toHaveCount(0);
} finally {
releaseResume();
if (resumeHandler) {
await page.unroute('**/api/agents/chat/resume', resumeHandler);
}
await cleanupAgent(page, agentId);
}
});
test('preserves a decision after a transient resume error and retries successfully', async ({
page,
}) => {
test.setTimeout(120000);
const label = uniqueLabel();
const responseText = `retry response ${label}`;
let agentId: string | undefined;
let resumeHandler: ((route: Route) => Promise<void>) | undefined;
try {
agentId = await createAndSelectApprovalAgent(page);
const card = await startApproval(page, label);
const submit = card.getByRole('button', { name: 'Submit' });
await card.getByRole('button', { name: 'Respond' }).click();
const responseInput = card.getByRole('textbox', { name: 'Respond' });
await responseInput.fill(responseText);
let resumeRequests = 0;
resumeHandler = async (route) => {
resumeRequests++;
if (resumeRequests === 1) {
await route.fulfill({
status: 500,
contentType: 'application/json',
body: JSON.stringify({ message: 'temporary e2e failure' }),
});
return;
}
await route.continue();
};
await page.route('**/api/agents/chat/resume', resumeHandler);
await Promise.all([
page.waitForResponse(
(response) => isResumeRequest(response.request()) && response.status() === 500,
),
submit.click(),
]);
await expect(card.getByText(APPROVAL_ERROR, { exact: true })).toBeVisible();
await expect(card.getByRole('button', { name: 'Respond' })).toHaveAttribute(
'aria-pressed',
'true',
);
await expect(responseInput).toHaveValue(responseText);
await expect(submit).toBeEnabled();
await Promise.all([
page.waitForResponse(
(response) => isResumeRequest(response.request()) && response.status() === 200,
),
submit.click(),
]);
await expect(messagesView(page).getByText(responseText)).toBeVisible({ timeout: 30000 });
await expect(
messagesView(page).getByText(`E2E approval probe executed: original-${label}`),
).toHaveCount(0);
expect(resumeRequests).toBe(2);
await expect(approvalCards(page)).toHaveCount(0);
} finally {
if (resumeHandler) {
await page.unroute('**/api/agents/chat/resume', resumeHandler);
}
await cleanupAgent(page, agentId);
}
});
test('locks the approval controls and explains an expired resume action', async ({ page }) => {
test.setTimeout(120000);
const label = uniqueLabel();
const executedText = `E2E approval probe executed: original-${label}`;
let agentId: string | undefined;
let capturedResumeBody: Record<string, unknown> | undefined;
let backendResolved = false;
let routeInstalled = false;
const resumeHandler = async (route: Route) => {
capturedResumeBody = route.request().postDataJSON() as Record<string, unknown>;
await route.fulfill({
status: 409,
contentType: 'application/json',
body: JSON.stringify({ message: 'expired e2e action' }),
});
};
try {
agentId = await createAndSelectApprovalAgent(page);
const card = await startApproval(page, label);
const approve = card.getByRole('button', { name: 'Approve' });
const submit = card.getByRole('button', { name: 'Submit' });
await approve.click();
await page.route('**/api/agents/chat/resume', resumeHandler);
routeInstalled = true;
await Promise.all([
page.waitForResponse(
(response) => isResumeRequest(response.request()) && response.status() === 409,
),
submit.click(),
]);
await expect(card.getByText(APPROVAL_EXPIRED, { exact: true })).toBeVisible();
await expect(approve).toHaveAttribute('aria-pressed', 'true');
await expect(approve).toBeDisabled();
await expect(card.getByRole('button', { name: 'Reject' })).toBeDisabled();
await expect(card.getByRole('button', { name: 'Edit' })).toBeDisabled();
await expect(card.getByRole('button', { name: 'Respond' })).toBeDisabled();
await expect(submit).toBeDisabled();
expect(capturedResumeBody).toBeDefined();
await page.unroute('**/api/agents/chat/resume', resumeHandler);
routeInstalled = false;
const token = await getAccessToken(page);
await requestJson(page, {
path: '/api/agents/chat/resume',
token,
method: 'POST',
body: capturedResumeBody,
});
backendResolved = true;
await expect(messagesView(page).getByText(executedText)).toBeVisible({ timeout: 30000 });
await expect(approvalCards(page)).toHaveCount(0);
} finally {
if (routeInstalled) {
await page.unroute('**/api/agents/chat/resume', resumeHandler);
}
if (!backendResolved && capturedResumeBody) {
const token = await getAccessToken(page);
await requestJson(page, {
path: '/api/agents/chat/resume',
token,
method: 'POST',
body: capturedResumeBody,
}).catch(() => undefined);
}
await cleanupAgent(page, agentId);
}
});
});