From 90ac3d25a52c5aa6e3f31448828f3fd483f19e97 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 24 Jul 2026 20:50:10 -0400 Subject: [PATCH] test: cover tool approval workflows end to end --- .../Chat/Messages/Content/ApprovalContext.tsx | 16 +- .../Chat/Messages/Content/ToolApproval.tsx | 6 +- .../Chat/Messages/Content/ToolCallGroup.tsx | 18 +- .../__tests__/ApprovalContext.test.tsx | 76 ++ .../Content/__tests__/ToolCallGroup.test.tsx | 42 ++ client/src/utils/approval.spec.ts | 31 + client/src/utils/approval.ts | 4 + e2e/config/librechat.e2e.yaml | 12 + e2e/setup/fake-mcp-server.js | 20 + e2e/setup/fake-model.js | 122 ++- e2e/setup/tool-approval-hook.js | 29 + e2e/specs/mock/tool-approvals.spec.ts | 693 ++++++++++++++++++ 12 files changed, 1064 insertions(+), 5 deletions(-) create mode 100644 client/src/components/Chat/Messages/Content/__tests__/ApprovalContext.test.tsx create mode 100644 e2e/setup/tool-approval-hook.js create mode 100644 e2e/specs/mock/tool-approvals.spec.ts diff --git a/client/src/components/Chat/Messages/Content/ApprovalContext.tsx b/client/src/components/Chat/Messages/Content/ApprovalContext.tsx index 7b5bcf9477..8fa66614da 100644 --- a/client/src/components/Chat/Messages/Content/ApprovalContext.tsx +++ b/client/src/components/Chat/Messages/Content/ApprovalContext.tsx @@ -281,6 +281,9 @@ export function useResumeSubmit() { const approvalMutation = useSubmitToolApprovalMutation(); const askMutation = useSubmitAskAnswerMutation(); const { getDecisions, isReady, setStatus } = useApprovalContext(); + /** React state cannot lock a second click in the same browser task. Keep a + * synchronous action-id guard alongside the rendered submission status. */ + const submittingToolActionIdsRef = useRef(new Set()); /** Ask status lives in Recoil so it works from the composer (outside the * provider); tool-approval status stays on the context. */ const { setAskStatus } = useAskSubmitStatus(); @@ -311,12 +314,23 @@ export function useResumeSubmit() { if (!fields || decisions.length === 0 || !isReady(actionId)) { return; } + if (submittingToolActionIdsRef.current.has(actionId)) { + return; + } + submittingToolActionIdsRef.current.add(actionId); setStatus(actionId, 'submitting'); approvalMutation.mutate( { ...fields, actionId, decisions }, { onSuccess: () => setStatus(actionId, 'submitted'), - onError: (error) => setStatus(actionId, isExpiredError(error) ? 'expired' : 'error'), + onError: (error) => { + const expired = isExpiredError(error); + if (!expired) { + // Network/validation failures are retryable; a 409 is terminal. + submittingToolActionIdsRef.current.delete(actionId); + } + setStatus(actionId, expired ? 'expired' : 'error'); + }, }, ); }, diff --git a/client/src/components/Chat/Messages/Content/ToolApproval.tsx b/client/src/components/Chat/Messages/Content/ToolApproval.tsx index 1068eadd1e..31ee82f654 100644 --- a/client/src/components/Chat/Messages/Content/ToolApproval.tsx +++ b/client/src/components/Chat/Messages/Content/ToolApproval.tsx @@ -174,7 +174,11 @@ export default function ToolApproval({ } return ( -
+
{description != null && description.length > 0 && (

{description}

)} diff --git a/client/src/components/Chat/Messages/Content/ToolCallGroup.tsx b/client/src/components/Chat/Messages/Content/ToolCallGroup.tsx index 955f9b2b47..30bd69a6ab 100644 --- a/client/src/components/Chat/Messages/Content/ToolCallGroup.tsx +++ b/client/src/components/Chat/Messages/Content/ToolCallGroup.tsx @@ -110,6 +110,17 @@ export default function ToolCallGroup({ const count = parts.length; const toolMetadata = useMemo(() => parts.map((p) => getToolMeta(p.part)), [parts]); + const hasPendingApproval = useMemo( + () => + parts.some(({ part }) => { + if (part.type !== ContentTypes.TOOL_CALL) { + return false; + } + const toolCall = part[ContentTypes.TOOL_CALL] as Agents.ToolCall | undefined; + return toolCall?.approval != null && (toolCall.output?.length ?? 0) === 0; + }), + [parts], + ); const allCompleted = useMemo( () => toolMetadata.every((m) => m?.hasOutput === true), [toolMetadata], @@ -223,13 +234,16 @@ export default function ToolCallGroup({ if (event.target !== event.currentTarget) { return; } - if (isExpanded) { + if (isExpanded || hasPendingApproval) { return; } + // Approval controls own unsent local form state. Keep unresolved cards + // mounted (the collapsed panel is inert/hidden) so collapsing a batch + // cannot erase decisions the reviewer already made. setShouldRenderBody(false); notifyLayoutChange(); }, - [isExpanded, notifyLayoutChange], + [hasPendingApproval, isExpanded, notifyLayoutChange], ); /** Category-aware header verb: subagents and questions read as their own diff --git a/client/src/components/Chat/Messages/Content/__tests__/ApprovalContext.test.tsx b/client/src/components/Chat/Messages/Content/__tests__/ApprovalContext.test.tsx new file mode 100644 index 0000000000..55c09c9164 --- /dev/null +++ b/client/src/components/Chat/Messages/Content/__tests__/ApprovalContext.test.tsx @@ -0,0 +1,76 @@ +import React from 'react'; +import { RecoilRoot } from 'recoil'; +import { act, renderHook } from '@testing-library/react'; +import { ChatContext } from '~/Providers/ChatContext'; +import ApprovalProvider, { useApprovalContext, useResumeSubmit } from '../ApprovalContext'; + +const mockApprovalMutate = jest.fn(); +const mockAskMutate = jest.fn(); + +jest.mock('~/data-provider', () => ({ + useSubmitToolApprovalMutation: () => ({ mutate: mockApprovalMutate }), + useSubmitAskAnswerMutation: () => ({ mutate: mockAskMutate }), +})); + +jest.mock('~/store/agents', () => ({ + useGetEphemeralAgent: () => () => undefined, +})); + +const chatContextValue = { + conversation: { + conversationId: 'conversation-1', + endpoint: 'agents', + agent_id: 'agent-1', + }, +} as unknown as React.ContextType; + +function wrapper({ children }: { children: React.ReactNode }) { + return ( + + + {children} + + + ); +} + +describe('useResumeSubmit', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + test('synchronously deduplicates tool approval submissions and unlocks a retryable error', () => { + const { result } = renderHook( + () => ({ + approval: useApprovalContext(), + resume: useResumeSubmit(), + }), + { wrapper }, + ); + + act(() => { + result.current.approval.registerToolCall('action-1', 'call-1'); + result.current.approval.setDecision('action-1', 'call-1', { + tool_call_id: 'call-1', + decision: 'approve', + }); + }); + + act(() => { + result.current.resume.submitToolApproval('action-1'); + result.current.resume.submitToolApproval('action-1'); + }); + expect(mockApprovalMutate).toHaveBeenCalledTimes(1); + + const firstOptions = mockApprovalMutate.mock.calls[0][1] as { + onError: (error: unknown) => void; + }; + act(() => firstOptions.onError(new Error('temporary failure'))); + + act(() => { + result.current.resume.submitToolApproval('action-1'); + result.current.resume.submitToolApproval('action-1'); + }); + expect(mockApprovalMutate).toHaveBeenCalledTimes(2); + }); +}); diff --git a/client/src/components/Chat/Messages/Content/__tests__/ToolCallGroup.test.tsx b/client/src/components/Chat/Messages/Content/__tests__/ToolCallGroup.test.tsx index 3abff264a0..14c66b0f73 100644 --- a/client/src/components/Chat/Messages/Content/__tests__/ToolCallGroup.test.tsx +++ b/client/src/components/Chat/Messages/Content/__tests__/ToolCallGroup.test.tsx @@ -83,6 +83,21 @@ const makePart = ( }, }) as unknown as TMessageContentParts; +const makePendingApprovalPart = (id: string): TMessageContentParts => + ({ + type: ContentTypes.TOOL_CALL, + [ContentTypes.TOOL_CALL]: { + id, + name: 'approval_probe', + args: {}, + output: '', + approval: { + actionId: 'action-1', + allowed_decisions: ['approve', 'reject'], + }, + }, + }) as unknown as TMessageContentParts; + const imageAttachment: TAttachment = { filename: 'foo.png', filepath: '/files/foo.png', @@ -211,6 +226,33 @@ describe('ToolCallGroup image hoisting', () => { expect(screen.queryByTestId('inner-0')).not.toBeInTheDocument(); }); + it('keeps unresolved approval bodies mounted while the group is collapsed', () => { + const approvalParts = [ + { part: makePendingApprovalPart('t1'), idx: 0 }, + { part: makePendingApprovalPart('t2'), idx: 1 }, + ]; + renderGroup({ + ...baseProps, + parts: approvalParts, + renderPart: (_p: TMessageContentParts, idx: number) => ( +
+ {'approval'} +
+ ), + }); + + const button = screen.getByRole('button', { name: 'Used 2 tools' }); + const collapsible = button.nextElementSibling as HTMLElement; + expect(screen.getByTestId('approval-0')).toBeInTheDocument(); + + fireEvent.click(button); + fireEvent.transitionEnd(collapsible); + + expect(button).toHaveAttribute('aria-expanded', 'false'); + expect(screen.getByTestId('approval-0')).toBeInTheDocument(); + expect(screen.getByTestId('approval-1')).toBeInTheDocument(); + }); + it('reconciles layout after the group collapses from an expanded state', async () => { renderGroup(baseProps); diff --git a/client/src/utils/approval.spec.ts b/client/src/utils/approval.spec.ts index 0f2de67d69..2a7d5cf9cc 100644 --- a/client/src/utils/approval.spec.ts +++ b/client/src/utils/approval.spec.ts @@ -71,6 +71,37 @@ describe('applyPendingAction — tool_approval', () => { }); }); + it('replaces displayed tool args with the matching action request arguments', () => { + const originalArgs = { query: 'original model args' }; + const rewrittenArgs = { query: 'rewritten by policy hook' }; + const message = msg({ content: [toolCallPart('tc1', { args: originalArgs })] }); + const action = toolApprovalAction({ + payload: { + type: 'tool_approval', + action_requests: [ + { + name: 'search', + arguments: rewrittenArgs, + tool_call_id: 'tc1', + description: 'Review rewritten search', + }, + ], + review_configs: [ + { + action_name: 'search', + tool_call_id: 'tc1', + allowed_decisions: ['approve', 'reject', 'edit', 'respond'], + }, + ], + }, + }); + + const result = applyPendingAction(message, action); + + expect(getToolCall(result.content?.[0] as TMessageContentParts)?.args).toEqual(rewrittenArgs); + expect(getToolCall(message.content?.[0] as TMessageContentParts)?.args).toEqual(originalArgs); + }); + it('leaves a completed tool call (with output) untouched and returns the same message reference', () => { const message = msg({ content: [toolCallPart('tc1', { output: 'already ran' })] }); const result = applyPendingAction(message, toolApprovalAction()); diff --git a/client/src/utils/approval.ts b/client/src/utils/approval.ts index eb99abda06..8a56ec44bc 100644 --- a/client/src/utils/approval.ts +++ b/client/src/utils/approval.ts @@ -95,6 +95,10 @@ function tagApprovalOnPart( const reviewConfig = reviewByToolCallId.get(toolCallId); nextToolCall = { ...nextToolCall, + // A PreToolUse hook may replace the model's original args before asking. + // The interrupt payload is authoritative so the reviewer sees, edits, and + // approves the same arguments the resumed tool will actually execute. + args: request.arguments, approval: { actionId, allowed_decisions: reviewConfig?.allowed_decisions ?? [], diff --git a/e2e/config/librechat.e2e.yaml b/e2e/config/librechat.e2e.yaml index 3a8e287f94..07062dce7d 100644 --- a/e2e/config/librechat.e2e.yaml +++ b/e2e/config/librechat.e2e.yaml @@ -60,6 +60,18 @@ endpoints: - chain - ocr - run_in_background + # Keep the shared mock profile non-interactive except for the dedicated + # approval probe. This exercises real HITL pause/resume without wedging the + # existing file-authoring, steering, background-tool, or MCP specs. + toolApproval: + enabled: true + mode: bypass + ask: + - approval_probe_mcp_e2e-memory + reason: E2E approval required before running {tool}. + hooks: + - module: e2e/setup/tool-approval-hook.js + matcher: ^approval_probe_mcp_e2e-memory$ custom: - name: 'Mock Provider A' apiKey: 'e2e-mock-key-a' diff --git a/e2e/setup/fake-mcp-server.js b/e2e/setup/fake-mcp-server.js index 182d46bd80..05ab1f18e5 100644 --- a/e2e/setup/fake-mcp-server.js +++ b/e2e/setup/fake-mcp-server.js @@ -66,6 +66,26 @@ server.registerTool( }, ); +server.registerTool( + 'approval_probe', + { + description: + 'Echoes reviewed input so LibreChat mock end-to-end tests can verify tool approval decisions.', + inputSchema: { + value: z.string(), + review: z.string().optional(), + }, + }, + async ({ value }) => ({ + content: [ + { + type: 'text', + text: `E2E approval probe executed: ${value}`, + }, + ], + }), +); + async function main() { await server.connect(new StdioServerTransport()); } diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.js index d432e687d0..f63ac9c8ad 100644 --- a/e2e/setup/fake-model.js +++ b/e2e/setup/fake-model.js @@ -34,6 +34,10 @@ const FORCED_ERROR_MARKER = 'E2E_FORCED_ERROR:'; const MARKDOWN_REPLY_MARKER = 'E2E_MARKDOWN_REPLY'; const BACKGROUND_DISPATCH_MARKER = 'E2E_BACKGROUND_DISPATCH:'; const BACKGROUND_COLLECT_MARKER = 'E2E_BACKGROUND_COLLECT:'; +const TOOL_APPROVAL_MARKER = 'E2E_TOOL_APPROVAL:'; +const TOOL_APPROVAL_BATCH_MARKER = 'E2E_TOOL_APPROVAL_BATCH:'; +const TOOL_APPROVAL_RESTRICTED_MARKER = 'E2E_TOOL_APPROVAL_RESTRICTED:'; +const TOOL_APPROVAL_REWRITE_MARKER = 'E2E_TOOL_APPROVAL_REWRITE:'; const CREATE_FILE_AUTHORING_FINAL_TEXT = 'E2E file authoring complete'; const EDIT_FILE_AUTHORING_FINAL_TEXT = 'E2E file edit complete'; const SKILL_ASSERTION_FINAL_TEXT = 'E2E skill assertion passed'; @@ -56,6 +60,8 @@ const CREATE_SKILL_TOOL_CALL_ID = 'call_e2e_create_skill'; const EDIT_SKILL_TOOL_CALL_ID = 'call_e2e_edit_skill'; const BACKGROUND_TOOL_NAME = 'slow_echo_mcp_e2e-memory'; const CHECK_BACKGROUND_TASK_TOOL_NAME = 'check_background_task'; +const APPROVAL_TOOL_NAME = 'approval_probe_mcp_e2e-memory'; +const APPROVAL_TOOL_CALL_PREFIX = 'call_e2e_approval_'; const BACKGROUND_DISPATCH_TOOL_CALL_ID = 'call_e2e_background_dispatch'; const BACKGROUND_COLLECT_TOOL_CALL_ID = 'call_e2e_background_collect'; const MODEL_SPEC_ACCESSIBLE_SKILL = 'e2e-model-spec-allowed'; @@ -832,6 +838,92 @@ function findLastToolMessageText(messages, requiredToken) { return ''; } +function approvalToolResponses(label, toolNames, review) { + if (!toolNames.has(APPROVAL_TOOL_NAME)) { + return { + responses: [`E2E approval unavailable: ${APPROVAL_TOOL_NAME} was not advertised.`], + }; + } + return { + responses: ['', ''], + toolCalls: [ + { + id: `${APPROVAL_TOOL_CALL_PREFIX}${label}`, + name: APPROVAL_TOOL_NAME, + args: { + value: `original-${label}`, + ...(review ? { review } : {}), + }, + type: 'tool_call', + }, + ], + }; +} + +function batchApprovalToolResponses(label, toolNames) { + if (!toolNames.has(APPROVAL_TOOL_NAME)) { + return { + responses: [`E2E approval unavailable: ${APPROVAL_TOOL_NAME} was not advertised.`], + }; + } + return { + responses: ['', ''], + toolCalls: [ + { + id: `${APPROVAL_TOOL_CALL_PREFIX}${label}_first`, + name: APPROVAL_TOOL_NAME, + args: { value: `first-${label}` }, + type: 'tool_call', + }, + { + id: `${APPROVAL_TOOL_CALL_PREFIX}${label}_second`, + name: APPROVAL_TOOL_NAME, + args: { value: `second-${label}` }, + type: 'tool_call', + }, + ], + }; +} + +/** + * Resume rebuilds the fake model without the original prompt in `context.messages`. + * Detect the checkpoint-restored approval tool messages on every model instance + * so the continuation can report the real approve/reject/edit/respond outcome. + */ +function approvalOutcomeResponses(messages) { + let latestHumanIndex = -1; + for (let index = 0; index < (messages ?? []).length; index++) { + const type = messageType(messages[index]); + if (type === 'human' || type === 'user') { + latestHumanIndex = index; + } + } + + const outcomeMessages = (messages ?? []) + .slice(latestHumanIndex + 1) + .filter( + (message) => + messageType(message) === 'tool' && + typeof message?.tool_call_id === 'string' && + message.tool_call_id.startsWith(APPROVAL_TOOL_CALL_PREFIX), + ); + + const isBatch = outcomeMessages.some( + (message) => + message.tool_call_id.endsWith('_first') || message.tool_call_id.endsWith('_second'), + ); + if (isBatch && outcomeMessages.length < 2) { + return null; + } + + const outcomes = outcomeMessages.map((message) => getContentText(message.content)); + + if (outcomes.length === 0) { + return null; + } + return { responses: [`E2E approval outcomes: ${outcomes.join(' | ')}`] }; +} + /** * Turn 1 of the background e2e: emit the MCP tool call with the injected * `run_in_background: true` arg, then (second model invocation, after the @@ -925,6 +1017,26 @@ function backgroundCollectResponses(messages, toolNames) { } function resolveResponses({ graph, messages, text, toolNames }) { + const batchApprovalLabel = getMarkerValue(text, TOOL_APPROVAL_BATCH_MARKER); + if (batchApprovalLabel) { + return batchApprovalToolResponses(batchApprovalLabel, toolNames); + } + + const restrictedApprovalLabel = getMarkerValue(text, TOOL_APPROVAL_RESTRICTED_MARKER); + if (restrictedApprovalLabel) { + return approvalToolResponses(restrictedApprovalLabel, toolNames, 'restricted'); + } + + const rewrittenApprovalLabel = getMarkerValue(text, TOOL_APPROVAL_REWRITE_MARKER); + if (rewrittenApprovalLabel) { + return approvalToolResponses(rewrittenApprovalLabel, toolNames, 'rewrite'); + } + + const approvalLabel = getMarkerValue(text, TOOL_APPROVAL_MARKER); + if (approvalLabel) { + return approvalToolResponses(approvalLabel, toolNames); + } + const reply = replyResponses(text); if (reply) { return reply; @@ -1055,5 +1167,13 @@ module.exports = function fakeModelHook(run, context) { text, toolNames, }); - overrideModel({ graph, responses, sleep, toolCalls, thrownError, resolveOnStream }); + overrideModel({ + graph, + responses, + sleep, + toolCalls, + thrownError, + resolveOnStream: (streamMessages) => + approvalOutcomeResponses(streamMessages) ?? resolveOnStream?.(streamMessages) ?? null, + }); }; diff --git a/e2e/setup/tool-approval-hook.js b/e2e/setup/tool-approval-hook.js new file mode 100644 index 0000000000..4eebaa751e --- /dev/null +++ b/e2e/setup/tool-approval-hook.js @@ -0,0 +1,29 @@ +/** + * Dynamic approval-policy fixture for the mock Playwright suite. + * + * The `review` argument selects behavior that cannot be expressed by the static + * ask list: a restricted decision set, or an authoritative argument rewrite. + */ +module.exports = () => () => async (input) => { + if (input.toolInput.review === 'restricted') { + return { + decision: 'ask', + reason: 'E2E approval offers approve or reject only.', + allowedDecisions: ['approve', 'reject'], + }; + } + + if (input.toolInput.review === 'rewrite') { + const originalValue = + typeof input.toolInput.value === 'string' ? input.toolInput.value : 'original-missing'; + return { + decision: 'ask', + reason: 'E2E approval reviews rewritten arguments.', + updatedInput: { + value: originalValue.replace(/^original-/, 'rewritten-'), + }, + }; + } + + return {}; +}; diff --git a/e2e/specs/mock/tool-approvals.spec.ts b/e2e/specs/mock/tool-approvals.spec.ts new file mode 100644 index 0000000000..9e35991938 --- /dev/null +++ b/e2e/specs/mock/tool-approvals.spec.ts @@ -0,0 +1,693 @@ +import { expect, test } from '@playwright/test'; +import type { Locator, Page, Request, Route } from '@playwright/test'; +import type { AgentDetail } from './agents.helpers'; +import { cleanupAgent, openAgentBuilder, uniqueAgentName } from './agents.helpers'; +import { + MOCK_ENDPOINTS, + NEW_CHAT_PATH, + fetchJson, + getAccessToken, + messagesView, + requestJson, + sendMessage, +} from './helpers'; + +const MCP_SERVER_NAME = 'e2e-memory'; +const MCP_SERVER_TOOL_ID = `sys__server__sys_mcp_${MCP_SERVER_NAME}`; +const APPROVAL_TOOL_NAME = 'approval_probe'; +const APPROVAL_TOOL_ID = `${APPROVAL_TOOL_NAME}_mcp_${MCP_SERVER_NAME}`; +const APPROVAL_PROMPT_MARKER = 'E2E_TOOL_APPROVAL:'; +const BATCH_APPROVAL_PROMPT_MARKER = 'E2E_TOOL_APPROVAL_BATCH:'; +const RESTRICTED_APPROVAL_PROMPT_MARKER = 'E2E_TOOL_APPROVAL_RESTRICTED:'; +const REWRITTEN_APPROVAL_PROMPT_MARKER = 'E2E_TOOL_APPROVAL_REWRITE:'; +const APPROVAL_REASON = `E2E approval required before running ${APPROVAL_TOOL_ID}.`; +const APPROVAL_ERROR = 'Something went wrong submitting your decision. Please try again.'; +const APPROVAL_EXPIRED = 'This request expired or was already handled.'; +const DESCRIPTION = 'Verifies human approval behavior for MCP tool calls in mock E2E tests.'; +const uniqueLabel = () => `${Date.now()}-${Math.floor(Math.random() * 1e4)}`; + +type MCPToolsResponse = { + servers?: Record }>; +}; + +type ApprovalResumeBody = { + actionId?: string; + agent_id?: string; + conversationId?: string; + endpoint?: string; + decisions?: Array<{ + tool_call_id?: string; + decision?: string; + reason?: string; + responseText?: string; + editedArguments?: Record; + }>; +}; + +type ApprovalResumeResponse = { + conversationId?: string; + status?: string; + streamId?: string; +}; + +const approvalCards = (page: Page) => messagesView(page).getByTestId('tool-approval'); +const approvalCard = (page: Page, toolCallId: string) => + messagesView(page).locator(`[data-testid="tool-approval"][data-tool-call-id="${toolCallId}"]`); + +function isResumeRequest(request: Request) { + return ( + request.method() === 'POST' && new URL(request.url()).pathname === '/api/agents/chat/resume' + ); +} + +async function waitForApprovalTool(page: Page) { + const token = await getAccessToken(page); + let latestTools: MCPToolsResponse | null = null; + + for (let attempt = 0; attempt < 20; attempt++) { + latestTools = await fetchJson(page, '/api/mcp/tools', token); + const tools = latestTools.servers?.[MCP_SERVER_NAME]?.tools ?? []; + if (tools.some((tool) => tool.pluginKey === APPROVAL_TOOL_ID)) { + return; + } + await page.waitForTimeout(500); + } + + expect( + latestTools?.servers?.[MCP_SERVER_NAME]?.tools, + `Expected ${MCP_SERVER_NAME} to expose ${APPROVAL_TOOL_ID}`, + ).toEqual(expect.arrayContaining([expect.objectContaining({ pluginKey: APPROVAL_TOOL_ID })])); +} + +async function createAndSelectApprovalAgent(page: Page): Promise { + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await waitForApprovalTool(page); + + const token = await getAccessToken(page); + const agentName = uniqueAgentName('E2E Tool Approval Agent'); + const agent = await requestJson(page, { + path: '/api/agents', + token, + method: 'POST', + body: { + name: agentName, + description: DESCRIPTION, + instructions: 'Use the requested approval probe tools and report their results.', + provider: MOCK_ENDPOINTS[0].label, + model: MOCK_ENDPOINTS[0].model, + tools: [MCP_SERVER_TOOL_ID, APPROVAL_TOOL_ID], + }, + }); + + const form = await openAgentBuilder(page); + await form.getByRole('combobox', { name: 'Agent', exact: true }).click(); + await page.getByRole('option', { name: agentName }).click(); + await expect(form.getByLabel('Agent name')).toHaveValue(agentName); + await form.getByRole('button', { name: 'Select Agent' }).click(); + return agent.id; +} + +async function startApproval( + page: Page, + label: string, + marker = APPROVAL_PROMPT_MARKER, + expectedReason = APPROVAL_REASON, +): Promise { + const response = await sendMessage(page, `${marker}${label}`); + expect(response.ok()).toBeTruthy(); + await expect(page).toHaveURL(/\/c\/(?!new)/, { timeout: 15000 }); + const card = approvalCards(page).first(); + await expect(card).toBeVisible({ timeout: 30000 }); + await expect(card).toContainText(expectedReason); + return card; +} + +async function submitAndCapture(page: Page, submit: Locator) { + const [request, response] = await Promise.all([ + page.waitForRequest(isResumeRequest), + page.waitForResponse( + (candidate) => isResumeRequest(candidate.request()) && candidate.status() === 200, + ), + submit.click(), + ]); + return { + body: request.postDataJSON() as ApprovalResumeBody, + response, + }; +} + +test.describe('tool approvals', () => { + test('approves a paused tool with its original arguments', async ({ page }) => { + test.setTimeout(120000); + const label = uniqueLabel(); + let agentId: string | undefined; + + try { + agentId = await createAndSelectApprovalAgent(page); + const card = await startApproval(page, label); + + await expect(card.getByRole('button', { name: 'Approve' })).toBeVisible(); + await expect(card.getByRole('button', { name: 'Reject' })).toBeVisible(); + await expect(card.getByRole('button', { name: 'Edit' })).toBeVisible(); + await expect(card.getByRole('button', { name: 'Respond' })).toBeVisible(); + + const submit = card.getByRole('button', { name: 'Submit' }); + await expect(submit).toBeDisabled(); + await card.getByRole('button', { name: 'Approve' }).click(); + await expect(submit).toBeEnabled(); + + const conversationId = new URL(page.url()).pathname.replace('/c/', ''); + const { body, response } = await submitAndCapture(page, submit); + expect(body.actionId).toBeTruthy(); + expect(body.agent_id).toBe(agentId); + expect(body.conversationId).toBe(conversationId); + expect(body.endpoint).toBe('agents'); + expect(body.decisions).toEqual([ + expect.objectContaining({ + decision: 'approve', + tool_call_id: `call_e2e_approval_${label}`, + }), + ]); + await expect(response.json() as Promise).resolves.toEqual( + expect.objectContaining({ + conversationId, + status: 'resuming', + streamId: conversationId, + }), + ); + + await expect( + messagesView(page).getByText(`E2E approval probe executed: original-${label}`), + ).toBeVisible({ timeout: 30000 }); + await expect(approvalCards(page)).toHaveCount(0); + } finally { + await cleanupAgent(page, agentId); + } + }); + + test('rejects with an optional reason without executing the tool', async ({ page }) => { + test.setTimeout(120000); + const label = uniqueLabel(); + const reason = `do not run ${label}`; + let agentId: string | undefined; + + try { + agentId = await createAndSelectApprovalAgent(page); + const card = await startApproval(page, label); + const submit = card.getByRole('button', { name: 'Submit' }); + + await card.getByRole('button', { name: 'Reject' }).click(); + await card.getByRole('textbox', { name: 'Reject' }).fill(` ${reason} `); + await expect(submit).toBeEnabled(); + + const { body } = await submitAndCapture(page, submit); + expect(body.decisions).toEqual([ + expect.objectContaining({ + decision: 'reject', + reason, + tool_call_id: `call_e2e_approval_${label}`, + }), + ]); + + await expect(messagesView(page).getByText(`Blocked: ${reason}`)).toBeVisible({ + timeout: 30000, + }); + await expect( + messagesView(page).getByText(`E2E approval probe executed: original-${label}`), + ).toHaveCount(0); + await expect(approvalCards(page)).toHaveCount(0); + } finally { + await cleanupAgent(page, agentId); + } + }); + + test('requires edited arguments to be a JSON object and executes only the edit', async ({ + page, + }) => { + test.setTimeout(120000); + const label = uniqueLabel(); + const editedValue = `edited-${label}`; + let agentId: string | undefined; + + try { + agentId = await createAndSelectApprovalAgent(page); + const card = await startApproval(page, label); + const submit = card.getByRole('button', { name: 'Submit' }); + + await card.getByRole('button', { name: 'Edit' }).click(); + const editor = card.getByRole('textbox', { name: 'Edit' }); + await expect(editor).toHaveValue(new RegExp(`original-${label}`)); + + for (const invalid of ['{', 'null', '[]', '"text"']) { + await editor.fill(invalid); + await expect(card.getByText('Invalid JSON')).toBeVisible(); + await expect(submit).toBeDisabled(); + } + + await editor.fill(JSON.stringify({ value: editedValue })); + await expect(card.getByText('Invalid JSON')).toHaveCount(0); + await expect(submit).toBeEnabled(); + + const { body } = await submitAndCapture(page, submit); + expect(body.decisions).toEqual([ + expect.objectContaining({ + decision: 'edit', + editedArguments: { value: editedValue }, + tool_call_id: `call_e2e_approval_${label}`, + }), + ]); + + await expect( + messagesView(page).getByText(`E2E approval probe executed: ${editedValue}`), + ).toBeVisible({ timeout: 30000 }); + await expect( + messagesView(page).getByText(`E2E approval probe executed: original-${label}`), + ).toHaveCount(0); + } finally { + await cleanupAgent(page, agentId); + } + }); + + test('requires a nonblank substitute response and skips tool execution', async ({ page }) => { + test.setTimeout(120000); + const label = uniqueLabel(); + const responseText = `manual result ${label}`; + let agentId: string | undefined; + + try { + agentId = await createAndSelectApprovalAgent(page); + const card = await startApproval(page, label); + const submit = card.getByRole('button', { name: 'Submit' }); + + await card.getByRole('button', { name: 'Respond' }).click(); + const responseInput = card.getByRole('textbox', { name: 'Respond' }); + await responseInput.fill(' '); + await expect(submit).toBeDisabled(); + await responseInput.fill(` ${responseText} `); + await expect(submit).toBeEnabled(); + + const { body } = await submitAndCapture(page, submit); + expect(body.decisions).toEqual([ + expect.objectContaining({ + decision: 'respond', + responseText, + tool_call_id: `call_e2e_approval_${label}`, + }), + ]); + + await expect(messagesView(page).getByText(responseText)).toBeVisible({ timeout: 30000 }); + await expect( + messagesView(page).getByText(`E2E approval probe executed: original-${label}`), + ).toHaveCount(0); + } finally { + await cleanupAgent(page, agentId); + } + }); + + test('honors a hook-restricted decision set', async ({ page }) => { + test.setTimeout(120000); + const label = uniqueLabel(); + let agentId: string | undefined; + + try { + agentId = await createAndSelectApprovalAgent(page); + const card = await startApproval( + page, + label, + RESTRICTED_APPROVAL_PROMPT_MARKER, + APPROVAL_REASON, + ); + + await expect(card.getByRole('button', { name: 'Approve' })).toBeVisible(); + await expect(card.getByRole('button', { name: 'Reject' })).toBeVisible(); + await expect(card.getByRole('button', { name: 'Edit' })).toHaveCount(0); + await expect(card.getByRole('button', { name: 'Respond' })).toHaveCount(0); + + const submit = card.getByRole('button', { name: 'Submit' }); + await card.getByRole('button', { name: 'Approve' }).click(); + const { body } = await submitAndCapture(page, submit); + expect(body.decisions).toEqual([ + expect.objectContaining({ + decision: 'approve', + tool_call_id: `call_e2e_approval_${label}`, + }), + ]); + await expect( + messagesView(page).getByText(`E2E approval probe executed: original-${label}`), + ).toBeVisible({ timeout: 30000 }); + } finally { + await cleanupAgent(page, agentId); + } + }); + + test('reviews and approves the authoritative hook-rewritten arguments', async ({ page }) => { + test.setTimeout(120000); + const label = uniqueLabel(); + let agentId: string | undefined; + + try { + agentId = await createAndSelectApprovalAgent(page); + const card = await startApproval( + page, + label, + REWRITTEN_APPROVAL_PROMPT_MARKER, + APPROVAL_REASON, + ); + + await card.getByRole('button', { name: 'Edit' }).click(); + const editor = card.getByRole('textbox', { name: 'Edit' }); + await expect(editor).toHaveValue(new RegExp(`rewritten-${label}`)); + await expect(editor).not.toHaveValue(new RegExp(`original-${label}`)); + + await card.getByRole('button', { name: 'Edit' }).click(); + const submit = card.getByRole('button', { name: 'Submit' }); + await card.getByRole('button', { name: 'Approve' }).click(); + const { body } = await submitAndCapture(page, submit); + expect(body.decisions).toEqual([ + expect.objectContaining({ + decision: 'approve', + tool_call_id: `call_e2e_approval_${label}`, + }), + ]); + + await expect( + messagesView(page).getByText(`E2E approval probe executed: rewritten-${label}`), + ).toBeVisible({ timeout: 30000 }); + await expect( + messagesView(page).getByText(`E2E approval probe executed: original-${label}`), + ).toHaveCount(0); + } finally { + await cleanupAgent(page, agentId); + } + }); + + test('submits a mixed batch once and preserves decisions through collapse', async ({ page }) => { + test.setTimeout(120000); + const label = uniqueLabel(); + const firstCallId = `call_e2e_approval_${label}_first`; + const secondCallId = `call_e2e_approval_${label}_second`; + const responseText = `manual batch result ${label}`; + let agentId: string | undefined; + + try { + agentId = await createAndSelectApprovalAgent(page); + await startApproval(page, label, BATCH_APPROVAL_PROMPT_MARKER); + await expect(approvalCards(page)).toHaveCount(2); + + const firstCard = approvalCard(page, firstCallId); + const secondCard = approvalCard(page, secondCallId); + const submit = messagesView(page).getByRole('button', { + name: 'Submit 2 decisions', + exact: true, + }); + + await secondCard.getByRole('button', { name: 'Respond' }).click(); + await secondCard.getByRole('textbox', { name: 'Respond' }).fill(responseText); + await expect(submit).toBeDisabled(); + await firstCard.getByRole('button', { name: 'Approve' }).click(); + await expect(submit).toBeEnabled(); + + const groupToggle = messagesView(page).getByRole('button', { + name: 'Used 2 tools', + exact: true, + }); + await groupToggle.click(); + await expect(groupToggle).toHaveAttribute('aria-expanded', 'false'); + await page.waitForTimeout(400); + await groupToggle.click(); + await expect(groupToggle).toHaveAttribute('aria-expanded', 'true'); + + const reopenedFirstCard = approvalCard(page, firstCallId); + const reopenedSecondCard = approvalCard(page, secondCallId); + await expect(reopenedFirstCard.getByRole('button', { name: 'Approve' })).toHaveAttribute( + 'aria-pressed', + 'true', + ); + await expect(reopenedSecondCard.getByRole('button', { name: 'Respond' })).toHaveAttribute( + 'aria-pressed', + 'true', + ); + await expect(reopenedSecondCard.getByRole('textbox', { name: 'Respond' })).toHaveValue( + responseText, + ); + await expect(submit).toBeEnabled(); + + const { body } = await submitAndCapture(page, submit); + expect(body.decisions).toHaveLength(2); + expect(body.decisions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + decision: 'approve', + tool_call_id: firstCallId, + }), + expect.objectContaining({ + decision: 'respond', + responseText, + tool_call_id: secondCallId, + }), + ]), + ); + + await expect( + messagesView(page).getByText(`E2E approval probe executed: first-${label}`), + ).toBeVisible({ timeout: 30000 }); + await expect(messagesView(page).getByText(responseText)).toBeVisible({ timeout: 30000 }); + await expect( + messagesView(page).getByText(`E2E approval probe executed: second-${label}`), + ).toHaveCount(0); + await expect(approvalCards(page)).toHaveCount(0); + } finally { + await cleanupAgent(page, agentId); + } + }); + + test('rehydrates a paused approval and its completed result across reloads', async ({ page }) => { + test.setTimeout(120000); + const label = uniqueLabel(); + const toolCallId = `call_e2e_approval_${label}`; + const executedText = `E2E approval probe executed: original-${label}`; + let agentId: string | undefined; + + try { + agentId = await createAndSelectApprovalAgent(page); + await startApproval(page, label); + const conversationPath = new URL(page.url()).pathname; + + await page.reload({ waitUntil: 'domcontentloaded' }); + await expect.poll(() => new URL(page.url()).pathname).toBe(conversationPath); + const rehydratedCard = approvalCard(page, toolCallId); + await expect(rehydratedCard).toBeVisible({ timeout: 30000 }); + await expect(rehydratedCard).toContainText(APPROVAL_REASON); + + await page.goto(NEW_CHAT_PATH, { waitUntil: 'domcontentloaded' }); + await expect(approvalCards(page)).toHaveCount(0); + await page.goto(conversationPath, { waitUntil: 'domcontentloaded' }); + const navigatedCard = approvalCard(page, toolCallId); + await expect(navigatedCard).toBeVisible({ timeout: 30000 }); + await expect(navigatedCard).toContainText(APPROVAL_REASON); + + await navigatedCard.getByRole('button', { name: 'Approve' }).click(); + await submitAndCapture(page, navigatedCard.getByRole('button', { name: 'Submit' })); + await expect(messagesView(page).getByText(executedText)).toBeVisible({ timeout: 30000 }); + await expect(approvalCards(page)).toHaveCount(0); + + await page.reload({ waitUntil: 'domcontentloaded' }); + await expect.poll(() => new URL(page.url()).pathname).toBe(conversationPath); + await expect(messagesView(page).getByText(executedText)).toBeVisible({ timeout: 30000 }); + await expect(approvalCards(page)).toHaveCount(0); + } finally { + await cleanupAgent(page, agentId); + } + }); + + test('sends only one resume request for two synchronous submit clicks', async ({ page }) => { + test.setTimeout(120000); + const label = uniqueLabel(); + const executedText = `E2E approval probe executed: original-${label}`; + let agentId: string | undefined; + let releaseResume = () => undefined; + let resumeHandler: ((route: Route) => Promise) | undefined; + + try { + agentId = await createAndSelectApprovalAgent(page); + const card = await startApproval(page, label); + const submit = card.getByRole('button', { name: 'Submit' }); + await card.getByRole('button', { name: 'Approve' }).click(); + await expect(submit).toBeEnabled(); + + let resumeRequests = 0; + const resumeGate = new Promise((resolve) => { + releaseResume = resolve; + }); + resumeHandler = async (route) => { + resumeRequests++; + if (resumeRequests === 1) { + await resumeGate; + await route.continue(); + return; + } + await route.fulfill({ + status: 409, + contentType: 'application/json', + body: JSON.stringify({ message: 'duplicate resume request' }), + }); + }; + await page.route('**/api/agents/chat/resume', resumeHandler); + + await submit.evaluate((button: HTMLButtonElement) => { + button.click(); + button.click(); + }); + await page.waitForTimeout(250); + expect(resumeRequests).toBe(1); + await expect(card.getByRole('button', { name: 'Submitting' })).toBeDisabled(); + await expect(card.getByRole('button', { name: 'Approve' })).toBeDisabled(); + releaseResume(); + + await expect(messagesView(page).getByText(executedText)).toBeVisible({ timeout: 30000 }); + await expect(approvalCards(page)).toHaveCount(0); + } finally { + releaseResume(); + if (resumeHandler) { + await page.unroute('**/api/agents/chat/resume', resumeHandler); + } + await cleanupAgent(page, agentId); + } + }); + + test('preserves a decision after a transient resume error and retries successfully', async ({ + page, + }) => { + test.setTimeout(120000); + const label = uniqueLabel(); + const responseText = `retry response ${label}`; + let agentId: string | undefined; + let resumeHandler: ((route: Route) => Promise) | undefined; + + try { + agentId = await createAndSelectApprovalAgent(page); + const card = await startApproval(page, label); + const submit = card.getByRole('button', { name: 'Submit' }); + await card.getByRole('button', { name: 'Respond' }).click(); + const responseInput = card.getByRole('textbox', { name: 'Respond' }); + await responseInput.fill(responseText); + + let resumeRequests = 0; + resumeHandler = async (route) => { + resumeRequests++; + if (resumeRequests === 1) { + await route.fulfill({ + status: 500, + contentType: 'application/json', + body: JSON.stringify({ message: 'temporary e2e failure' }), + }); + return; + } + await route.continue(); + }; + await page.route('**/api/agents/chat/resume', resumeHandler); + + await Promise.all([ + page.waitForResponse( + (response) => isResumeRequest(response.request()) && response.status() === 500, + ), + submit.click(), + ]); + await expect(card.getByText(APPROVAL_ERROR, { exact: true })).toBeVisible(); + await expect(card.getByRole('button', { name: 'Respond' })).toHaveAttribute( + 'aria-pressed', + 'true', + ); + await expect(responseInput).toHaveValue(responseText); + await expect(submit).toBeEnabled(); + + await Promise.all([ + page.waitForResponse( + (response) => isResumeRequest(response.request()) && response.status() === 200, + ), + submit.click(), + ]); + await expect(messagesView(page).getByText(responseText)).toBeVisible({ timeout: 30000 }); + await expect( + messagesView(page).getByText(`E2E approval probe executed: original-${label}`), + ).toHaveCount(0); + expect(resumeRequests).toBe(2); + await expect(approvalCards(page)).toHaveCount(0); + } finally { + if (resumeHandler) { + await page.unroute('**/api/agents/chat/resume', resumeHandler); + } + await cleanupAgent(page, agentId); + } + }); + + test('locks the approval controls and explains an expired resume action', async ({ page }) => { + test.setTimeout(120000); + const label = uniqueLabel(); + const executedText = `E2E approval probe executed: original-${label}`; + let agentId: string | undefined; + let capturedResumeBody: Record | undefined; + let backendResolved = false; + let routeInstalled = false; + const resumeHandler = async (route: Route) => { + capturedResumeBody = route.request().postDataJSON() as Record; + await route.fulfill({ + status: 409, + contentType: 'application/json', + body: JSON.stringify({ message: 'expired e2e action' }), + }); + }; + + try { + agentId = await createAndSelectApprovalAgent(page); + const card = await startApproval(page, label); + const approve = card.getByRole('button', { name: 'Approve' }); + const submit = card.getByRole('button', { name: 'Submit' }); + await approve.click(); + await page.route('**/api/agents/chat/resume', resumeHandler); + routeInstalled = true; + + await Promise.all([ + page.waitForResponse( + (response) => isResumeRequest(response.request()) && response.status() === 409, + ), + submit.click(), + ]); + await expect(card.getByText(APPROVAL_EXPIRED, { exact: true })).toBeVisible(); + await expect(approve).toHaveAttribute('aria-pressed', 'true'); + await expect(approve).toBeDisabled(); + await expect(card.getByRole('button', { name: 'Reject' })).toBeDisabled(); + await expect(card.getByRole('button', { name: 'Edit' })).toBeDisabled(); + await expect(card.getByRole('button', { name: 'Respond' })).toBeDisabled(); + await expect(submit).toBeDisabled(); + expect(capturedResumeBody).toBeDefined(); + + await page.unroute('**/api/agents/chat/resume', resumeHandler); + routeInstalled = false; + const token = await getAccessToken(page); + await requestJson(page, { + path: '/api/agents/chat/resume', + token, + method: 'POST', + body: capturedResumeBody, + }); + backendResolved = true; + await expect(messagesView(page).getByText(executedText)).toBeVisible({ timeout: 30000 }); + await expect(approvalCards(page)).toHaveCount(0); + } finally { + if (routeInstalled) { + await page.unroute('**/api/agents/chat/resume', resumeHandler); + } + if (!backendResolved && capturedResumeBody) { + const token = await getAccessToken(page); + await requestJson(page, { + path: '/api/agents/chat/resume', + token, + method: 'POST', + body: capturedResumeBody, + }).catch(() => undefined); + } + await cleanupAgent(page, agentId); + } + }); +});