mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-08-04 14:57:42 +00:00
test: cover tool approval workflows end to end
This commit is contained in:
parent
f3159f9891
commit
90ac3d25a5
12 changed files with 1064 additions and 5 deletions
|
|
@ -281,6 +281,9 @@ export function useResumeSubmit() {
|
|||
const approvalMutation = useSubmitToolApprovalMutation();
|
||||
const askMutation = useSubmitAskAnswerMutation();
|
||||
const { getDecisions, isReady, setStatus } = useApprovalContext();
|
||||
/** React state cannot lock a second click in the same browser task. Keep a
|
||||
* synchronous action-id guard alongside the rendered submission status. */
|
||||
const submittingToolActionIdsRef = useRef(new Set<string>());
|
||||
/** Ask status lives in Recoil so it works from the composer (outside the
|
||||
* provider); tool-approval status stays on the context. */
|
||||
const { setAskStatus } = useAskSubmitStatus();
|
||||
|
|
@ -311,12 +314,23 @@ export function useResumeSubmit() {
|
|||
if (!fields || decisions.length === 0 || !isReady(actionId)) {
|
||||
return;
|
||||
}
|
||||
if (submittingToolActionIdsRef.current.has(actionId)) {
|
||||
return;
|
||||
}
|
||||
submittingToolActionIdsRef.current.add(actionId);
|
||||
setStatus(actionId, 'submitting');
|
||||
approvalMutation.mutate(
|
||||
{ ...fields, actionId, decisions },
|
||||
{
|
||||
onSuccess: () => setStatus(actionId, 'submitted'),
|
||||
onError: (error) => setStatus(actionId, isExpiredError(error) ? 'expired' : 'error'),
|
||||
onError: (error) => {
|
||||
const expired = isExpiredError(error);
|
||||
if (!expired) {
|
||||
// Network/validation failures are retryable; a 409 is terminal.
|
||||
submittingToolActionIdsRef.current.delete(actionId);
|
||||
}
|
||||
setStatus(actionId, expired ? 'expired' : 'error');
|
||||
},
|
||||
},
|
||||
);
|
||||
},
|
||||
|
|
|
|||
|
|
@ -174,7 +174,11 @@ export default function ToolApproval({
|
|||
}
|
||||
|
||||
return (
|
||||
<div className="my-2 flex w-full flex-col gap-2 rounded-lg border border-border-light bg-surface-secondary p-3">
|
||||
<div
|
||||
className="my-2 flex w-full flex-col gap-2 rounded-lg border border-border-light bg-surface-secondary p-3"
|
||||
data-testid="tool-approval"
|
||||
data-tool-call-id={toolCallId}
|
||||
>
|
||||
{description != null && description.length > 0 && (
|
||||
<p className="text-sm text-text-secondary">{description}</p>
|
||||
)}
|
||||
|
|
|
|||
|
|
@ -110,6 +110,17 @@ export default function ToolCallGroup({
|
|||
const count = parts.length;
|
||||
|
||||
const toolMetadata = useMemo(() => parts.map((p) => getToolMeta(p.part)), [parts]);
|
||||
const hasPendingApproval = useMemo(
|
||||
() =>
|
||||
parts.some(({ part }) => {
|
||||
if (part.type !== ContentTypes.TOOL_CALL) {
|
||||
return false;
|
||||
}
|
||||
const toolCall = part[ContentTypes.TOOL_CALL] as Agents.ToolCall | undefined;
|
||||
return toolCall?.approval != null && (toolCall.output?.length ?? 0) === 0;
|
||||
}),
|
||||
[parts],
|
||||
);
|
||||
const allCompleted = useMemo(
|
||||
() => toolMetadata.every((m) => m?.hasOutput === true),
|
||||
[toolMetadata],
|
||||
|
|
@ -223,13 +234,16 @@ export default function ToolCallGroup({
|
|||
if (event.target !== event.currentTarget) {
|
||||
return;
|
||||
}
|
||||
if (isExpanded) {
|
||||
if (isExpanded || hasPendingApproval) {
|
||||
return;
|
||||
}
|
||||
// Approval controls own unsent local form state. Keep unresolved cards
|
||||
// mounted (the collapsed panel is inert/hidden) so collapsing a batch
|
||||
// cannot erase decisions the reviewer already made.
|
||||
setShouldRenderBody(false);
|
||||
notifyLayoutChange();
|
||||
},
|
||||
[isExpanded, notifyLayoutChange],
|
||||
[hasPendingApproval, isExpanded, notifyLayoutChange],
|
||||
);
|
||||
|
||||
/** Category-aware header verb: subagents and questions read as their own
|
||||
|
|
|
|||
|
|
@ -0,0 +1,76 @@
|
|||
import React from 'react';
|
||||
import { RecoilRoot } from 'recoil';
|
||||
import { act, renderHook } from '@testing-library/react';
|
||||
import { ChatContext } from '~/Providers/ChatContext';
|
||||
import ApprovalProvider, { useApprovalContext, useResumeSubmit } from '../ApprovalContext';
|
||||
|
||||
const mockApprovalMutate = jest.fn();
|
||||
const mockAskMutate = jest.fn();
|
||||
|
||||
jest.mock('~/data-provider', () => ({
|
||||
useSubmitToolApprovalMutation: () => ({ mutate: mockApprovalMutate }),
|
||||
useSubmitAskAnswerMutation: () => ({ mutate: mockAskMutate }),
|
||||
}));
|
||||
|
||||
jest.mock('~/store/agents', () => ({
|
||||
useGetEphemeralAgent: () => () => undefined,
|
||||
}));
|
||||
|
||||
const chatContextValue = {
|
||||
conversation: {
|
||||
conversationId: 'conversation-1',
|
||||
endpoint: 'agents',
|
||||
agent_id: 'agent-1',
|
||||
},
|
||||
} as unknown as React.ContextType<typeof ChatContext>;
|
||||
|
||||
function wrapper({ children }: { children: React.ReactNode }) {
|
||||
return (
|
||||
<RecoilRoot>
|
||||
<ChatContext.Provider value={chatContextValue}>
|
||||
<ApprovalProvider>{children}</ApprovalProvider>
|
||||
</ChatContext.Provider>
|
||||
</RecoilRoot>
|
||||
);
|
||||
}
|
||||
|
||||
describe('useResumeSubmit', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
test('synchronously deduplicates tool approval submissions and unlocks a retryable error', () => {
|
||||
const { result } = renderHook(
|
||||
() => ({
|
||||
approval: useApprovalContext(),
|
||||
resume: useResumeSubmit(),
|
||||
}),
|
||||
{ wrapper },
|
||||
);
|
||||
|
||||
act(() => {
|
||||
result.current.approval.registerToolCall('action-1', 'call-1');
|
||||
result.current.approval.setDecision('action-1', 'call-1', {
|
||||
tool_call_id: 'call-1',
|
||||
decision: 'approve',
|
||||
});
|
||||
});
|
||||
|
||||
act(() => {
|
||||
result.current.resume.submitToolApproval('action-1');
|
||||
result.current.resume.submitToolApproval('action-1');
|
||||
});
|
||||
expect(mockApprovalMutate).toHaveBeenCalledTimes(1);
|
||||
|
||||
const firstOptions = mockApprovalMutate.mock.calls[0][1] as {
|
||||
onError: (error: unknown) => void;
|
||||
};
|
||||
act(() => firstOptions.onError(new Error('temporary failure')));
|
||||
|
||||
act(() => {
|
||||
result.current.resume.submitToolApproval('action-1');
|
||||
result.current.resume.submitToolApproval('action-1');
|
||||
});
|
||||
expect(mockApprovalMutate).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
|
@ -83,6 +83,21 @@ const makePart = (
|
|||
},
|
||||
}) as unknown as TMessageContentParts;
|
||||
|
||||
const makePendingApprovalPart = (id: string): TMessageContentParts =>
|
||||
({
|
||||
type: ContentTypes.TOOL_CALL,
|
||||
[ContentTypes.TOOL_CALL]: {
|
||||
id,
|
||||
name: 'approval_probe',
|
||||
args: {},
|
||||
output: '',
|
||||
approval: {
|
||||
actionId: 'action-1',
|
||||
allowed_decisions: ['approve', 'reject'],
|
||||
},
|
||||
},
|
||||
}) as unknown as TMessageContentParts;
|
||||
|
||||
const imageAttachment: TAttachment = {
|
||||
filename: 'foo.png',
|
||||
filepath: '/files/foo.png',
|
||||
|
|
@ -211,6 +226,33 @@ describe('ToolCallGroup image hoisting', () => {
|
|||
expect(screen.queryByTestId('inner-0')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('keeps unresolved approval bodies mounted while the group is collapsed', () => {
|
||||
const approvalParts = [
|
||||
{ part: makePendingApprovalPart('t1'), idx: 0 },
|
||||
{ part: makePendingApprovalPart('t2'), idx: 1 },
|
||||
];
|
||||
renderGroup({
|
||||
...baseProps,
|
||||
parts: approvalParts,
|
||||
renderPart: (_p: TMessageContentParts, idx: number) => (
|
||||
<div data-testid={`approval-${idx}`} key={idx}>
|
||||
{'approval'}
|
||||
</div>
|
||||
),
|
||||
});
|
||||
|
||||
const button = screen.getByRole('button', { name: 'Used 2 tools' });
|
||||
const collapsible = button.nextElementSibling as HTMLElement;
|
||||
expect(screen.getByTestId('approval-0')).toBeInTheDocument();
|
||||
|
||||
fireEvent.click(button);
|
||||
fireEvent.transitionEnd(collapsible);
|
||||
|
||||
expect(button).toHaveAttribute('aria-expanded', 'false');
|
||||
expect(screen.getByTestId('approval-0')).toBeInTheDocument();
|
||||
expect(screen.getByTestId('approval-1')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('reconciles layout after the group collapses from an expanded state', async () => {
|
||||
renderGroup(baseProps);
|
||||
|
||||
|
|
|
|||
|
|
@ -71,6 +71,37 @@ describe('applyPendingAction — tool_approval', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('replaces displayed tool args with the matching action request arguments', () => {
|
||||
const originalArgs = { query: 'original model args' };
|
||||
const rewrittenArgs = { query: 'rewritten by policy hook' };
|
||||
const message = msg({ content: [toolCallPart('tc1', { args: originalArgs })] });
|
||||
const action = toolApprovalAction({
|
||||
payload: {
|
||||
type: 'tool_approval',
|
||||
action_requests: [
|
||||
{
|
||||
name: 'search',
|
||||
arguments: rewrittenArgs,
|
||||
tool_call_id: 'tc1',
|
||||
description: 'Review rewritten search',
|
||||
},
|
||||
],
|
||||
review_configs: [
|
||||
{
|
||||
action_name: 'search',
|
||||
tool_call_id: 'tc1',
|
||||
allowed_decisions: ['approve', 'reject', 'edit', 'respond'],
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
const result = applyPendingAction(message, action);
|
||||
|
||||
expect(getToolCall(result.content?.[0] as TMessageContentParts)?.args).toEqual(rewrittenArgs);
|
||||
expect(getToolCall(message.content?.[0] as TMessageContentParts)?.args).toEqual(originalArgs);
|
||||
});
|
||||
|
||||
it('leaves a completed tool call (with output) untouched and returns the same message reference', () => {
|
||||
const message = msg({ content: [toolCallPart('tc1', { output: 'already ran' })] });
|
||||
const result = applyPendingAction(message, toolApprovalAction());
|
||||
|
|
|
|||
|
|
@ -95,6 +95,10 @@ function tagApprovalOnPart(
|
|||
const reviewConfig = reviewByToolCallId.get(toolCallId);
|
||||
nextToolCall = {
|
||||
...nextToolCall,
|
||||
// A PreToolUse hook may replace the model's original args before asking.
|
||||
// The interrupt payload is authoritative so the reviewer sees, edits, and
|
||||
// approves the same arguments the resumed tool will actually execute.
|
||||
args: request.arguments,
|
||||
approval: {
|
||||
actionId,
|
||||
allowed_decisions: reviewConfig?.allowed_decisions ?? [],
|
||||
|
|
|
|||
|
|
@ -60,6 +60,18 @@ endpoints:
|
|||
- chain
|
||||
- ocr
|
||||
- run_in_background
|
||||
# Keep the shared mock profile non-interactive except for the dedicated
|
||||
# approval probe. This exercises real HITL pause/resume without wedging the
|
||||
# existing file-authoring, steering, background-tool, or MCP specs.
|
||||
toolApproval:
|
||||
enabled: true
|
||||
mode: bypass
|
||||
ask:
|
||||
- approval_probe_mcp_e2e-memory
|
||||
reason: E2E approval required before running {tool}.
|
||||
hooks:
|
||||
- module: e2e/setup/tool-approval-hook.js
|
||||
matcher: ^approval_probe_mcp_e2e-memory$
|
||||
custom:
|
||||
- name: 'Mock Provider A'
|
||||
apiKey: 'e2e-mock-key-a'
|
||||
|
|
|
|||
|
|
@ -66,6 +66,26 @@ server.registerTool(
|
|||
},
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
'approval_probe',
|
||||
{
|
||||
description:
|
||||
'Echoes reviewed input so LibreChat mock end-to-end tests can verify tool approval decisions.',
|
||||
inputSchema: {
|
||||
value: z.string(),
|
||||
review: z.string().optional(),
|
||||
},
|
||||
},
|
||||
async ({ value }) => ({
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `E2E approval probe executed: ${value}`,
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
async function main() {
|
||||
await server.connect(new StdioServerTransport());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,6 +34,10 @@ const FORCED_ERROR_MARKER = 'E2E_FORCED_ERROR:';
|
|||
const MARKDOWN_REPLY_MARKER = 'E2E_MARKDOWN_REPLY';
|
||||
const BACKGROUND_DISPATCH_MARKER = 'E2E_BACKGROUND_DISPATCH:';
|
||||
const BACKGROUND_COLLECT_MARKER = 'E2E_BACKGROUND_COLLECT:';
|
||||
const TOOL_APPROVAL_MARKER = 'E2E_TOOL_APPROVAL:';
|
||||
const TOOL_APPROVAL_BATCH_MARKER = 'E2E_TOOL_APPROVAL_BATCH:';
|
||||
const TOOL_APPROVAL_RESTRICTED_MARKER = 'E2E_TOOL_APPROVAL_RESTRICTED:';
|
||||
const TOOL_APPROVAL_REWRITE_MARKER = 'E2E_TOOL_APPROVAL_REWRITE:';
|
||||
const CREATE_FILE_AUTHORING_FINAL_TEXT = 'E2E file authoring complete';
|
||||
const EDIT_FILE_AUTHORING_FINAL_TEXT = 'E2E file edit complete';
|
||||
const SKILL_ASSERTION_FINAL_TEXT = 'E2E skill assertion passed';
|
||||
|
|
@ -56,6 +60,8 @@ const CREATE_SKILL_TOOL_CALL_ID = 'call_e2e_create_skill';
|
|||
const EDIT_SKILL_TOOL_CALL_ID = 'call_e2e_edit_skill';
|
||||
const BACKGROUND_TOOL_NAME = 'slow_echo_mcp_e2e-memory';
|
||||
const CHECK_BACKGROUND_TASK_TOOL_NAME = 'check_background_task';
|
||||
const APPROVAL_TOOL_NAME = 'approval_probe_mcp_e2e-memory';
|
||||
const APPROVAL_TOOL_CALL_PREFIX = 'call_e2e_approval_';
|
||||
const BACKGROUND_DISPATCH_TOOL_CALL_ID = 'call_e2e_background_dispatch';
|
||||
const BACKGROUND_COLLECT_TOOL_CALL_ID = 'call_e2e_background_collect';
|
||||
const MODEL_SPEC_ACCESSIBLE_SKILL = 'e2e-model-spec-allowed';
|
||||
|
|
@ -832,6 +838,92 @@ function findLastToolMessageText(messages, requiredToken) {
|
|||
return '';
|
||||
}
|
||||
|
||||
function approvalToolResponses(label, toolNames, review) {
|
||||
if (!toolNames.has(APPROVAL_TOOL_NAME)) {
|
||||
return {
|
||||
responses: [`E2E approval unavailable: ${APPROVAL_TOOL_NAME} was not advertised.`],
|
||||
};
|
||||
}
|
||||
return {
|
||||
responses: ['', ''],
|
||||
toolCalls: [
|
||||
{
|
||||
id: `${APPROVAL_TOOL_CALL_PREFIX}${label}`,
|
||||
name: APPROVAL_TOOL_NAME,
|
||||
args: {
|
||||
value: `original-${label}`,
|
||||
...(review ? { review } : {}),
|
||||
},
|
||||
type: 'tool_call',
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function batchApprovalToolResponses(label, toolNames) {
|
||||
if (!toolNames.has(APPROVAL_TOOL_NAME)) {
|
||||
return {
|
||||
responses: [`E2E approval unavailable: ${APPROVAL_TOOL_NAME} was not advertised.`],
|
||||
};
|
||||
}
|
||||
return {
|
||||
responses: ['', ''],
|
||||
toolCalls: [
|
||||
{
|
||||
id: `${APPROVAL_TOOL_CALL_PREFIX}${label}_first`,
|
||||
name: APPROVAL_TOOL_NAME,
|
||||
args: { value: `first-${label}` },
|
||||
type: 'tool_call',
|
||||
},
|
||||
{
|
||||
id: `${APPROVAL_TOOL_CALL_PREFIX}${label}_second`,
|
||||
name: APPROVAL_TOOL_NAME,
|
||||
args: { value: `second-${label}` },
|
||||
type: 'tool_call',
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume rebuilds the fake model without the original prompt in `context.messages`.
|
||||
* Detect the checkpoint-restored approval tool messages on every model instance
|
||||
* so the continuation can report the real approve/reject/edit/respond outcome.
|
||||
*/
|
||||
function approvalOutcomeResponses(messages) {
|
||||
let latestHumanIndex = -1;
|
||||
for (let index = 0; index < (messages ?? []).length; index++) {
|
||||
const type = messageType(messages[index]);
|
||||
if (type === 'human' || type === 'user') {
|
||||
latestHumanIndex = index;
|
||||
}
|
||||
}
|
||||
|
||||
const outcomeMessages = (messages ?? [])
|
||||
.slice(latestHumanIndex + 1)
|
||||
.filter(
|
||||
(message) =>
|
||||
messageType(message) === 'tool' &&
|
||||
typeof message?.tool_call_id === 'string' &&
|
||||
message.tool_call_id.startsWith(APPROVAL_TOOL_CALL_PREFIX),
|
||||
);
|
||||
|
||||
const isBatch = outcomeMessages.some(
|
||||
(message) =>
|
||||
message.tool_call_id.endsWith('_first') || message.tool_call_id.endsWith('_second'),
|
||||
);
|
||||
if (isBatch && outcomeMessages.length < 2) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const outcomes = outcomeMessages.map((message) => getContentText(message.content));
|
||||
|
||||
if (outcomes.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return { responses: [`E2E approval outcomes: ${outcomes.join(' | ')}`] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn 1 of the background e2e: emit the MCP tool call with the injected
|
||||
* `run_in_background: true` arg, then (second model invocation, after the
|
||||
|
|
@ -925,6 +1017,26 @@ function backgroundCollectResponses(messages, toolNames) {
|
|||
}
|
||||
|
||||
function resolveResponses({ graph, messages, text, toolNames }) {
|
||||
const batchApprovalLabel = getMarkerValue(text, TOOL_APPROVAL_BATCH_MARKER);
|
||||
if (batchApprovalLabel) {
|
||||
return batchApprovalToolResponses(batchApprovalLabel, toolNames);
|
||||
}
|
||||
|
||||
const restrictedApprovalLabel = getMarkerValue(text, TOOL_APPROVAL_RESTRICTED_MARKER);
|
||||
if (restrictedApprovalLabel) {
|
||||
return approvalToolResponses(restrictedApprovalLabel, toolNames, 'restricted');
|
||||
}
|
||||
|
||||
const rewrittenApprovalLabel = getMarkerValue(text, TOOL_APPROVAL_REWRITE_MARKER);
|
||||
if (rewrittenApprovalLabel) {
|
||||
return approvalToolResponses(rewrittenApprovalLabel, toolNames, 'rewrite');
|
||||
}
|
||||
|
||||
const approvalLabel = getMarkerValue(text, TOOL_APPROVAL_MARKER);
|
||||
if (approvalLabel) {
|
||||
return approvalToolResponses(approvalLabel, toolNames);
|
||||
}
|
||||
|
||||
const reply = replyResponses(text);
|
||||
if (reply) {
|
||||
return reply;
|
||||
|
|
@ -1055,5 +1167,13 @@ module.exports = function fakeModelHook(run, context) {
|
|||
text,
|
||||
toolNames,
|
||||
});
|
||||
overrideModel({ graph, responses, sleep, toolCalls, thrownError, resolveOnStream });
|
||||
overrideModel({
|
||||
graph,
|
||||
responses,
|
||||
sleep,
|
||||
toolCalls,
|
||||
thrownError,
|
||||
resolveOnStream: (streamMessages) =>
|
||||
approvalOutcomeResponses(streamMessages) ?? resolveOnStream?.(streamMessages) ?? null,
|
||||
});
|
||||
};
|
||||
|
|
|
|||
29
e2e/setup/tool-approval-hook.js
Normal file
29
e2e/setup/tool-approval-hook.js
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
/**
|
||||
* Dynamic approval-policy fixture for the mock Playwright suite.
|
||||
*
|
||||
* The `review` argument selects behavior that cannot be expressed by the static
|
||||
* ask list: a restricted decision set, or an authoritative argument rewrite.
|
||||
*/
|
||||
module.exports = () => () => async (input) => {
|
||||
if (input.toolInput.review === 'restricted') {
|
||||
return {
|
||||
decision: 'ask',
|
||||
reason: 'E2E approval offers approve or reject only.',
|
||||
allowedDecisions: ['approve', 'reject'],
|
||||
};
|
||||
}
|
||||
|
||||
if (input.toolInput.review === 'rewrite') {
|
||||
const originalValue =
|
||||
typeof input.toolInput.value === 'string' ? input.toolInput.value : 'original-missing';
|
||||
return {
|
||||
decision: 'ask',
|
||||
reason: 'E2E approval reviews rewritten arguments.',
|
||||
updatedInput: {
|
||||
value: originalValue.replace(/^original-/, 'rewritten-'),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return {};
|
||||
};
|
||||
693
e2e/specs/mock/tool-approvals.spec.ts
Normal file
693
e2e/specs/mock/tool-approvals.spec.ts
Normal file
|
|
@ -0,0 +1,693 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import type { Locator, Page, Request, Route } from '@playwright/test';
|
||||
import type { AgentDetail } from './agents.helpers';
|
||||
import { cleanupAgent, openAgentBuilder, uniqueAgentName } from './agents.helpers';
|
||||
import {
|
||||
MOCK_ENDPOINTS,
|
||||
NEW_CHAT_PATH,
|
||||
fetchJson,
|
||||
getAccessToken,
|
||||
messagesView,
|
||||
requestJson,
|
||||
sendMessage,
|
||||
} from './helpers';
|
||||
|
||||
const MCP_SERVER_NAME = 'e2e-memory';
|
||||
const MCP_SERVER_TOOL_ID = `sys__server__sys_mcp_${MCP_SERVER_NAME}`;
|
||||
const APPROVAL_TOOL_NAME = 'approval_probe';
|
||||
const APPROVAL_TOOL_ID = `${APPROVAL_TOOL_NAME}_mcp_${MCP_SERVER_NAME}`;
|
||||
const APPROVAL_PROMPT_MARKER = 'E2E_TOOL_APPROVAL:';
|
||||
const BATCH_APPROVAL_PROMPT_MARKER = 'E2E_TOOL_APPROVAL_BATCH:';
|
||||
const RESTRICTED_APPROVAL_PROMPT_MARKER = 'E2E_TOOL_APPROVAL_RESTRICTED:';
|
||||
const REWRITTEN_APPROVAL_PROMPT_MARKER = 'E2E_TOOL_APPROVAL_REWRITE:';
|
||||
const APPROVAL_REASON = `E2E approval required before running ${APPROVAL_TOOL_ID}.`;
|
||||
const APPROVAL_ERROR = 'Something went wrong submitting your decision. Please try again.';
|
||||
const APPROVAL_EXPIRED = 'This request expired or was already handled.';
|
||||
const DESCRIPTION = 'Verifies human approval behavior for MCP tool calls in mock E2E tests.';
|
||||
const uniqueLabel = () => `${Date.now()}-${Math.floor(Math.random() * 1e4)}`;
|
||||
|
||||
type MCPToolsResponse = {
|
||||
servers?: Record<string, { tools?: Array<{ pluginKey: string }> }>;
|
||||
};
|
||||
|
||||
type ApprovalResumeBody = {
|
||||
actionId?: string;
|
||||
agent_id?: string;
|
||||
conversationId?: string;
|
||||
endpoint?: string;
|
||||
decisions?: Array<{
|
||||
tool_call_id?: string;
|
||||
decision?: string;
|
||||
reason?: string;
|
||||
responseText?: string;
|
||||
editedArguments?: Record<string, unknown>;
|
||||
}>;
|
||||
};
|
||||
|
||||
type ApprovalResumeResponse = {
|
||||
conversationId?: string;
|
||||
status?: string;
|
||||
streamId?: string;
|
||||
};
|
||||
|
||||
const approvalCards = (page: Page) => messagesView(page).getByTestId('tool-approval');
|
||||
const approvalCard = (page: Page, toolCallId: string) =>
|
||||
messagesView(page).locator(`[data-testid="tool-approval"][data-tool-call-id="${toolCallId}"]`);
|
||||
|
||||
function isResumeRequest(request: Request) {
|
||||
return (
|
||||
request.method() === 'POST' && new URL(request.url()).pathname === '/api/agents/chat/resume'
|
||||
);
|
||||
}
|
||||
|
||||
async function waitForApprovalTool(page: Page) {
|
||||
const token = await getAccessToken(page);
|
||||
let latestTools: MCPToolsResponse | null = null;
|
||||
|
||||
for (let attempt = 0; attempt < 20; attempt++) {
|
||||
latestTools = await fetchJson<MCPToolsResponse>(page, '/api/mcp/tools', token);
|
||||
const tools = latestTools.servers?.[MCP_SERVER_NAME]?.tools ?? [];
|
||||
if (tools.some((tool) => tool.pluginKey === APPROVAL_TOOL_ID)) {
|
||||
return;
|
||||
}
|
||||
await page.waitForTimeout(500);
|
||||
}
|
||||
|
||||
expect(
|
||||
latestTools?.servers?.[MCP_SERVER_NAME]?.tools,
|
||||
`Expected ${MCP_SERVER_NAME} to expose ${APPROVAL_TOOL_ID}`,
|
||||
).toEqual(expect.arrayContaining([expect.objectContaining({ pluginKey: APPROVAL_TOOL_ID })]));
|
||||
}
|
||||
|
||||
async function createAndSelectApprovalAgent(page: Page): Promise<string> {
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
await waitForApprovalTool(page);
|
||||
|
||||
const token = await getAccessToken(page);
|
||||
const agentName = uniqueAgentName('E2E Tool Approval Agent');
|
||||
const agent = await requestJson<AgentDetail>(page, {
|
||||
path: '/api/agents',
|
||||
token,
|
||||
method: 'POST',
|
||||
body: {
|
||||
name: agentName,
|
||||
description: DESCRIPTION,
|
||||
instructions: 'Use the requested approval probe tools and report their results.',
|
||||
provider: MOCK_ENDPOINTS[0].label,
|
||||
model: MOCK_ENDPOINTS[0].model,
|
||||
tools: [MCP_SERVER_TOOL_ID, APPROVAL_TOOL_ID],
|
||||
},
|
||||
});
|
||||
|
||||
const form = await openAgentBuilder(page);
|
||||
await form.getByRole('combobox', { name: 'Agent', exact: true }).click();
|
||||
await page.getByRole('option', { name: agentName }).click();
|
||||
await expect(form.getByLabel('Agent name')).toHaveValue(agentName);
|
||||
await form.getByRole('button', { name: 'Select Agent' }).click();
|
||||
return agent.id;
|
||||
}
|
||||
|
||||
async function startApproval(
|
||||
page: Page,
|
||||
label: string,
|
||||
marker = APPROVAL_PROMPT_MARKER,
|
||||
expectedReason = APPROVAL_REASON,
|
||||
): Promise<Locator> {
|
||||
const response = await sendMessage(page, `${marker}${label}`);
|
||||
expect(response.ok()).toBeTruthy();
|
||||
await expect(page).toHaveURL(/\/c\/(?!new)/, { timeout: 15000 });
|
||||
const card = approvalCards(page).first();
|
||||
await expect(card).toBeVisible({ timeout: 30000 });
|
||||
await expect(card).toContainText(expectedReason);
|
||||
return card;
|
||||
}
|
||||
|
||||
async function submitAndCapture(page: Page, submit: Locator) {
|
||||
const [request, response] = await Promise.all([
|
||||
page.waitForRequest(isResumeRequest),
|
||||
page.waitForResponse(
|
||||
(candidate) => isResumeRequest(candidate.request()) && candidate.status() === 200,
|
||||
),
|
||||
submit.click(),
|
||||
]);
|
||||
return {
|
||||
body: request.postDataJSON() as ApprovalResumeBody,
|
||||
response,
|
||||
};
|
||||
}
|
||||
|
||||
test.describe('tool approvals', () => {
|
||||
test('approves a paused tool with its original arguments', async ({ page }) => {
|
||||
test.setTimeout(120000);
|
||||
const label = uniqueLabel();
|
||||
let agentId: string | undefined;
|
||||
|
||||
try {
|
||||
agentId = await createAndSelectApprovalAgent(page);
|
||||
const card = await startApproval(page, label);
|
||||
|
||||
await expect(card.getByRole('button', { name: 'Approve' })).toBeVisible();
|
||||
await expect(card.getByRole('button', { name: 'Reject' })).toBeVisible();
|
||||
await expect(card.getByRole('button', { name: 'Edit' })).toBeVisible();
|
||||
await expect(card.getByRole('button', { name: 'Respond' })).toBeVisible();
|
||||
|
||||
const submit = card.getByRole('button', { name: 'Submit' });
|
||||
await expect(submit).toBeDisabled();
|
||||
await card.getByRole('button', { name: 'Approve' }).click();
|
||||
await expect(submit).toBeEnabled();
|
||||
|
||||
const conversationId = new URL(page.url()).pathname.replace('/c/', '');
|
||||
const { body, response } = await submitAndCapture(page, submit);
|
||||
expect(body.actionId).toBeTruthy();
|
||||
expect(body.agent_id).toBe(agentId);
|
||||
expect(body.conversationId).toBe(conversationId);
|
||||
expect(body.endpoint).toBe('agents');
|
||||
expect(body.decisions).toEqual([
|
||||
expect.objectContaining({
|
||||
decision: 'approve',
|
||||
tool_call_id: `call_e2e_approval_${label}`,
|
||||
}),
|
||||
]);
|
||||
await expect(response.json() as Promise<ApprovalResumeResponse>).resolves.toEqual(
|
||||
expect.objectContaining({
|
||||
conversationId,
|
||||
status: 'resuming',
|
||||
streamId: conversationId,
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
messagesView(page).getByText(`E2E approval probe executed: original-${label}`),
|
||||
).toBeVisible({ timeout: 30000 });
|
||||
await expect(approvalCards(page)).toHaveCount(0);
|
||||
} finally {
|
||||
await cleanupAgent(page, agentId);
|
||||
}
|
||||
});
|
||||
|
||||
test('rejects with an optional reason without executing the tool', async ({ page }) => {
|
||||
test.setTimeout(120000);
|
||||
const label = uniqueLabel();
|
||||
const reason = `do not run ${label}`;
|
||||
let agentId: string | undefined;
|
||||
|
||||
try {
|
||||
agentId = await createAndSelectApprovalAgent(page);
|
||||
const card = await startApproval(page, label);
|
||||
const submit = card.getByRole('button', { name: 'Submit' });
|
||||
|
||||
await card.getByRole('button', { name: 'Reject' }).click();
|
||||
await card.getByRole('textbox', { name: 'Reject' }).fill(` ${reason} `);
|
||||
await expect(submit).toBeEnabled();
|
||||
|
||||
const { body } = await submitAndCapture(page, submit);
|
||||
expect(body.decisions).toEqual([
|
||||
expect.objectContaining({
|
||||
decision: 'reject',
|
||||
reason,
|
||||
tool_call_id: `call_e2e_approval_${label}`,
|
||||
}),
|
||||
]);
|
||||
|
||||
await expect(messagesView(page).getByText(`Blocked: ${reason}`)).toBeVisible({
|
||||
timeout: 30000,
|
||||
});
|
||||
await expect(
|
||||
messagesView(page).getByText(`E2E approval probe executed: original-${label}`),
|
||||
).toHaveCount(0);
|
||||
await expect(approvalCards(page)).toHaveCount(0);
|
||||
} finally {
|
||||
await cleanupAgent(page, agentId);
|
||||
}
|
||||
});
|
||||
|
||||
test('requires edited arguments to be a JSON object and executes only the edit', async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(120000);
|
||||
const label = uniqueLabel();
|
||||
const editedValue = `edited-${label}`;
|
||||
let agentId: string | undefined;
|
||||
|
||||
try {
|
||||
agentId = await createAndSelectApprovalAgent(page);
|
||||
const card = await startApproval(page, label);
|
||||
const submit = card.getByRole('button', { name: 'Submit' });
|
||||
|
||||
await card.getByRole('button', { name: 'Edit' }).click();
|
||||
const editor = card.getByRole('textbox', { name: 'Edit' });
|
||||
await expect(editor).toHaveValue(new RegExp(`original-${label}`));
|
||||
|
||||
for (const invalid of ['{', 'null', '[]', '"text"']) {
|
||||
await editor.fill(invalid);
|
||||
await expect(card.getByText('Invalid JSON')).toBeVisible();
|
||||
await expect(submit).toBeDisabled();
|
||||
}
|
||||
|
||||
await editor.fill(JSON.stringify({ value: editedValue }));
|
||||
await expect(card.getByText('Invalid JSON')).toHaveCount(0);
|
||||
await expect(submit).toBeEnabled();
|
||||
|
||||
const { body } = await submitAndCapture(page, submit);
|
||||
expect(body.decisions).toEqual([
|
||||
expect.objectContaining({
|
||||
decision: 'edit',
|
||||
editedArguments: { value: editedValue },
|
||||
tool_call_id: `call_e2e_approval_${label}`,
|
||||
}),
|
||||
]);
|
||||
|
||||
await expect(
|
||||
messagesView(page).getByText(`E2E approval probe executed: ${editedValue}`),
|
||||
).toBeVisible({ timeout: 30000 });
|
||||
await expect(
|
||||
messagesView(page).getByText(`E2E approval probe executed: original-${label}`),
|
||||
).toHaveCount(0);
|
||||
} finally {
|
||||
await cleanupAgent(page, agentId);
|
||||
}
|
||||
});
|
||||
|
||||
test('requires a nonblank substitute response and skips tool execution', async ({ page }) => {
|
||||
test.setTimeout(120000);
|
||||
const label = uniqueLabel();
|
||||
const responseText = `manual result ${label}`;
|
||||
let agentId: string | undefined;
|
||||
|
||||
try {
|
||||
agentId = await createAndSelectApprovalAgent(page);
|
||||
const card = await startApproval(page, label);
|
||||
const submit = card.getByRole('button', { name: 'Submit' });
|
||||
|
||||
await card.getByRole('button', { name: 'Respond' }).click();
|
||||
const responseInput = card.getByRole('textbox', { name: 'Respond' });
|
||||
await responseInput.fill(' ');
|
||||
await expect(submit).toBeDisabled();
|
||||
await responseInput.fill(` ${responseText} `);
|
||||
await expect(submit).toBeEnabled();
|
||||
|
||||
const { body } = await submitAndCapture(page, submit);
|
||||
expect(body.decisions).toEqual([
|
||||
expect.objectContaining({
|
||||
decision: 'respond',
|
||||
responseText,
|
||||
tool_call_id: `call_e2e_approval_${label}`,
|
||||
}),
|
||||
]);
|
||||
|
||||
await expect(messagesView(page).getByText(responseText)).toBeVisible({ timeout: 30000 });
|
||||
await expect(
|
||||
messagesView(page).getByText(`E2E approval probe executed: original-${label}`),
|
||||
).toHaveCount(0);
|
||||
} finally {
|
||||
await cleanupAgent(page, agentId);
|
||||
}
|
||||
});
|
||||
|
||||
test('honors a hook-restricted decision set', async ({ page }) => {
|
||||
test.setTimeout(120000);
|
||||
const label = uniqueLabel();
|
||||
let agentId: string | undefined;
|
||||
|
||||
try {
|
||||
agentId = await createAndSelectApprovalAgent(page);
|
||||
const card = await startApproval(
|
||||
page,
|
||||
label,
|
||||
RESTRICTED_APPROVAL_PROMPT_MARKER,
|
||||
APPROVAL_REASON,
|
||||
);
|
||||
|
||||
await expect(card.getByRole('button', { name: 'Approve' })).toBeVisible();
|
||||
await expect(card.getByRole('button', { name: 'Reject' })).toBeVisible();
|
||||
await expect(card.getByRole('button', { name: 'Edit' })).toHaveCount(0);
|
||||
await expect(card.getByRole('button', { name: 'Respond' })).toHaveCount(0);
|
||||
|
||||
const submit = card.getByRole('button', { name: 'Submit' });
|
||||
await card.getByRole('button', { name: 'Approve' }).click();
|
||||
const { body } = await submitAndCapture(page, submit);
|
||||
expect(body.decisions).toEqual([
|
||||
expect.objectContaining({
|
||||
decision: 'approve',
|
||||
tool_call_id: `call_e2e_approval_${label}`,
|
||||
}),
|
||||
]);
|
||||
await expect(
|
||||
messagesView(page).getByText(`E2E approval probe executed: original-${label}`),
|
||||
).toBeVisible({ timeout: 30000 });
|
||||
} finally {
|
||||
await cleanupAgent(page, agentId);
|
||||
}
|
||||
});
|
||||
|
||||
test('reviews and approves the authoritative hook-rewritten arguments', async ({ page }) => {
|
||||
test.setTimeout(120000);
|
||||
const label = uniqueLabel();
|
||||
let agentId: string | undefined;
|
||||
|
||||
try {
|
||||
agentId = await createAndSelectApprovalAgent(page);
|
||||
const card = await startApproval(
|
||||
page,
|
||||
label,
|
||||
REWRITTEN_APPROVAL_PROMPT_MARKER,
|
||||
APPROVAL_REASON,
|
||||
);
|
||||
|
||||
await card.getByRole('button', { name: 'Edit' }).click();
|
||||
const editor = card.getByRole('textbox', { name: 'Edit' });
|
||||
await expect(editor).toHaveValue(new RegExp(`rewritten-${label}`));
|
||||
await expect(editor).not.toHaveValue(new RegExp(`original-${label}`));
|
||||
|
||||
await card.getByRole('button', { name: 'Edit' }).click();
|
||||
const submit = card.getByRole('button', { name: 'Submit' });
|
||||
await card.getByRole('button', { name: 'Approve' }).click();
|
||||
const { body } = await submitAndCapture(page, submit);
|
||||
expect(body.decisions).toEqual([
|
||||
expect.objectContaining({
|
||||
decision: 'approve',
|
||||
tool_call_id: `call_e2e_approval_${label}`,
|
||||
}),
|
||||
]);
|
||||
|
||||
await expect(
|
||||
messagesView(page).getByText(`E2E approval probe executed: rewritten-${label}`),
|
||||
).toBeVisible({ timeout: 30000 });
|
||||
await expect(
|
||||
messagesView(page).getByText(`E2E approval probe executed: original-${label}`),
|
||||
).toHaveCount(0);
|
||||
} finally {
|
||||
await cleanupAgent(page, agentId);
|
||||
}
|
||||
});
|
||||
|
||||
test('submits a mixed batch once and preserves decisions through collapse', async ({ page }) => {
|
||||
test.setTimeout(120000);
|
||||
const label = uniqueLabel();
|
||||
const firstCallId = `call_e2e_approval_${label}_first`;
|
||||
const secondCallId = `call_e2e_approval_${label}_second`;
|
||||
const responseText = `manual batch result ${label}`;
|
||||
let agentId: string | undefined;
|
||||
|
||||
try {
|
||||
agentId = await createAndSelectApprovalAgent(page);
|
||||
await startApproval(page, label, BATCH_APPROVAL_PROMPT_MARKER);
|
||||
await expect(approvalCards(page)).toHaveCount(2);
|
||||
|
||||
const firstCard = approvalCard(page, firstCallId);
|
||||
const secondCard = approvalCard(page, secondCallId);
|
||||
const submit = messagesView(page).getByRole('button', {
|
||||
name: 'Submit 2 decisions',
|
||||
exact: true,
|
||||
});
|
||||
|
||||
await secondCard.getByRole('button', { name: 'Respond' }).click();
|
||||
await secondCard.getByRole('textbox', { name: 'Respond' }).fill(responseText);
|
||||
await expect(submit).toBeDisabled();
|
||||
await firstCard.getByRole('button', { name: 'Approve' }).click();
|
||||
await expect(submit).toBeEnabled();
|
||||
|
||||
const groupToggle = messagesView(page).getByRole('button', {
|
||||
name: 'Used 2 tools',
|
||||
exact: true,
|
||||
});
|
||||
await groupToggle.click();
|
||||
await expect(groupToggle).toHaveAttribute('aria-expanded', 'false');
|
||||
await page.waitForTimeout(400);
|
||||
await groupToggle.click();
|
||||
await expect(groupToggle).toHaveAttribute('aria-expanded', 'true');
|
||||
|
||||
const reopenedFirstCard = approvalCard(page, firstCallId);
|
||||
const reopenedSecondCard = approvalCard(page, secondCallId);
|
||||
await expect(reopenedFirstCard.getByRole('button', { name: 'Approve' })).toHaveAttribute(
|
||||
'aria-pressed',
|
||||
'true',
|
||||
);
|
||||
await expect(reopenedSecondCard.getByRole('button', { name: 'Respond' })).toHaveAttribute(
|
||||
'aria-pressed',
|
||||
'true',
|
||||
);
|
||||
await expect(reopenedSecondCard.getByRole('textbox', { name: 'Respond' })).toHaveValue(
|
||||
responseText,
|
||||
);
|
||||
await expect(submit).toBeEnabled();
|
||||
|
||||
const { body } = await submitAndCapture(page, submit);
|
||||
expect(body.decisions).toHaveLength(2);
|
||||
expect(body.decisions).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
decision: 'approve',
|
||||
tool_call_id: firstCallId,
|
||||
}),
|
||||
expect.objectContaining({
|
||||
decision: 'respond',
|
||||
responseText,
|
||||
tool_call_id: secondCallId,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
await expect(
|
||||
messagesView(page).getByText(`E2E approval probe executed: first-${label}`),
|
||||
).toBeVisible({ timeout: 30000 });
|
||||
await expect(messagesView(page).getByText(responseText)).toBeVisible({ timeout: 30000 });
|
||||
await expect(
|
||||
messagesView(page).getByText(`E2E approval probe executed: second-${label}`),
|
||||
).toHaveCount(0);
|
||||
await expect(approvalCards(page)).toHaveCount(0);
|
||||
} finally {
|
||||
await cleanupAgent(page, agentId);
|
||||
}
|
||||
});
|
||||
|
||||
test('rehydrates a paused approval and its completed result across reloads', async ({ page }) => {
|
||||
test.setTimeout(120000);
|
||||
const label = uniqueLabel();
|
||||
const toolCallId = `call_e2e_approval_${label}`;
|
||||
const executedText = `E2E approval probe executed: original-${label}`;
|
||||
let agentId: string | undefined;
|
||||
|
||||
try {
|
||||
agentId = await createAndSelectApprovalAgent(page);
|
||||
await startApproval(page, label);
|
||||
const conversationPath = new URL(page.url()).pathname;
|
||||
|
||||
await page.reload({ waitUntil: 'domcontentloaded' });
|
||||
await expect.poll(() => new URL(page.url()).pathname).toBe(conversationPath);
|
||||
const rehydratedCard = approvalCard(page, toolCallId);
|
||||
await expect(rehydratedCard).toBeVisible({ timeout: 30000 });
|
||||
await expect(rehydratedCard).toContainText(APPROVAL_REASON);
|
||||
|
||||
await page.goto(NEW_CHAT_PATH, { waitUntil: 'domcontentloaded' });
|
||||
await expect(approvalCards(page)).toHaveCount(0);
|
||||
await page.goto(conversationPath, { waitUntil: 'domcontentloaded' });
|
||||
const navigatedCard = approvalCard(page, toolCallId);
|
||||
await expect(navigatedCard).toBeVisible({ timeout: 30000 });
|
||||
await expect(navigatedCard).toContainText(APPROVAL_REASON);
|
||||
|
||||
await navigatedCard.getByRole('button', { name: 'Approve' }).click();
|
||||
await submitAndCapture(page, navigatedCard.getByRole('button', { name: 'Submit' }));
|
||||
await expect(messagesView(page).getByText(executedText)).toBeVisible({ timeout: 30000 });
|
||||
await expect(approvalCards(page)).toHaveCount(0);
|
||||
|
||||
await page.reload({ waitUntil: 'domcontentloaded' });
|
||||
await expect.poll(() => new URL(page.url()).pathname).toBe(conversationPath);
|
||||
await expect(messagesView(page).getByText(executedText)).toBeVisible({ timeout: 30000 });
|
||||
await expect(approvalCards(page)).toHaveCount(0);
|
||||
} finally {
|
||||
await cleanupAgent(page, agentId);
|
||||
}
|
||||
});
|
||||
|
||||
test('sends only one resume request for two synchronous submit clicks', async ({ page }) => {
|
||||
test.setTimeout(120000);
|
||||
const label = uniqueLabel();
|
||||
const executedText = `E2E approval probe executed: original-${label}`;
|
||||
let agentId: string | undefined;
|
||||
let releaseResume = () => undefined;
|
||||
let resumeHandler: ((route: Route) => Promise<void>) | undefined;
|
||||
|
||||
try {
|
||||
agentId = await createAndSelectApprovalAgent(page);
|
||||
const card = await startApproval(page, label);
|
||||
const submit = card.getByRole('button', { name: 'Submit' });
|
||||
await card.getByRole('button', { name: 'Approve' }).click();
|
||||
await expect(submit).toBeEnabled();
|
||||
|
||||
let resumeRequests = 0;
|
||||
const resumeGate = new Promise<void>((resolve) => {
|
||||
releaseResume = resolve;
|
||||
});
|
||||
resumeHandler = async (route) => {
|
||||
resumeRequests++;
|
||||
if (resumeRequests === 1) {
|
||||
await resumeGate;
|
||||
await route.continue();
|
||||
return;
|
||||
}
|
||||
await route.fulfill({
|
||||
status: 409,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ message: 'duplicate resume request' }),
|
||||
});
|
||||
};
|
||||
await page.route('**/api/agents/chat/resume', resumeHandler);
|
||||
|
||||
await submit.evaluate((button: HTMLButtonElement) => {
|
||||
button.click();
|
||||
button.click();
|
||||
});
|
||||
await page.waitForTimeout(250);
|
||||
expect(resumeRequests).toBe(1);
|
||||
await expect(card.getByRole('button', { name: 'Submitting' })).toBeDisabled();
|
||||
await expect(card.getByRole('button', { name: 'Approve' })).toBeDisabled();
|
||||
releaseResume();
|
||||
|
||||
await expect(messagesView(page).getByText(executedText)).toBeVisible({ timeout: 30000 });
|
||||
await expect(approvalCards(page)).toHaveCount(0);
|
||||
} finally {
|
||||
releaseResume();
|
||||
if (resumeHandler) {
|
||||
await page.unroute('**/api/agents/chat/resume', resumeHandler);
|
||||
}
|
||||
await cleanupAgent(page, agentId);
|
||||
}
|
||||
});
|
||||
|
||||
test('preserves a decision after a transient resume error and retries successfully', async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(120000);
|
||||
const label = uniqueLabel();
|
||||
const responseText = `retry response ${label}`;
|
||||
let agentId: string | undefined;
|
||||
let resumeHandler: ((route: Route) => Promise<void>) | undefined;
|
||||
|
||||
try {
|
||||
agentId = await createAndSelectApprovalAgent(page);
|
||||
const card = await startApproval(page, label);
|
||||
const submit = card.getByRole('button', { name: 'Submit' });
|
||||
await card.getByRole('button', { name: 'Respond' }).click();
|
||||
const responseInput = card.getByRole('textbox', { name: 'Respond' });
|
||||
await responseInput.fill(responseText);
|
||||
|
||||
let resumeRequests = 0;
|
||||
resumeHandler = async (route) => {
|
||||
resumeRequests++;
|
||||
if (resumeRequests === 1) {
|
||||
await route.fulfill({
|
||||
status: 500,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ message: 'temporary e2e failure' }),
|
||||
});
|
||||
return;
|
||||
}
|
||||
await route.continue();
|
||||
};
|
||||
await page.route('**/api/agents/chat/resume', resumeHandler);
|
||||
|
||||
await Promise.all([
|
||||
page.waitForResponse(
|
||||
(response) => isResumeRequest(response.request()) && response.status() === 500,
|
||||
),
|
||||
submit.click(),
|
||||
]);
|
||||
await expect(card.getByText(APPROVAL_ERROR, { exact: true })).toBeVisible();
|
||||
await expect(card.getByRole('button', { name: 'Respond' })).toHaveAttribute(
|
||||
'aria-pressed',
|
||||
'true',
|
||||
);
|
||||
await expect(responseInput).toHaveValue(responseText);
|
||||
await expect(submit).toBeEnabled();
|
||||
|
||||
await Promise.all([
|
||||
page.waitForResponse(
|
||||
(response) => isResumeRequest(response.request()) && response.status() === 200,
|
||||
),
|
||||
submit.click(),
|
||||
]);
|
||||
await expect(messagesView(page).getByText(responseText)).toBeVisible({ timeout: 30000 });
|
||||
await expect(
|
||||
messagesView(page).getByText(`E2E approval probe executed: original-${label}`),
|
||||
).toHaveCount(0);
|
||||
expect(resumeRequests).toBe(2);
|
||||
await expect(approvalCards(page)).toHaveCount(0);
|
||||
} finally {
|
||||
if (resumeHandler) {
|
||||
await page.unroute('**/api/agents/chat/resume', resumeHandler);
|
||||
}
|
||||
await cleanupAgent(page, agentId);
|
||||
}
|
||||
});
|
||||
|
||||
test('locks the approval controls and explains an expired resume action', async ({ page }) => {
|
||||
test.setTimeout(120000);
|
||||
const label = uniqueLabel();
|
||||
const executedText = `E2E approval probe executed: original-${label}`;
|
||||
let agentId: string | undefined;
|
||||
let capturedResumeBody: Record<string, unknown> | undefined;
|
||||
let backendResolved = false;
|
||||
let routeInstalled = false;
|
||||
const resumeHandler = async (route: Route) => {
|
||||
capturedResumeBody = route.request().postDataJSON() as Record<string, unknown>;
|
||||
await route.fulfill({
|
||||
status: 409,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ message: 'expired e2e action' }),
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
agentId = await createAndSelectApprovalAgent(page);
|
||||
const card = await startApproval(page, label);
|
||||
const approve = card.getByRole('button', { name: 'Approve' });
|
||||
const submit = card.getByRole('button', { name: 'Submit' });
|
||||
await approve.click();
|
||||
await page.route('**/api/agents/chat/resume', resumeHandler);
|
||||
routeInstalled = true;
|
||||
|
||||
await Promise.all([
|
||||
page.waitForResponse(
|
||||
(response) => isResumeRequest(response.request()) && response.status() === 409,
|
||||
),
|
||||
submit.click(),
|
||||
]);
|
||||
await expect(card.getByText(APPROVAL_EXPIRED, { exact: true })).toBeVisible();
|
||||
await expect(approve).toHaveAttribute('aria-pressed', 'true');
|
||||
await expect(approve).toBeDisabled();
|
||||
await expect(card.getByRole('button', { name: 'Reject' })).toBeDisabled();
|
||||
await expect(card.getByRole('button', { name: 'Edit' })).toBeDisabled();
|
||||
await expect(card.getByRole('button', { name: 'Respond' })).toBeDisabled();
|
||||
await expect(submit).toBeDisabled();
|
||||
expect(capturedResumeBody).toBeDefined();
|
||||
|
||||
await page.unroute('**/api/agents/chat/resume', resumeHandler);
|
||||
routeInstalled = false;
|
||||
const token = await getAccessToken(page);
|
||||
await requestJson(page, {
|
||||
path: '/api/agents/chat/resume',
|
||||
token,
|
||||
method: 'POST',
|
||||
body: capturedResumeBody,
|
||||
});
|
||||
backendResolved = true;
|
||||
await expect(messagesView(page).getByText(executedText)).toBeVisible({ timeout: 30000 });
|
||||
await expect(approvalCards(page)).toHaveCount(0);
|
||||
} finally {
|
||||
if (routeInstalled) {
|
||||
await page.unroute('**/api/agents/chat/resume', resumeHandler);
|
||||
}
|
||||
if (!backendResolved && capturedResumeBody) {
|
||||
const token = await getAccessToken(page);
|
||||
await requestJson(page, {
|
||||
path: '/api/agents/chat/resume',
|
||||
token,
|
||||
method: 'POST',
|
||||
body: capturedResumeBody,
|
||||
}).catch(() => undefined);
|
||||
}
|
||||
await cleanupAgent(page, agentId);
|
||||
}
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue