mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-08-27 12:13:30 +00:00
🎬 test: Cover Detached Subagent Activity Lifecycle (#15117)
* test: cover detached subagent activity lifecycle * test: strengthen detached activity lifecycle gates * test: tighten detached activity assertions
This commit is contained in:
parent
1de88e7e91
commit
caa938fec6
3 changed files with 247 additions and 0 deletions
|
|
@ -21,6 +21,7 @@ export default defineConfig({
|
|||
/steering\.spec\.ts/,
|
||||
/steering-escalation\.spec\.ts/,
|
||||
/streaming\.spec\.ts/,
|
||||
/subagent-activity\.spec\.ts/,
|
||||
/thread-fold\.spec\.ts/,
|
||||
/tool-approvals\.spec\.ts/,
|
||||
/usage\.spec\.ts/,
|
||||
|
|
|
|||
|
|
@ -55,6 +55,8 @@ const DEFERRED_HITL_MARKER = 'E2E_DEFERRED_HITL:';
|
|||
const HANDOFF_MARKER = 'E2E_HANDOFF:';
|
||||
const SUBAGENT_RESULT_MARKER = 'E2E_SUBAGENT_RESULT:';
|
||||
const SUBAGENT_CHILD_MARKER = 'E2E_SUBAGENT_CHILD:';
|
||||
const SUBAGENT_ACTIVITY_MARKER = 'E2E_SUBAGENT_ACTIVITY:';
|
||||
const SUBAGENT_ACTIVITY_CHILD_MARKER = 'E2E_SUBAGENT_ACTIVITY_CHILD:';
|
||||
const SUBAGENT_MODEL_OVERRIDE_ERROR =
|
||||
'[e2e] Streamed subagent result coverage requires an @librechat/agents release with ' +
|
||||
'StandardGraph.setSubagentModelOverride';
|
||||
|
|
@ -685,10 +687,21 @@ function overrideModel({
|
|||
toolCalls,
|
||||
thrownError,
|
||||
overrideSubagentModel,
|
||||
disableHumanInTheLoop,
|
||||
resolveInvocation,
|
||||
resolveOnStream,
|
||||
modelCallbacks,
|
||||
}) {
|
||||
/** The shared mock profile enables approval HITL for its dedicated specs.
|
||||
* Detached subagents reject that run-level mode before executing, so the
|
||||
* credential-free activity scenario explicitly models a deployment with
|
||||
* approval HITL disabled without weakening the shared profile. */
|
||||
if (disableHumanInTheLoop) {
|
||||
graph.humanInTheLoop = undefined;
|
||||
for (const executor of graph._subagentExecutors ?? []) {
|
||||
executor.humanInTheLoop = undefined;
|
||||
}
|
||||
}
|
||||
if (overrideSubagentModel && typeof graph.setSubagentModelOverride !== 'function') {
|
||||
overrideModel({
|
||||
graph,
|
||||
|
|
@ -1364,6 +1377,79 @@ function subagentResultResponses(text) {
|
|||
};
|
||||
}
|
||||
|
||||
function parseSubagentActivityMarker(text) {
|
||||
const value = getMarkerValue(text, SUBAGENT_ACTIVITY_MARKER);
|
||||
const separator = value.indexOf(':');
|
||||
if (separator <= 0 || separator === value.length - 1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const childIds = value.slice(0, separator).split(',').filter(Boolean);
|
||||
if (childIds.length !== 2) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
childIds,
|
||||
label: value.slice(separator + 1),
|
||||
};
|
||||
}
|
||||
|
||||
function subagentActivityResponses(text) {
|
||||
const marker = parseSubagentActivityMarker(text);
|
||||
if (!marker) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
responses: [''],
|
||||
sleep: 50,
|
||||
overrideSubagentModel: true,
|
||||
disableHumanInTheLoop: true,
|
||||
resolveInvocation: (messages) => {
|
||||
const latestUserText = getLatestUserText(messages);
|
||||
for (const [index] of marker.childIds.entries()) {
|
||||
const childPrompt = `${SUBAGENT_ACTIVITY_CHILD_MARKER}${marker.label}:${index + 1}`;
|
||||
if (!latestUserText.includes(childPrompt)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const progress = Array.from(
|
||||
{ length: 100 },
|
||||
(_, phase) => `child-${index + 1}-phase-${phase + 1}`,
|
||||
).join(' ');
|
||||
return {
|
||||
response: `E2E detached child ${index + 1} activity ${marker.label} ${progress} E2E detached child ${index + 1} complete ${marker.label}`,
|
||||
};
|
||||
}
|
||||
|
||||
const backgroundTaskResults = (messages ?? []).filter(
|
||||
(message) =>
|
||||
messageType(message) === 'tool' &&
|
||||
typeof message?.tool_call_id === 'string' &&
|
||||
message.tool_call_id.startsWith('call_e2e_subagent_activity_'),
|
||||
);
|
||||
if (backgroundTaskResults.length >= marker.childIds.length) {
|
||||
return { response: `E2E detached subagents dispatched ${marker.label}` };
|
||||
}
|
||||
|
||||
return {
|
||||
response: '',
|
||||
toolCalls: marker.childIds.map((childId, index) => ({
|
||||
id: `call_e2e_subagent_activity_${marker.label}_${index + 1}`,
|
||||
name: 'subagent',
|
||||
args: {
|
||||
description: `${SUBAGENT_ACTIVITY_CHILD_MARKER}${marker.label}:${index + 1}`,
|
||||
subagent_type: childId,
|
||||
run_in_background: true,
|
||||
},
|
||||
type: 'tool_call',
|
||||
})),
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function approvalToolResponses(label, toolNames, review) {
|
||||
if (!toolNames.has(APPROVAL_TOOL_NAME)) {
|
||||
return {
|
||||
|
|
@ -2129,6 +2215,11 @@ function buildHandoffResponses(graph, parsed) {
|
|||
}
|
||||
|
||||
function resolveResponses({ graph, messages, text, toolNames }) {
|
||||
const subagentActivity = subagentActivityResponses(text);
|
||||
if (subagentActivity) {
|
||||
return subagentActivity;
|
||||
}
|
||||
|
||||
const subagentResult = subagentResultResponses(text);
|
||||
if (subagentResult) {
|
||||
return subagentResult;
|
||||
|
|
@ -2310,6 +2401,7 @@ module.exports = function fakeModelHook(run, context) {
|
|||
toolCalls,
|
||||
thrownError,
|
||||
overrideSubagentModel,
|
||||
disableHumanInTheLoop,
|
||||
resolveInvocation,
|
||||
resolveOnStream,
|
||||
} = handoffScript
|
||||
|
|
@ -2327,6 +2419,7 @@ module.exports = function fakeModelHook(run, context) {
|
|||
toolCalls,
|
||||
thrownError,
|
||||
overrideSubagentModel,
|
||||
disableHumanInTheLoop,
|
||||
resolveInvocation: async (streamMessages, streamOptions, runManager) =>
|
||||
deferredHitlInvocationResponse({
|
||||
graph,
|
||||
|
|
|
|||
153
e2e/specs/mock/subagent-activity.spec.ts
Normal file
153
e2e/specs/mock/subagent-activity.spec.ts
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import type { Page } from '@playwright/test';
|
||||
import type { AgentDetail } from './agents.helpers';
|
||||
import { cleanupAgent, openAgentBuilder, uniqueAgentName } from './agents.helpers';
|
||||
import {
|
||||
MOCK_ENDPOINTS,
|
||||
getAccessToken,
|
||||
requestJson,
|
||||
sendMessageAndWaitForCompletion,
|
||||
} from './helpers';
|
||||
|
||||
const DETACHED_ACTIVITY_MARKER = 'E2E_SUBAGENT_ACTIVITY:';
|
||||
/** The activity hook's first reconnect is scheduled after 500 ms. */
|
||||
const ACTIVITY_RECONNECT_GUARD_MS = 1_000;
|
||||
const ACTIVITY_PATH = /\/api\/convos\/[^/]+\/subagents\/[^/]+\/tasks\/[^/]+\/activity$/;
|
||||
|
||||
async function createAgent(
|
||||
page: Page,
|
||||
token: string,
|
||||
name: string,
|
||||
subagents?: AgentDetail['subagents'],
|
||||
): Promise<AgentDetail> {
|
||||
return requestJson<AgentDetail>(page, {
|
||||
path: '/api/agents',
|
||||
token,
|
||||
method: 'POST',
|
||||
body: {
|
||||
name,
|
||||
description: 'Playwright verification of detached child activity.',
|
||||
instructions: 'Follow the deterministic end-to-end request exactly.',
|
||||
provider: MOCK_ENDPOINTS[0].label,
|
||||
model: MOCK_ENDPOINTS[0].model,
|
||||
subagents,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function selectAgent(page: Page, name: string): Promise<void> {
|
||||
const form = await openAgentBuilder(page);
|
||||
await form.getByRole('combobox', { name: 'Agent', exact: true }).click();
|
||||
await page.getByRole('option', { name }).click();
|
||||
await expect(form.getByLabel('Agent name')).toHaveValue(name);
|
||||
await form.getByRole('button', { name: 'Select Agent' }).click();
|
||||
}
|
||||
|
||||
test.describe('detached subagent activity', () => {
|
||||
test('streams two child runs into the shared panel and restores terminal activity', async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(120_000);
|
||||
const label = `activity-${Date.now().toString(36)}`;
|
||||
const childNames = [
|
||||
uniqueAgentName('E2E Activity Child A'),
|
||||
uniqueAgentName('E2E Activity Child B'),
|
||||
];
|
||||
const parentName = uniqueAgentName('E2E Activity Parent');
|
||||
const createdAgentIds: string[] = [];
|
||||
const activityRequests: string[] = [];
|
||||
const finishedActivityRequests: string[] = [];
|
||||
page.on('request', (request) => {
|
||||
const url = new URL(request.url());
|
||||
if (ACTIVITY_PATH.test(url.pathname)) {
|
||||
activityRequests.push(url.pathname);
|
||||
}
|
||||
});
|
||||
page.on('requestfinished', (request) => {
|
||||
const url = new URL(request.url());
|
||||
if (ACTIVITY_PATH.test(url.pathname)) {
|
||||
finishedActivityRequests.push(url.pathname);
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
await page.goto('/c/new');
|
||||
const token = await getAccessToken(page);
|
||||
const children: AgentDetail[] = [];
|
||||
for (const childName of childNames) {
|
||||
const child = await createAgent(page, token, childName);
|
||||
children.push(child);
|
||||
createdAgentIds.push(child.id);
|
||||
}
|
||||
const parent = await createAgent(page, token, parentName, {
|
||||
enabled: true,
|
||||
allowSelf: false,
|
||||
agent_ids: children.map((child) => child.id),
|
||||
});
|
||||
createdAgentIds.push(parent.id);
|
||||
|
||||
await selectAgent(page, parentName);
|
||||
const response = await sendMessageAndWaitForCompletion(
|
||||
page,
|
||||
`${DETACHED_ACTIVITY_MARKER}${children.map((child) => child.id).join(',')}:${label}`,
|
||||
);
|
||||
expect(response.ok()).toBeTruthy();
|
||||
|
||||
await page.getByRole('button', { name: 'Ran 2 agents' }).click();
|
||||
const cards = page.locator('[data-subagent-tool-call^="call_e2e_subagent_activity_"]');
|
||||
await expect(cards).toHaveCount(2, { timeout: 30_000 });
|
||||
await expect(cards.first()).toHaveAttribute('data-subagent-thread', /.+/);
|
||||
const activityResponsePromise = page.waitForResponse((candidate) => {
|
||||
const url = new URL(candidate.url());
|
||||
return ACTIVITY_PATH.test(url.pathname);
|
||||
});
|
||||
await cards.first().click();
|
||||
|
||||
const panel = page.getByRole('region', { name: 'Child agent activity' });
|
||||
const activityResponse = await activityResponsePromise;
|
||||
await expect(panel).toBeVisible();
|
||||
await expect(panel.getByText('Running', { exact: true })).toBeVisible();
|
||||
await expect(panel.getByText('Writing', { exact: true })).toBeVisible();
|
||||
await expect(panel).toContainText('child-1-phase-10');
|
||||
await expect.poll(() => activityRequests.length).toBe(1);
|
||||
|
||||
await expect(panel.getByText('Completed', { exact: true })).toBeVisible({ timeout: 30_000 });
|
||||
await expect(panel).toContainText(`E2E detached child 1 complete ${label}`);
|
||||
await expect.poll(() => finishedActivityRequests.length).toBe(1);
|
||||
const activityStreamBody = await activityResponse.text();
|
||||
expect(activityStreamBody).toContain('"event":"on_subagent_update"');
|
||||
expect(activityStreamBody).toContain('"phase":"message_delta"');
|
||||
await page.waitForTimeout(ACTIVITY_RECONNECT_GUARD_MS);
|
||||
expect(activityRequests).toHaveLength(1);
|
||||
|
||||
await panel.getByRole('button', { name: 'Close' }).click();
|
||||
await expect(panel).not.toBeVisible();
|
||||
await cards.nth(1).click();
|
||||
await expect(panel.getByText('Completed', { exact: true })).toBeVisible({ timeout: 30_000 });
|
||||
await expect(panel).toContainText(`E2E detached child 2 complete ${label}`);
|
||||
|
||||
await panel.getByRole('button', { name: 'Close' }).click();
|
||||
await page.reload();
|
||||
await page.getByRole('button', { name: 'Ran 2 agents' }).click();
|
||||
const restoredCards = page.locator(
|
||||
'[data-subagent-tool-call^="call_e2e_subagent_activity_"]',
|
||||
);
|
||||
await expect(restoredCards).toHaveCount(2);
|
||||
await restoredCards.first().click();
|
||||
await expect(panel.getByText('Completed', { exact: true })).toBeVisible();
|
||||
await expect(panel).toContainText(`E2E detached child 1 complete ${label}`);
|
||||
|
||||
await panel.getByRole('button', { name: 'Close' }).click();
|
||||
await page.getByRole('button', { name: 'Chat History' }).click();
|
||||
const conversationRows = page.getByTestId('convo-item');
|
||||
await expect(conversationRows.locator('button[aria-current="page"]')).toBeVisible();
|
||||
for (const child of children) {
|
||||
await expect(conversationRows.filter({ hasText: `Subagent: ${child.id}` })).toHaveCount(0);
|
||||
}
|
||||
} finally {
|
||||
for (const agentId of createdAgentIds.reverse()) {
|
||||
await cleanupAgent(page, agentId);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue