diff --git a/api/package.json b/api/package.json index 00b164646b..f25158853c 100644 --- a/api/package.json +++ b/api/package.json @@ -46,7 +46,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.8.0", "@keyv/redis": "^4.3.3", - "@librechat/agents": "^3.3.13", + "@librechat/agents": "^3.4.0", "@librechat/api": "*", "@librechat/data-schemas": "*", "@microsoft/microsoft-graph-client": "^3.0.7", diff --git a/e2e/config/librechat.real.yaml b/e2e/config/librechat.real.yaml index 3263215b06..32797e86a0 100644 --- a/e2e/config/librechat.real.yaml +++ b/e2e/config/librechat.real.yaml @@ -33,3 +33,4 @@ endpoints: - actions - tools - tool_intents + - subagents diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.js index baade45c28..f6dc367f26 100644 --- a/e2e/setup/fake-model.js +++ b/e2e/setup/fake-model.js @@ -45,6 +45,11 @@ const TOOL_APPROVAL_RESTRICTED_MARKER = 'E2E_TOOL_APPROVAL_RESTRICTED:'; const TOOL_APPROVAL_REWRITE_MARKER = 'E2E_TOOL_APPROVAL_REWRITE:'; const DEFERRED_HITL_MARKER = 'E2E_DEFERRED_HITL:'; const HANDOFF_MARKER = 'E2E_HANDOFF:'; +const SUBAGENT_RESULT_MARKER = 'E2E_SUBAGENT_RESULT:'; +const SUBAGENT_CHILD_MARKER = 'E2E_SUBAGENT_CHILD:'; +const SUBAGENT_MODEL_OVERRIDE_ERROR = + '[e2e] Streamed subagent result coverage requires an @librechat/agents release with ' + + 'StandardGraph.setSubagentModelOverride'; const HANDOFF_TOOL_PREFIX = 'lc_transfer_to_'; const CREATE_FILE_AUTHORING_FINAL_TEXT = 'E2E file authoring complete'; const EDIT_FILE_AUTHORING_FINAL_TEXT = 'E2E file edit complete'; @@ -482,7 +487,7 @@ class UsageEmittingFakeChatModel extends FakeChatModel { this.streamSleep = sleep ?? CHUNK_DELAY_MS; } - async *streamScriptedResponseChunks({ response, toolCalls, runManager }) { + async *streamScriptedResponseChunks({ response, toolCalls, textDeltaBlocks, runManager }) { if (this.emitCustomEvent) { await runManager?.handleCustomEvent('some_test_event', { someval: true, @@ -492,7 +497,14 @@ class UsageEmittingFakeChatModel extends FakeChatModel { const chunks = response ? response.split(/(?<=\s+)|(?=\s+)/) : []; for await (const chunk of chunks) { await new Promise((resolve) => setTimeout(resolve, this.streamSleep)); - const responseChunk = this._createResponseChunk(chunk); + const responseChunk = textDeltaBlocks + ? new ChatGenerationChunk({ + text: chunk, + message: new AIMessageChunk({ + content: [{ type: 'text_delta', index: 0, text: chunk }], + }), + }) + : this._createResponseChunk(chunk); yield responseChunk; void runManager?.handleLLMNewToken(chunk); } @@ -544,6 +556,7 @@ class UsageEmittingFakeChatModel extends FakeChatModel { chunkStream = this.streamScriptedResponseChunks({ response: scriptedResponse.response ?? '', toolCalls: scriptedResponse.toolCalls, + textDeltaBlocks: scriptedResponse.textDeltaBlocks === true, runManager, }); } else if (dynamicResponse) { @@ -582,11 +595,22 @@ function overrideModel({ sleep, toolCalls, thrownError, + overrideSubagentModel, resolveInvocation, resolveOnStream, }) { + if (overrideSubagentModel && typeof graph.setSubagentModelOverride !== 'function') { + overrideModel({ + graph, + responses: [''], + sleep, + thrownError: SUBAGENT_MODEL_OVERRIDE_ERROR, + }); + return; + } + if (!thrownError) { - graph.overrideModel = new UsageEmittingFakeChatModel({ + const model = new UsageEmittingFakeChatModel({ responses, sleep: sleep ?? CHUNK_DELAY_MS, emitCustomEvent: true, @@ -594,6 +618,10 @@ function overrideModel({ resolveInvocation, resolveOnStream, }); + graph.overrideModel = model; + if (overrideSubagentModel) { + graph.setSubagentModelOverride(model); + } return; } @@ -607,12 +635,13 @@ function overrideModel({ } } - graph.overrideModel = new ThrowingFakeChatModel({ + const model = new ThrowingFakeChatModel({ responses, sleep: sleep ?? CHUNK_DELAY_MS, emitCustomEvent: true, toolCalls, }); + graph.overrideModel = model; } function parseSkillAssertion(text, agentId) { @@ -1099,6 +1128,57 @@ function findLastToolMessageText(messages, requiredToken) { return ''; } +function parseSubagentResultMarker(text) { + const value = getMarkerValue(text, SUBAGENT_RESULT_MARKER); + const separator = value.indexOf(':'); + if (separator <= 0 || separator === value.length - 1) { + return null; + } + return { + childId: value.slice(0, separator), + label: value.slice(separator + 1), + }; +} + +function subagentResultResponses(text) { + const marker = parseSubagentResultMarker(text); + if (!marker) { + return null; + } + + const childPrompt = `${SUBAGENT_CHILD_MARKER}${marker.label}`; + const expectedResult = `E2E subagent streamed result ${marker.label}`; + return { + responses: [''], + overrideSubagentModel: true, + resolveInvocation: (messages) => { + const toolResult = findLastToolMessageText(messages, expectedResult); + if (toolResult) { + return { response: toolResult }; + } + + if (getLatestUserText(messages).includes(childPrompt)) { + return { response: expectedResult, textDeltaBlocks: true }; + } + + return { + response: '', + toolCalls: [ + { + id: `call_e2e_subagent_${marker.label}`, + name: 'subagent', + args: { + description: childPrompt, + subagent_type: marker.childId, + }, + type: 'tool_call', + }, + ], + }; + }, + }; +} + function approvalToolResponses(label, toolNames, review) { if (!toolNames.has(APPROVAL_TOOL_NAME)) { return { @@ -1859,6 +1939,11 @@ function buildHandoffResponses(graph, parsed) { } function resolveResponses({ graph, messages, text, toolNames }) { + const subagentResult = subagentResultResponses(text); + if (subagentResult) { + return subagentResult; + } + const batchApprovalLabel = getMarkerValue(text, TOOL_APPROVAL_BATCH_MARKER); if (batchApprovalLabel) { return batchApprovalToolResponses(batchApprovalLabel, toolNames); @@ -2019,21 +2104,29 @@ module.exports = function fakeModelHook(run, context) { const text = getLatestUserText(context?.messages); const toolNames = collectToolNames(context?.agents); const handoffScript = parseHandoffScript(text); - const { responses, sleep, toolCalls, thrownError, resolveInvocation, resolveOnStream } = - handoffScript - ? buildHandoffResponses(graph, handoffScript) - : resolveResponses({ - graph, - messages: context?.messages, - text, - toolNames, - }); + const { + responses, + sleep, + toolCalls, + thrownError, + overrideSubagentModel, + resolveInvocation, + resolveOnStream, + } = handoffScript + ? buildHandoffResponses(graph, handoffScript) + : resolveResponses({ + graph, + messages: context?.messages, + text, + toolNames, + }); overrideModel({ graph, responses, sleep, toolCalls, thrownError, + overrideSubagentModel, resolveInvocation: async (streamMessages, streamOptions, runManager) => deferredHitlInvocationResponse({ graph, diff --git a/e2e/specs/mock/agents.helpers.ts b/e2e/specs/mock/agents.helpers.ts index cdfded2f2e..191945730d 100644 --- a/e2e/specs/mock/agents.helpers.ts +++ b/e2e/specs/mock/agents.helpers.ts @@ -1,5 +1,5 @@ import { expect } from '@playwright/test'; -import type { GraphEdge } from 'librechat-data-provider'; +import type { AgentSubagentsConfig, GraphEdge } from 'librechat-data-provider'; import type { Page } from '@playwright/test'; import { MOCK_ENDPOINTS, NEW_CHAT_PATH, fetchJson, getAccessToken, requestJson } from './helpers'; @@ -35,6 +35,7 @@ export type AgentDetail = AgentSummary & { tools?: string[]; mcpServerNames?: string[]; edges?: GraphEdge[]; + subagents?: AgentSubagentsConfig; }; export const uniqueAgentName = (prefix: string) => diff --git a/e2e/specs/mock/subagent-results.spec.ts b/e2e/specs/mock/subagent-results.spec.ts new file mode 100644 index 0000000000..26d08b4380 --- /dev/null +++ b/e2e/specs/mock/subagent-results.spec.ts @@ -0,0 +1,79 @@ +import { expect, test } from '@playwright/test'; +import type { Page } from '@playwright/test'; +import type { AgentDetail } from './agents.helpers'; +import { cleanupAgent, openAgentBuilder, uniqueAgentName } from './agents.helpers'; +import { MOCK_ENDPOINTS, getAccessToken, messagesView, requestJson, sendMessage } from './helpers'; + +async function createAgent( + page: Page, + token: string, + name: string, + subagents?: AgentDetail['subagents'], +): Promise { + return requestJson(page, { + path: '/api/agents', + token, + method: 'POST', + body: { + name, + description: 'Playwright verification of isolated subagent result propagation.', + instructions: 'Follow the test request exactly.', + provider: MOCK_ENDPOINTS[0].label, + model: MOCK_ENDPOINTS[0].model, + subagents, + }, + }); +} + +async function selectAgent(page: Page, name: string): Promise { + const form = await openAgentBuilder(page); + await form.getByRole('combobox', { name: 'Agent', exact: true }).click(); + await page.getByRole('option', { name }).click(); + await expect(form.getByLabel('Agent name')).toHaveValue(name); + await form.getByRole('button', { name: 'Select Agent' }).click(); +} + +test.describe('isolated subagent results', () => { + test('renders the streamed child final answer instead of fallback or stale text', async ({ + page, + }) => { + test.setTimeout(120_000); + const label = `result-${Date.now()}`; + const childName = uniqueAgentName('E2E Child'); + const parentName = uniqueAgentName('E2E Parent'); + let childId: string | undefined; + let parentId: string | undefined; + + try { + await page.goto('/c/new'); + const token = await getAccessToken(page); + const child = await createAgent(page, token, childName); + childId = child.id; + const parent = await createAgent(page, token, parentName, { + enabled: true, + allowSelf: false, + agent_ids: [child.id], + }); + parentId = parent.id; + + await selectAgent(page, parentName); + const response = await sendMessage(page, `E2E_SUBAGENT_RESULT:${child.id}:${label}`); + expect(response.ok()).toBeTruthy(); + + const expected = `E2E subagent streamed result ${label}`; + await expect(page.getByRole('button', { name: 'Stop generating' })).toBeHidden({ + timeout: 60_000, + }); + const finalAnswer = messagesView(page) + .locator('.message-render') + .last() + .getByRole('paragraph') + .filter({ hasText: expected }); + await expect(finalAnswer).toHaveText(expected, { timeout: 30_000 }); + await expect(messagesView(page).getByText('Task completed', { exact: true })).toHaveCount(0); + } finally { + await cleanupAgent(page, parentId); + await cleanupAgent(page, childId); + } + }); +}); diff --git a/e2e/specs/real/subagent-results.spec.ts b/e2e/specs/real/subagent-results.spec.ts new file mode 100644 index 0000000000..aaad60c079 --- /dev/null +++ b/e2e/specs/real/subagent-results.spec.ts @@ -0,0 +1,127 @@ +import { expect, test } from '@playwright/test'; +import type { Page } from '@playwright/test'; +import type { AgentDetail } from '../mock/agents.helpers'; +import { cleanupAgent, openAgentBuilder, uniqueAgentName } from '../mock/agents.helpers'; +import { fetchJson, getAccessToken, messagesView, requestJson, sendMessage } from '../mock/helpers'; + +const REAL_MODEL = process.env.E2E_REAL_ANTHROPIC_MODEL ?? 'claude-haiku-4-5'; + +type PersistedMessage = { + content?: Array<{ + type?: string; + tool_call?: { name?: string }; + }>; +}; + +async function createAgent( + page: Page, + token: string, + body: { + name: string; + description: string; + instructions: string; + subagents?: AgentDetail['subagents']; + }, +): Promise { + return requestJson(page, { + path: '/api/agents', + token, + method: 'POST', + body: { + ...body, + provider: 'anthropic', + model: REAL_MODEL, + }, + }); +} + +async function selectAgent(page: Page, name: string): Promise { + const form = await openAgentBuilder(page); + await form.getByRole('combobox', { name: 'Agent', exact: true }).click(); + await page.getByRole('option', { name }).click(); + await expect(form.getByLabel('Agent name')).toHaveValue(name); + await form.getByRole('button', { name: 'Select Agent' }).click(); +} + +async function readMessages(page: Page, conversationId: string): Promise { + const token = await getAccessToken(page); + return fetchJson(page, `/api/messages/${conversationId}`, token); +} + +test.describe('isolated subagent results with a real provider', () => { + test('delegates and renders the child final answer', async ({ page }) => { + test.setTimeout(240_000); + const leftOperand = 48_723; + const rightOperand = 19_642; + const expectedResult = String(leftOperand + rightOperand); + const childName = uniqueAgentName('Real Child'); + const parentName = uniqueAgentName('Real Parent'); + let childId: string | undefined; + let parentId: string | undefined; + + try { + await page.goto('/c/new'); + const token = await getAccessToken(page); + const child = await createAgent(page, token, { + name: childName, + description: 'Solves arithmetic requests delegated by a parent agent.', + instructions: + 'You are an arithmetic specialist. Add the requested integers and reply with only the ' + + 'decimal result, without punctuation or explanation.', + }); + childId = child.id; + const parent = await createAgent(page, token, { + name: parentName, + description: 'Delegates arithmetic requests to one isolated child.', + instructions: + 'Always use the subagent tool for every user request. Never answer from your own ' + + 'knowledge. After the child finishes, return its answer verbatim with no added text.', + subagents: { + enabled: true, + allowSelf: false, + agent_ids: [child.id], + }, + }); + parentId = parent.id; + + await selectAgent(page, parentName); + const response = await sendMessage( + page, + `Ask the configured child to calculate ${leftOperand} + ${rightOperand}. Return only the ` + + 'integer it provides.', + ); + expect(response.ok()).toBeTruthy(); + await expect(page.getByRole('button', { name: 'Stop generating' })).toBeHidden({ + timeout: 180_000, + }); + const finalAnswer = messagesView(page) + .locator('.message-render') + .last() + .getByRole('paragraph') + .filter({ hasText: expectedResult }); + await expect(finalAnswer).toHaveText(expectedResult, { timeout: 30_000 }); + await expect(messagesView(page).getByText('Task completed', { exact: true })).toHaveCount(0); + + await expect(page).toHaveURL(/\/c\/(?!new)/, { timeout: 30_000 }); + const conversationId = new URL(page.url()).pathname.split('/c/')[1]; + expect(conversationId).toBeTruthy(); + let messages: PersistedMessage[] = []; + await expect + .poll( + async () => { + messages = await readMessages(page, conversationId); + return messages.some((message) => + (message.content ?? []).some( + (part) => part.type === 'tool_call' && part.tool_call?.name === 'subagent', + ), + ); + }, + { timeout: 30_000, intervals: [500, 1000, 2000] }, + ) + .toBe(true); + } finally { + await cleanupAgent(page, parentId); + await cleanupAgent(page, childId); + } + }); +}); diff --git a/package-lock.json b/package-lock.json index 110f552ca6..afa11a7b93 100644 --- a/package-lock.json +++ b/package-lock.json @@ -61,7 +61,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.8.0", "@keyv/redis": "^4.3.3", - "@librechat/agents": "^3.3.13", + "@librechat/agents": "^3.4.0", "@librechat/api": "*", "@librechat/data-schemas": "*", "@microsoft/microsoft-graph-client": "^3.0.7", @@ -10616,9 +10616,9 @@ } }, "node_modules/@librechat/agents": { - "version": "3.3.13", - "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-3.3.13.tgz", - "integrity": "sha512-NOaUCvMaq82ftzZr/Z/3KL1LzfKO0YRHibhFigcx22KxrmeioyJ32w4RnrBsYGhLx0gDrI4nWlW6Sk0ZAH/YBQ==", + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-3.4.0.tgz", + "integrity": "sha512-VKEckMv6sk40PIKQ/l3GXFugWJitZ8K4EqRu+ZJ9gpA8+3AI+FhWmenQh/n3prE9HpSq7PK9S4EngkNYPQTMlw==", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "^0.115.0", @@ -10653,7 +10653,7 @@ "nanoid": "^3.3.7", "okapibm25": "^1.4.1", "openai": "^6.46.0", - "reo-census": "^1.2.9", + "reo-census": "^1.2.10", "socks-proxy-agent": "^8.0.5", "uuid": "^11.1.1" }, @@ -37851,9 +37851,9 @@ "dev": true }, "node_modules/reo-census": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/reo-census/-/reo-census-1.2.9.tgz", - "integrity": "sha512-QPnAyRk6gUK9h2XJy9yTXiA91+LEHahmzbRdfb6DigAoe1S/nKg2LXc+/m3//HGyMPAkCFVz/FJJ8YrKm5IrXw==", + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/reo-census/-/reo-census-1.2.10.tgz", + "integrity": "sha512-Wr/cNvk1S2EMNUwLISWtO3VgSaoaKRsaAwc/t7TMPmSu1i4LhyS/mEHhQg9PhiK3bpAqSUqI5dY8riqAukXuyA==", "hasInstallScript": true, "license": "MIT", "engines": { @@ -42707,7 +42707,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.8.0", "@keyv/redis": "^4.3.3", - "@librechat/agents": "^3.3.13", + "@librechat/agents": "^3.4.0", "@librechat/data-schemas": "*", "@modelcontextprotocol/sdk": "^1.29.0", "@opentelemetry/api": "^1.9.0", diff --git a/packages/api/package.json b/packages/api/package.json index 32af7f7a35..6d933121fc 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -107,7 +107,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.8.0", "@keyv/redis": "^4.3.3", - "@librechat/agents": "^3.3.13", + "@librechat/agents": "^3.4.0", "@librechat/data-schemas": "*", "@modelcontextprotocol/sdk": "^1.29.0", "@opentelemetry/api": "^1.9.0",