CERBERUS/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2026-08-27 12:13:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

🛰️ chore: bump @librechat/agents to v3.4.0, cover streamed subagent results e2e (#14647)

Browse source 
* test: cover streamed subagent results end to end

* test: assert real e2e conversation id

* test: harden streamed subagent e2e

* test: stop incompatible subagent fixtures

* chore: update @librechat/agents to version 3.4.0 in package.json and package-lock.json
This commit is contained in:
Danny Avila 2026-08-05 22:25:03 -04:00 • committed by GitHub
parent 305107998d
commit 45cc53c40b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 326 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

2
api/package.json
View file

@ -46,7 +46,7 @@
"@azure/storage-blob": "^12.30.0",
"@google/genai": "^2.8.0",
"@keyv/redis": "^4.3.3",
"@librechat/agents": "^3.3.13",
"@librechat/agents": "^3.4.0",
"@librechat/api": "*",
"@librechat/data-schemas": "*",
"@microsoft/microsoft-graph-client": "^3.0.7",

1
e2e/config/librechat.real.yaml
View file

@ -33,3 +33,4 @@ endpoints:
- actions
- tools
- tool_intents
- subagents

119
e2e/setup/fake-model.js
View file

@ -45,6 +45,11 @@ const TOOL_APPROVAL_RESTRICTED_MARKER = 'E2E_TOOL_APPROVAL_RESTRICTED:';
const TOOL_APPROVAL_REWRITE_MARKER = 'E2E_TOOL_APPROVAL_REWRITE:';
const DEFERRED_HITL_MARKER = 'E2E_DEFERRED_HITL:';
const HANDOFF_MARKER = 'E2E_HANDOFF:';
const SUBAGENT_RESULT_MARKER = 'E2E_SUBAGENT_RESULT:';
const SUBAGENT_CHILD_MARKER = 'E2E_SUBAGENT_CHILD:';
const SUBAGENT_MODEL_OVERRIDE_ERROR =
'[e2e] Streamed subagent result coverage requires an @librechat/agents release with ' +
'StandardGraph.setSubagentModelOverride';
const HANDOFF_TOOL_PREFIX = 'lc_transfer_to_';
const CREATE_FILE_AUTHORING_FINAL_TEXT = 'E2E file authoring complete';
const EDIT_FILE_AUTHORING_FINAL_TEXT = 'E2E file edit complete';
@ -482,7 +487,7 @@ class UsageEmittingFakeChatModel extends FakeChatModel {
this.streamSleep = sleep ?? CHUNK_DELAY_MS;
}
async *streamScriptedResponseChunks({ response, toolCalls, runManager }) {
async *streamScriptedResponseChunks({ response, toolCalls, textDeltaBlocks, runManager }) {
if (this.emitCustomEvent) {
await runManager?.handleCustomEvent('some_test_event', {
someval: true,
@ -492,7 +497,14 @@ class UsageEmittingFakeChatModel extends FakeChatModel {
const chunks = response ? response.split(/(?<=\s+)|(?=\s+)/) : [];
for await (const chunk of chunks) {
await new Promise((resolve) => setTimeout(resolve, this.streamSleep));
const responseChunk = this._createResponseChunk(chunk);
const responseChunk = textDeltaBlocks
? new ChatGenerationChunk({
text: chunk,
message: new AIMessageChunk({
content: [{ type: 'text_delta', index: 0, text: chunk }],
}),
})
: this._createResponseChunk(chunk);
yield responseChunk;
void runManager?.handleLLMNewToken(chunk);
}
@ -544,6 +556,7 @@ class UsageEmittingFakeChatModel extends FakeChatModel {
chunkStream = this.streamScriptedResponseChunks({
response: scriptedResponse.response ?? '',
toolCalls: scriptedResponse.toolCalls,
textDeltaBlocks: scriptedResponse.textDeltaBlocks === true,
runManager,
});
} else if (dynamicResponse) {
@ -582,11 +595,22 @@ function overrideModel({
sleep,
toolCalls,
thrownError,
overrideSubagentModel,
resolveInvocation,
resolveOnStream,
}) {
if (overrideSubagentModel && typeof graph.setSubagentModelOverride !== 'function') {
overrideModel({
graph,
responses: [''],
sleep,
thrownError: SUBAGENT_MODEL_OVERRIDE_ERROR,
});
return;
}
if (!thrownError) {
graph.overrideModel = new UsageEmittingFakeChatModel({
const model = new UsageEmittingFakeChatModel({
responses,
sleep: sleep ?? CHUNK_DELAY_MS,
emitCustomEvent: true,
@ -594,6 +618,10 @@ function overrideModel({
resolveInvocation,
resolveOnStream,
});
graph.overrideModel = model;
if (overrideSubagentModel) {
graph.setSubagentModelOverride(model);
}
return;
}
@ -607,12 +635,13 @@ function overrideModel({
}
}
graph.overrideModel = new ThrowingFakeChatModel({
const model = new ThrowingFakeChatModel({
responses,
sleep: sleep ?? CHUNK_DELAY_MS,
emitCustomEvent: true,
toolCalls,
});
graph.overrideModel = model;
}
function parseSkillAssertion(text, agentId) {
@ -1099,6 +1128,57 @@ function findLastToolMessageText(messages, requiredToken) {
return '';
}
function parseSubagentResultMarker(text) {
const value = getMarkerValue(text, SUBAGENT_RESULT_MARKER);
const separator = value.indexOf(':');
if (separator <= 0 || separator === value.length - 1) {
return null;
}
return {
childId: value.slice(0, separator),
label: value.slice(separator + 1),
};
}
function subagentResultResponses(text) {
const marker = parseSubagentResultMarker(text);
if (!marker) {
return null;
}
const childPrompt = `${SUBAGENT_CHILD_MARKER}${marker.label}`;
const expectedResult = `E2E subagent streamed result ${marker.label}`;
return {
responses: [''],
overrideSubagentModel: true,
resolveInvocation: (messages) => {
const toolResult = findLastToolMessageText(messages, expectedResult);
if (toolResult) {
return { response: toolResult };
}
if (getLatestUserText(messages).includes(childPrompt)) {
return { response: expectedResult, textDeltaBlocks: true };
}
return {
response: '',
toolCalls: [
{
id: `call_e2e_subagent_${marker.label}`,
name: 'subagent',
args: {
description: childPrompt,
subagent_type: marker.childId,
},
type: 'tool_call',
},
],
};
},
};
}
function approvalToolResponses(label, toolNames, review) {
if (!toolNames.has(APPROVAL_TOOL_NAME)) {
return {
@ -1859,6 +1939,11 @@ function buildHandoffResponses(graph, parsed) {
}
function resolveResponses({ graph, messages, text, toolNames }) {
const subagentResult = subagentResultResponses(text);
if (subagentResult) {
return subagentResult;
}
const batchApprovalLabel = getMarkerValue(text, TOOL_APPROVAL_BATCH_MARKER);
if (batchApprovalLabel) {
return batchApprovalToolResponses(batchApprovalLabel, toolNames);
@ -2019,21 +2104,29 @@ module.exports = function fakeModelHook(run, context) {
const text = getLatestUserText(context?.messages);
const toolNames = collectToolNames(context?.agents);
const handoffScript = parseHandoffScript(text);
const { responses, sleep, toolCalls, thrownError, resolveInvocation, resolveOnStream } =
handoffScript
? buildHandoffResponses(graph, handoffScript)
: resolveResponses({
graph,
messages: context?.messages,
text,
toolNames,
});
const {
responses,
sleep,
toolCalls,
thrownError,
overrideSubagentModel,
resolveInvocation,
resolveOnStream,
} = handoffScript
? buildHandoffResponses(graph, handoffScript)
: resolveResponses({
graph,
messages: context?.messages,
text,
toolNames,
});
overrideModel({
graph,
responses,
sleep,
toolCalls,
thrownError,
overrideSubagentModel,
resolveInvocation: async (streamMessages, streamOptions, runManager) =>
deferredHitlInvocationResponse({
graph,

3
e2e/specs/mock/agents.helpers.ts
View file

@ -1,5 +1,5 @@
import { expect } from '@playwright/test';
import type { GraphEdge } from 'librechat-data-provider';
import type { AgentSubagentsConfig, GraphEdge } from 'librechat-data-provider';
import type { Page } from '@playwright/test';
import { MOCK_ENDPOINTS, NEW_CHAT_PATH, fetchJson, getAccessToken, requestJson } from './helpers';
@ -35,6 +35,7 @@ export type AgentDetail = AgentSummary & {
tools?: string[];
mcpServerNames?: string[];
edges?: GraphEdge[];
subagents?: AgentSubagentsConfig;
};
export const uniqueAgentName = (prefix: string) =>

79
e2e/specs/mock/subagent-results.spec.ts Normal file
View file

@ -0,0 +1,79 @@
import { expect, test } from '@playwright/test';
import type { Page } from '@playwright/test';
import type { AgentDetail } from './agents.helpers';
import { cleanupAgent, openAgentBuilder, uniqueAgentName } from './agents.helpers';
import { MOCK_ENDPOINTS, getAccessToken, messagesView, requestJson, sendMessage } from './helpers';
async function createAgent(
page: Page,
token: string,
name: string,
subagents?: AgentDetail['subagents'],
): Promise<AgentDetail> {
return requestJson<AgentDetail>(page, {
path: '/api/agents',
token,
method: 'POST',
body: {
name,
description: 'Playwright verification of isolated subagent result propagation.',
instructions: 'Follow the test request exactly.',
provider: MOCK_ENDPOINTS[0].label,
model: MOCK_ENDPOINTS[0].model,
subagents,
},
});
}
async function selectAgent(page: Page, name: string): Promise<void> {
const form = await openAgentBuilder(page);
await form.getByRole('combobox', { name: 'Agent', exact: true }).click();
await page.getByRole('option', { name }).click();
await expect(form.getByLabel('Agent name')).toHaveValue(name);
await form.getByRole('button', { name: 'Select Agent' }).click();
}
test.describe('isolated subagent results', () => {
test('renders the streamed child final answer instead of fallback or stale text', async ({
page,
}) => {
test.setTimeout(120_000);
const label = `result-${Date.now()}`;
const childName = uniqueAgentName('E2E Child');
const parentName = uniqueAgentName('E2E Parent');
let childId: string | undefined;
let parentId: string | undefined;
try {
await page.goto('/c/new');
const token = await getAccessToken(page);
const child = await createAgent(page, token, childName);
childId = child.id;
const parent = await createAgent(page, token, parentName, {
enabled: true,
allowSelf: false,
agent_ids: [child.id],
});
parentId = parent.id;
await selectAgent(page, parentName);
const response = await sendMessage(page, `E2E_SUBAGENT_RESULT:${child.id}:${label}`);
expect(response.ok()).toBeTruthy();
const expected = `E2E subagent streamed result ${label}`;
await expect(page.getByRole('button', { name: 'Stop generating' })).toBeHidden({
timeout: 60_000,
});
const finalAnswer = messagesView(page)
.locator('.message-render')
.last()
.getByRole('paragraph')
.filter({ hasText: expected });
await expect(finalAnswer).toHaveText(expected, { timeout: 30_000 });
await expect(messagesView(page).getByText('Task completed', { exact: true })).toHaveCount(0);
} finally {
await cleanupAgent(page, parentId);
await cleanupAgent(page, childId);
}
});
});

127
e2e/specs/real/subagent-results.spec.ts Normal file
View file

@ -0,0 +1,127 @@
import { expect, test } from '@playwright/test';
import type { Page } from '@playwright/test';
import type { AgentDetail } from '../mock/agents.helpers';
import { cleanupAgent, openAgentBuilder, uniqueAgentName } from '../mock/agents.helpers';
import { fetchJson, getAccessToken, messagesView, requestJson, sendMessage } from '../mock/helpers';
const REAL_MODEL = process.env.E2E_REAL_ANTHROPIC_MODEL ?? 'claude-haiku-4-5';
type PersistedMessage = {
content?: Array<{
type?: string;
tool_call?: { name?: string };
}>;
};
async function createAgent(
page: Page,
token: string,
body: {
name: string;
description: string;
instructions: string;
subagents?: AgentDetail['subagents'];
},
): Promise<AgentDetail> {
return requestJson<AgentDetail>(page, {
path: '/api/agents',
token,
method: 'POST',
body: {
...body,
provider: 'anthropic',
model: REAL_MODEL,
},
});
}
async function selectAgent(page: Page, name: string): Promise<void> {
const form = await openAgentBuilder(page);
await form.getByRole('combobox', { name: 'Agent', exact: true }).click();
await page.getByRole('option', { name }).click();
await expect(form.getByLabel('Agent name')).toHaveValue(name);
await form.getByRole('button', { name: 'Select Agent' }).click();
}
async function readMessages(page: Page, conversationId: string): Promise<PersistedMessage[]> {
const token = await getAccessToken(page);
return fetchJson<PersistedMessage[]>(page, `/api/messages/${conversationId}`, token);
}
test.describe('isolated subagent results with a real provider', () => {
test('delegates and renders the child final answer', async ({ page }) => {
test.setTimeout(240_000);
const leftOperand = 48_723;
const rightOperand = 19_642;
const expectedResult = String(leftOperand + rightOperand);
const childName = uniqueAgentName('Real Child');
const parentName = uniqueAgentName('Real Parent');
let childId: string | undefined;
let parentId: string | undefined;
try {
await page.goto('/c/new');
const token = await getAccessToken(page);
const child = await createAgent(page, token, {
name: childName,
description: 'Solves arithmetic requests delegated by a parent agent.',
instructions:
'You are an arithmetic specialist. Add the requested integers and reply with only the ' +
'decimal result, without punctuation or explanation.',
});
childId = child.id;
const parent = await createAgent(page, token, {
name: parentName,
description: 'Delegates arithmetic requests to one isolated child.',
instructions:
'Always use the subagent tool for every user request. Never answer from your own ' +
'knowledge. After the child finishes, return its answer verbatim with no added text.',
subagents: {
enabled: true,
allowSelf: false,
agent_ids: [child.id],
},
});
parentId = parent.id;
await selectAgent(page, parentName);
const response = await sendMessage(
page,
`Ask the configured child to calculate ${leftOperand} + ${rightOperand}. Return only the ` +
'integer it provides.',
);
expect(response.ok()).toBeTruthy();
await expect(page.getByRole('button', { name: 'Stop generating' })).toBeHidden({
timeout: 180_000,
});
const finalAnswer = messagesView(page)
.locator('.message-render')
.last()
.getByRole('paragraph')
.filter({ hasText: expectedResult });
await expect(finalAnswer).toHaveText(expectedResult, { timeout: 30_000 });
await expect(messagesView(page).getByText('Task completed', { exact: true })).toHaveCount(0);
await expect(page).toHaveURL(/\/c\/(?!new)/, { timeout: 30_000 });
const conversationId = new URL(page.url()).pathname.split('/c/')[1];
expect(conversationId).toBeTruthy();
let messages: PersistedMessage[] = [];
await expect
.poll(
async () => {
messages = await readMessages(page, conversationId);
return messages.some((message) =>
(message.content ?? []).some(
(part) => part.type === 'tool_call' && part.tool_call?.name === 'subagent',
),
);
},
{ timeout: 30_000, intervals: [500, 1000, 2000] },
)
.toBe(true);
} finally {
await cleanupAgent(page, parentId);
await cleanupAgent(page, childId);
}
});
});

18
package-lock.json generated
View file

@ -61,7 +61,7 @@
"@azure/storage-blob": "^12.30.0",
"@google/genai": "^2.8.0",
"@keyv/redis": "^4.3.3",
"@librechat/agents": "^3.3.13",
"@librechat/agents": "^3.4.0",
"@librechat/api": "*",
"@librechat/data-schemas": "*",
"@microsoft/microsoft-graph-client": "^3.0.7",
@ -10616,9 +10616,9 @@
}
},
"node_modules/@librechat/agents": {
"version": "3.3.13",
"resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-3.3.13.tgz",
"integrity": "sha512-NOaUCvMaq82ftzZr/Z/3KL1LzfKO0YRHibhFigcx22KxrmeioyJ32w4RnrBsYGhLx0gDrI4nWlW6Sk0ZAH/YBQ==",
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-3.4.0.tgz",
"integrity": "sha512-VKEckMv6sk40PIKQ/l3GXFugWJitZ8K4EqRu+ZJ9gpA8+3AI+FhWmenQh/n3prE9HpSq7PK9S4EngkNYPQTMlw==",
"license": "MIT",
"dependencies": {
"@anthropic-ai/sdk": "^0.115.0",
@ -10653,7 +10653,7 @@
"nanoid": "^3.3.7",
"okapibm25": "^1.4.1",
"openai": "^6.46.0",
"reo-census": "^1.2.9",
"reo-census": "^1.2.10",
"socks-proxy-agent": "^8.0.5",
"uuid": "^11.1.1"
},
@ -37851,9 +37851,9 @@
"dev": true
},
"node_modules/reo-census": {
"version": "1.2.9",
"resolved": "https://registry.npmjs.org/reo-census/-/reo-census-1.2.9.tgz",
"integrity": "sha512-QPnAyRk6gUK9h2XJy9yTXiA91+LEHahmzbRdfb6DigAoe1S/nKg2LXc+/m3//HGyMPAkCFVz/FJJ8YrKm5IrXw==",
"version": "1.2.10",
"resolved": "https://registry.npmjs.org/reo-census/-/reo-census-1.2.10.tgz",
"integrity": "sha512-Wr/cNvk1S2EMNUwLISWtO3VgSaoaKRsaAwc/t7TMPmSu1i4LhyS/mEHhQg9PhiK3bpAqSUqI5dY8riqAukXuyA==",
"hasInstallScript": true,
"license": "MIT",
"engines": {
@ -42707,7 +42707,7 @@
"@azure/storage-blob": "^12.30.0",
"@google/genai": "^2.8.0",
"@keyv/redis": "^4.3.3",
"@librechat/agents": "^3.3.13",
"@librechat/agents": "^3.4.0",
"@librechat/data-schemas": "*",
"@modelcontextprotocol/sdk": "^1.29.0",
"@opentelemetry/api": "^1.9.0",

2
packages/api/package.json
View file

@ -107,7 +107,7 @@
"@azure/storage-blob": "^12.30.0",
"@google/genai": "^2.8.0",
"@keyv/redis": "^4.3.3",
"@librechat/agents": "^3.3.13",
"@librechat/agents": "^3.4.0",
"@librechat/data-schemas": "*",
"@modelcontextprotocol/sdk": "^1.29.0",
"@opentelemetry/api": "^1.9.0",