mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-20 06:51:59 +00:00
🔎 feat: Observe Server Tool Results in Chat Completions (#18450)
* 🔎 feat: Observe Server Tool Results in Chat Completions * 🐛 fix: Finalize Observed Tools After Model Acceptance
This commit is contained in:
parent
eacd937f2e
commit
a9034aa6a2
16 changed files with 1484 additions and 95 deletions
|
|
@ -180,6 +180,8 @@ jest.mock('~/server/services/ApiKeys/grants', () => ({
|
|||
}));
|
||||
|
||||
jest.mock('@librechat/api', () => ({
|
||||
createChatCompletionToolProjection: (...args) =>
|
||||
jest.requireActual('@librechat/api').createChatCompletionToolProjection(...args),
|
||||
getModelEndUsage: (...args) => jest.requireActual('@librechat/api').getModelEndUsage(...args),
|
||||
createApiModelCallbacks: (...args) =>
|
||||
jest.requireActual('@librechat/api').createApiModelCallbacks(...args),
|
||||
|
|
@ -635,7 +637,10 @@ describe('OpenAIChatCompletionController', () => {
|
|||
req.body.stream = streaming;
|
||||
api.validateRequest.mockReturnValueOnce({ request: req.body });
|
||||
mockProcessStream.mockImplementationOnce(async () => {
|
||||
const { customHandlers: handlers } = api.createRun.mock.calls.at(-1)[0];
|
||||
const { customHandlers: handlers, eagerToolExecution } = api.createRun.mock.calls.at(-1)[0];
|
||||
expect(eagerToolExecution).toBe(
|
||||
req.body.include?.includes('server_tool_results') ? false : undefined,
|
||||
);
|
||||
expect(handlers.on_run_step).toBeUndefined();
|
||||
expect(handlers.on_run_step_delta).toBeUndefined();
|
||||
expect(handlers.on_run_step_completed).toBeUndefined();
|
||||
|
|
@ -679,6 +684,91 @@ describe('OpenAIChatCompletionController', () => {
|
|||
},
|
||||
);
|
||||
|
||||
it.each([true, false])(
|
||||
'includes completed server results without advertising execution requests (stream=%s)',
|
||||
async (streaming) => {
|
||||
const api = require('@librechat/api');
|
||||
const actual = jest.requireActual('@librechat/api');
|
||||
const names = [
|
||||
'createChunk',
|
||||
'writeSSE',
|
||||
'sendFinalChunk',
|
||||
'buildNonStreamingResponse',
|
||||
'createOpenAIStreamTracker',
|
||||
'createOpenAIContentAggregator',
|
||||
];
|
||||
const originals = names.map((name) => [name, api[name].getMockImplementation()]);
|
||||
for (const name of names) api[name].mockImplementation(actual[name]);
|
||||
req.body.stream = streaming;
|
||||
req.body.include = ['server_tool_results'];
|
||||
let completion;
|
||||
api.validateRequest.mockReturnValueOnce({ request: req.body });
|
||||
mockProcessStream.mockImplementationOnce(async () => {
|
||||
const { customHandlers: handlers, eagerToolExecution } = api.createRun.mock.calls.at(-1)[0];
|
||||
expect(eagerToolExecution).toBe(
|
||||
req.body.include?.includes('server_tool_results') ? false : undefined,
|
||||
);
|
||||
await handlers.on_model_response.handle('on_model_response', {
|
||||
type: 'model_response',
|
||||
id: 'accepted-server',
|
||||
agentId: 'agent-123',
|
||||
toolCalls: [{ id: 'lookup-1', name: 'lookup', args: { city: 'Madrid' } }],
|
||||
toolCallDispositions: ['sdk'],
|
||||
invalidToolCalls: [],
|
||||
});
|
||||
completion = handlers.on_run_step_completed;
|
||||
await completion?.handle(
|
||||
'on_run_step_completed',
|
||||
{
|
||||
result: {
|
||||
id: 'step-1',
|
||||
type: 'tool_call',
|
||||
tool_call: {
|
||||
id: 'lookup-1',
|
||||
name: 'lookup',
|
||||
args: '{"city":"Madrid"}',
|
||||
output: 'Sunny',
|
||||
},
|
||||
},
|
||||
},
|
||||
{ agentId: 'agent-123' },
|
||||
);
|
||||
await handlers.on_message_delta.handle('on_message_delta', {
|
||||
delta: { content: [{ type: 'text', text: 'It is sunny.' }] },
|
||||
});
|
||||
});
|
||||
try {
|
||||
await OpenAIChatCompletionController(req, res);
|
||||
expect(completion).toBeDefined();
|
||||
const expected = {
|
||||
call_id: 'lookup-1',
|
||||
name: 'lookup',
|
||||
arguments: '{"city":"Madrid"}',
|
||||
output: 'Sunny',
|
||||
};
|
||||
if (streaming) {
|
||||
const frames = res.write.mock.calls
|
||||
.map(([frame]) => frame)
|
||||
.filter((frame) => frame !== 'data: [DONE]\n\n')
|
||||
.map((frame) => JSON.parse(frame.slice(6)));
|
||||
expect(frames.flatMap((frame) => frame.choices[0].delta.tool_results ?? [])).toEqual([
|
||||
{ index: 0, ...expected },
|
||||
]);
|
||||
expect(frames.flatMap((frame) => frame.choices[0].delta.tool_calls ?? [])).toEqual([]);
|
||||
expect(frames.at(-1).choices[0].finish_reason).toBe('stop');
|
||||
} else {
|
||||
const choice = res.json.mock.calls[0][0].choices[0];
|
||||
expect(choice.message.tool_results).toEqual([expected]);
|
||||
expect(choice.message.tool_calls).toBeUndefined();
|
||||
expect(choice.finish_reason).toBe('stop');
|
||||
}
|
||||
} finally {
|
||||
for (const [name, implementation] of originals)
|
||||
api[name].mockImplementation(implementation);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each([true, false])(
|
||||
'publishes accepted client calls after text and a model claim (stream=%s)',
|
||||
async (streaming) => {
|
||||
|
|
@ -697,7 +787,10 @@ describe('OpenAIChatCompletionController', () => {
|
|||
req.body.stream = streaming;
|
||||
api.validateRequest.mockReturnValueOnce({ request: req.body });
|
||||
mockProcessStream.mockImplementationOnce(async () => {
|
||||
const { customHandlers: handlers } = api.createRun.mock.calls.at(-1)[0];
|
||||
const { customHandlers: handlers, eagerToolExecution } = api.createRun.mock.calls.at(-1)[0];
|
||||
expect(eagerToolExecution).toBe(
|
||||
req.body.include?.includes('server_tool_results') ? false : undefined,
|
||||
);
|
||||
await handlers.on_model_response.handle('on_model_response', {
|
||||
type: 'model_response',
|
||||
id: 'discarded',
|
||||
|
|
@ -1030,6 +1123,7 @@ describe('OpenAIChatCompletionController', () => {
|
|||
expect.anything(),
|
||||
mockCompletionUsage,
|
||||
true,
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,6 @@
|
|||
const { nanoid } = require('nanoid');
|
||||
const { logger } = require('@librechat/data-schemas');
|
||||
const { Callback, formatAgentMessages } = require('@librechat/agents');
|
||||
const {
|
||||
createOpenAIToolCallStream: createAcceptedToolCallStream,
|
||||
} = require('@librechat/agents/openai');
|
||||
const {
|
||||
EModelEndpoint,
|
||||
ResourceType,
|
||||
|
|
@ -17,6 +14,7 @@ const {
|
|||
createRemoteExecutionContext,
|
||||
getRemoteRunKeyId,
|
||||
createChunk,
|
||||
createChatCompletionToolProjection,
|
||||
applyContextToAgent,
|
||||
buildRunToolSet,
|
||||
buildInitialToolSessions,
|
||||
|
|
@ -1033,17 +1031,19 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
}
|
||||
};
|
||||
|
||||
const toolCallStream = createAcceptedToolCallStream({
|
||||
const projection = createChatCompletionToolProjection({
|
||||
streaming: isStreaming,
|
||||
signal: execution.signal,
|
||||
toolCalls: isStreaming ? tracker.toolCalls : aggregator.toolCalls,
|
||||
...(isStreaming && {
|
||||
emit: (delta) => writeSSE(res, createChunk(context, delta)),
|
||||
}),
|
||||
tracker,
|
||||
aggregator,
|
||||
writer: res,
|
||||
context,
|
||||
include: request.include,
|
||||
});
|
||||
|
||||
// Event handlers for OpenAI-compatible streaming
|
||||
const handlers = {
|
||||
...toolCallStream.handlers,
|
||||
...projection.handlers,
|
||||
// Text content streaming
|
||||
on_message_delta: createHandler((data) => {
|
||||
const content = data?.delta?.content;
|
||||
|
|
@ -1145,6 +1145,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
const initialSessions = buildInitialToolSessions({ agents: runAgents });
|
||||
|
||||
const run = await createRun({
|
||||
...projection.runOptions,
|
||||
sdkModelHost: req?.app?.locals?.sdkModelHost,
|
||||
agents: runAgents,
|
||||
messages: formattedMessages,
|
||||
|
|
@ -1191,7 +1192,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
version: 'v2',
|
||||
};
|
||||
|
||||
await completeOpenAIToolCalls(toolCallStream, async () => {
|
||||
await completeOpenAIToolCalls(projection, async () => {
|
||||
await run.processStream({ messages: formattedMessages }, config, {
|
||||
callbacks: {
|
||||
[Callback.TOOL_ERROR]: (graph, error, toolId) => {
|
||||
|
|
@ -1277,6 +1278,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
aggregator.toolCalls,
|
||||
usage,
|
||||
true,
|
||||
projection.response(),
|
||||
);
|
||||
res.json(response);
|
||||
logger.debug(
|
||||
|
|
|
|||
|
|
@ -106,7 +106,9 @@ mcpServers:
|
|||
timeout: 30000
|
||||
e2e-http:
|
||||
type: streamable-http
|
||||
url: http://127.0.0.1:8765/mcp
|
||||
url: &e2e-http-url http://127.0.0.1:8765/mcp
|
||||
# The fixture is explicitly anonymous; enrollment and execution bind the same auth descriptor.
|
||||
requiresOAuth: false
|
||||
title: E2E HTTP
|
||||
description: Local HTTP MCP fixture for allowlist-override e2e tests.
|
||||
timeout: 30000
|
||||
|
|
@ -183,6 +185,15 @@ endpoints:
|
|||
# Default capabilities plus run_in_background (off by default upstream) so the
|
||||
# background tool-call e2e can opt an MCP tool into detached dispatch.
|
||||
agents:
|
||||
remoteApi:
|
||||
integrations:
|
||||
resources:
|
||||
e2e-http:
|
||||
url: *e2e-http-url
|
||||
credentialMode: anonymous
|
||||
issuer: null
|
||||
audience: null
|
||||
scopes: []
|
||||
sdkHostFiles: {}
|
||||
sdkRuntimes:
|
||||
remote-example:
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
*/
|
||||
|
||||
const http = require('http');
|
||||
const { z } = require('zod');
|
||||
const { randomUUID } = require('crypto');
|
||||
const { keepIdleConnections } = require('./keepalive');
|
||||
const { McpServer } = require('@modelcontextprotocol/sdk/server/mcp.js');
|
||||
|
|
@ -36,6 +37,17 @@ function createMcpServer() {
|
|||
content: [{ type: 'text', text: 'E2E HTTP MCP pong' }],
|
||||
}),
|
||||
);
|
||||
// URL-backed tool result fixture: delegated API-key grants require a declared recipient.
|
||||
server.registerTool(
|
||||
'remember_fact',
|
||||
{
|
||||
description: 'Records a deterministic fact for API tool result tests.',
|
||||
inputSchema: { fact: z.string().optional() },
|
||||
},
|
||||
async ({ fact = 'LibreChat MCP e2e fact' }) => ({
|
||||
content: [{ type: 'text', text: `E2E MCP memory noted: ${fact}` }],
|
||||
}),
|
||||
);
|
||||
return server;
|
||||
}
|
||||
|
||||
|
|
|
|||
206
e2e/specs/mock/scenarios/chat-tool-results.spec.ts
Normal file
206
e2e/specs/mock/scenarios/chat-tool-results.spec.ts
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
import { randomUUID } from 'node:crypto';
|
||||
import { expect, test } from '@playwright/test';
|
||||
import type { APIRequestContext } from '@playwright/test';
|
||||
import { loginAdmin, restoreRuntimeFilters, setRuntimeFilters } from '../content-filters.helpers';
|
||||
|
||||
type Result = { call_id: string; name: string; arguments: string; output: string; index?: number };
|
||||
|
||||
async function remote(request: APIRequestContext) {
|
||||
const token = await loginAdmin(request);
|
||||
const headers = { Authorization: `Bearer ${token}` };
|
||||
const profile = await request.get('/api/user', { headers });
|
||||
expect(profile.status()).toBe(200);
|
||||
const user = await profile.json();
|
||||
const userId = user.id ?? user._id;
|
||||
const origin = `http://127.0.0.1:${process.env.E2E_MCP_HTTP_PORT || '8765'}`;
|
||||
const overridePath = `/api/admin/config/user/${userId}`;
|
||||
const created: { agentId?: string; keyId?: string } = {};
|
||||
const cleanup = async () => {
|
||||
if (created.keyId) await request.delete(`/api/api-keys/${created.keyId}`, { headers });
|
||||
if (created.agentId) await request.delete(`/api/agents/${created.agentId}`, { headers });
|
||||
const cleared = await request.delete(overridePath, { headers });
|
||||
expect([200, 204, 404]).toContain(cleared.status());
|
||||
};
|
||||
try {
|
||||
const configured = await request.put(overridePath, {
|
||||
headers,
|
||||
data: {
|
||||
overrides: {
|
||||
mcpSettings: { allowedDomains: [origin] },
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(configured.ok(), await configured.text()).toBe(true);
|
||||
await expect
|
||||
.poll(
|
||||
async () => {
|
||||
const initialized = await request.post('/api/mcp/e2e-http/reinitialize', { headers });
|
||||
return initialized.ok() && (await initialized.json()).success === true;
|
||||
},
|
||||
{ timeout: 30000, intervals: [500, 1000] },
|
||||
)
|
||||
.toBe(true);
|
||||
const agent = await request.post('/api/agents', {
|
||||
headers,
|
||||
data: {
|
||||
name: `Tool results ${randomUUID().slice(0, 8)}`,
|
||||
provider: 'Mock Provider A',
|
||||
model: 'mock-model-a',
|
||||
model_parameters: {},
|
||||
tools: ['remember_fact_mcp_e2e-http'],
|
||||
},
|
||||
});
|
||||
expect(agent.status(), await agent.text()).toBe(201);
|
||||
const agentId = (await agent.json()).id;
|
||||
created.agentId = agentId;
|
||||
const key = await request.post('/api/api-keys', {
|
||||
headers,
|
||||
data: { name: 'tool result contract' },
|
||||
});
|
||||
expect(key.status()).toBe(201);
|
||||
const credentials = await key.json();
|
||||
created.keyId = credentials.id;
|
||||
// The API key owns a narrow per-agent grant; observation must not bypass MCP authorization.
|
||||
const consentPath = `/api/api-keys/${credentials.id}/agents/${agentId}/mcp-consent`;
|
||||
const offered = await request.get(consentPath, { headers });
|
||||
expect(offered.status(), await offered.text()).toBe(200);
|
||||
const view = await offered.json();
|
||||
expect(view.offer).not.toBeNull();
|
||||
const target = view.targets.find(
|
||||
(entry: { resource: { serverName: string } }) => entry.resource.serverName === 'e2e-http',
|
||||
);
|
||||
expect(target).toBeDefined();
|
||||
const selected = target.permittedTools.find(
|
||||
(entry: { agentId: string }) => entry.agentId === agentId,
|
||||
);
|
||||
const tools = selected.tools.filter(
|
||||
(name: string) => name === 'remember_fact' || name === 'remember_fact_mcp_e2e-http',
|
||||
);
|
||||
expect(tools).toHaveLength(1);
|
||||
const granted = await request.post(consentPath, {
|
||||
headers,
|
||||
data: {
|
||||
offerDigest: view.offer.digest,
|
||||
expectedRevision: view.revision,
|
||||
lifetimeHours: 1,
|
||||
allow: { 'e2e-http': tools },
|
||||
},
|
||||
});
|
||||
expect(granted.status(), await granted.text()).toBe(200);
|
||||
|
||||
return {
|
||||
token,
|
||||
agentId,
|
||||
apiKey: credentials.key as string,
|
||||
cleanup,
|
||||
};
|
||||
} catch (error) {
|
||||
await cleanup();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function execute(
|
||||
request: APIRequestContext,
|
||||
agent: Awaited<ReturnType<typeof remote>>,
|
||||
stream: boolean,
|
||||
include: boolean,
|
||||
) {
|
||||
const label = randomUUID().replace(/-/g, '').slice(0, 10);
|
||||
const response = await request.post('/api/agents/v1/chat/completions', {
|
||||
headers: { Authorization: `Bearer ${agent.apiKey}` },
|
||||
data: {
|
||||
model: agent.agentId,
|
||||
messages: [{ role: 'user', content: `E2E_ACTIVITY_REPLY:${label}` }],
|
||||
stream,
|
||||
...(include && { include: ['server_tool_results'] }),
|
||||
},
|
||||
timeout: 60000,
|
||||
});
|
||||
const text = await response.text();
|
||||
expect(response.status(), text).toBe(200);
|
||||
if (!stream) {
|
||||
const choice = JSON.parse(text).choices[0];
|
||||
expect(choice.finish_reason).toBe('stop');
|
||||
expect(choice.message.content).toContain(`E2E activity reply done ${label}`);
|
||||
expect(choice.message.tool_calls).toBeUndefined();
|
||||
if (!include) expect(choice.message.tool_results).toBeUndefined();
|
||||
return { label, results: (choice.message.tool_results ?? []) as Result[], text };
|
||||
}
|
||||
const frames = text
|
||||
.split('\n\n')
|
||||
.map((frame) => frame.trim())
|
||||
.filter((frame) => frame.startsWith('data: ') && frame !== 'data: [DONE]')
|
||||
.map((frame) => JSON.parse(frame.slice(6)));
|
||||
expect(frames[frames.length - 1].choices[0].finish_reason).toBe('stop');
|
||||
expect(frames.map((frame) => frame.choices[0].delta.content ?? '').join('')).toContain(
|
||||
`E2E activity reply done ${label}`,
|
||||
);
|
||||
expect(frames.flatMap((frame) => frame.choices[0].delta.tool_calls ?? [])).toEqual([]);
|
||||
return {
|
||||
label,
|
||||
results: frames.flatMap((frame) => frame.choices[0].delta.tool_results ?? []) as Result[],
|
||||
text,
|
||||
};
|
||||
}
|
||||
|
||||
test('Chat tool result observation preserves execution ownership in JSON and SSE @scenario:chat-server-tool-results', async ({
|
||||
playwright,
|
||||
baseURL,
|
||||
}) => {
|
||||
test.setTimeout(180000);
|
||||
const request = await playwright.request.newContext({ baseURL });
|
||||
const agent = await remote(request);
|
||||
try {
|
||||
for (const stream of [false, true])
|
||||
for (const include of [false, true]) {
|
||||
const { label, results } = await execute(request, agent, stream, include);
|
||||
expect(results).toHaveLength(include ? 2 : 0);
|
||||
if (!include) continue;
|
||||
expect(new Set(results.map((result) => result.call_id)).size).toBe(2);
|
||||
for (const result of results) {
|
||||
expect(result.name).toBe('remember_fact_mcp_e2e-http');
|
||||
expect(JSON.parse(result.arguments).fact).toContain(label);
|
||||
expect(result.output).toContain('E2E MCP memory noted:');
|
||||
}
|
||||
if (stream) expect(results.map((result) => result.index)).toEqual([0, 1]);
|
||||
}
|
||||
} finally {
|
||||
await agent.cleanup();
|
||||
await request.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
test('Chat observed results retain the tool-output protection policy @scenario:chat-tool-results-protected', async ({
|
||||
playwright,
|
||||
baseURL,
|
||||
}) => {
|
||||
test.setTimeout(120000);
|
||||
const request = await playwright.request.newContext({ baseURL });
|
||||
const agent = await remote(request);
|
||||
try {
|
||||
await setRuntimeFilters(request, agent.token, {
|
||||
toolArguments: {
|
||||
pii: {
|
||||
fields: ['output'],
|
||||
starterPatterns: [],
|
||||
customPatterns: [
|
||||
{ id: 'tool-result-probe', label: 'Tool result probe', regex: 'E2E MCP memory noted:' },
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
for (const stream of [false, true]) {
|
||||
const { results, text } = await execute(request, agent, stream, true);
|
||||
expect(results).toHaveLength(2);
|
||||
expect(text).not.toContain('E2E MCP memory noted:');
|
||||
expect(text).not.toContain('mcp_reauth_required');
|
||||
expect(results.every((result) => /blocked/i.test(result.output))).toBe(true);
|
||||
expect(results.every((result) => result.output.length > 0)).toBe(true);
|
||||
}
|
||||
} finally {
|
||||
await restoreRuntimeFilters(request, agent.token);
|
||||
await agent.cleanup();
|
||||
await request.dispose();
|
||||
}
|
||||
});
|
||||
|
|
@ -522,6 +522,20 @@ describe('createRun code-tool eager/session wiring', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('lets observed Chat results defer tool execution until model acceptance', async () => {
|
||||
await createRun({
|
||||
agents: [makeAgent()] as never,
|
||||
signal: new AbortController().signal,
|
||||
eagerToolExecution: false,
|
||||
});
|
||||
expect((Run.create as jest.Mock).mock.calls[0][0].eagerEventToolExecution).toEqual(
|
||||
expect.objectContaining({
|
||||
enabled: false,
|
||||
excludeToolNames: expect.arrayContaining(['create_file', 'execute_code']),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('declares create_file/edit_file/read_file as code-session participants', async () => {
|
||||
const runConfig = await captureRunConfig();
|
||||
expect(runConfig.codeSessionToolNames).toEqual(
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Agents API response projection
|
||||
|
||||
This module projects an **agent run**, which can include several model invocations and server-executed tools, onto an OpenAI-compatible response. It is not a transparent proxy for a single model completion. Only explicitly client-owned calls in graph-accepted model responses may become outward `tool_calls`; SDK- and provider-executed calls stay inside the run. Binding caller-supplied tool schemas and client execution remains a separate feature.
|
||||
This module projects an **agent run**, which can include several model invocations and server-executed tools, onto an OpenAI-compatible response. It is not a transparent proxy for a single model completion. Only explicitly client-owned calls in graph-accepted model responses may become outward `tool_calls`; SDK- and provider-executed calls never become client execution requests. Completed SDK executions can be observed separately with the opt-in extension below. Binding caller-supplied tool schemas and client execution remains a separate feature.
|
||||
|
||||
## Owners and boundaries
|
||||
|
||||
|
|
@ -69,3 +69,64 @@ The SDK projector accepts only client-disposition calls from graph-accepted resp
|
|||
- `service.spec.ts` and the Express controller spec: both response modes, actual serializers, internal-call suppression, accepted client calls, claim discard and failure behavior.
|
||||
|
||||
These tests do not run a live provider, a distributed multi-agent deployment or a persisted resume. The graph test executes a local tool but does not exercise the mounted HTTP server. There is no new database, cache, authorization policy or migration. Code and built packages must deploy together.
|
||||
|
||||
## Opt-in server execution results
|
||||
|
||||
Both Chat hosts accept the LibreChat extension `include: ["server_tool_results"]`.
|
||||
It adds completed execution records to `choices[0].message.tool_results` in JSON,
|
||||
or `choices[0].delta.tool_results` in SSE. A record is `{ call_id, name, arguments,
|
||||
output }`, all strings; streamed records also carry a stable zero-based completion
|
||||
`index`. JSON includes an empty array if no results are observed. Omitted `include`
|
||||
or `include: []` omits the fields. Unknown/malformed selectors return 400 before
|
||||
execution; duplicate supported selectors are harmless.
|
||||
|
||||
```text
|
||||
accepted model response + sdk disposition (owner/call identity)
|
||||
-> SDK ToolNode processed completion + authoritative metadata.agentId
|
||||
-> run-local bounded observation projector
|
||||
-> JSON tool_results / indexed SSE delta.tool_results
|
||||
```
|
||||
|
||||
For this selector, both Chat hosts disable the SDK's speculative eager tool
|
||||
prestart. Tools execute after model acceptance, so the observer receives final
|
||||
processed ToolNode results rather than trying to associate a failed provider
|
||||
attempt's early output with an accepted fallback. Normal Chat requests and the
|
||||
Responses API retain their existing execution policy. Custom `createRun` adapters
|
||||
must forward the supplied `eagerToolExecution: false` option.
|
||||
|
||||
This trades tool/model-stream overlap for authoritative result observation:
|
||||
opted-in tools start after the model finishes its accepted call, and failed
|
||||
provider attempts do not speculatively execute them. Tool latency may therefore
|
||||
be higher for opted-in requests; there is no production latency estimate.
|
||||
|
||||
The collector never changes execution requests, tool ownership, `finish_reason`,
|
||||
authorization, billing or artifact callbacks. It consumes only accepted SDK-owned
|
||||
calls with matching completed execution owner, name and call ID. Pinned SDK 4.2.10
|
||||
`ToolNode.withToolScope` stamps `metadata.agentId`; `Run` forwards it unchanged.
|
||||
Missing owner metadata fails closed with an incomplete flag: an ownerless result
|
||||
cannot be inferred safely from a pending ID when provider/client siblings reuse
|
||||
that ID. Acceptance deduplication is also scoped by execution owner.
|
||||
|
||||
Only processed string arguments/results are exposed; no raw ToolMessage, artifact
|
||||
or graph-state serialization is performed. Existing tool-output protection and
|
||||
SDK truncation remain authoritative. Provider-hosted tools without SDK completion
|
||||
and unfinished/client-executed calls have no invented records. These observations
|
||||
are distinct from the actionable client `tool_calls` projection above. Responses
|
||||
API output, persistence and retrieval are unchanged; this selector applies only
|
||||
to Chat Completions and is not an Open Responses requirement.
|
||||
|
||||
Projection-only safety guards bound capture to 1024 accepted server calls and
|
||||
4 MiB of identity/argument/output UTF-8 text per run, matching the SDK client-call
|
||||
projection envelope. They are independent of tool execution limits; exhaustion
|
||||
fences additional observation while preserving completed records and normal graph
|
||||
success. Smaller guards can be injected into the collector for constrained hosts
|
||||
and focused tests. Overflow or unattributable ownership adds
|
||||
`message.tool_results_incomplete: true` in JSON or
|
||||
`delta.tool_results_incomplete: true` on the terminal SSE chunk. Such records are
|
||||
not a complete audit log. Partial SSE records also do not imply run success if the
|
||||
usual execution error follows. No durable Chat storage is introduced.
|
||||
|
||||
Successful records survive the existing `completeOpenAIToolCalls` finally-abort
|
||||
cleanup after `finish`. Failure/cancellation clears unpublished records; late and
|
||||
duplicate completions cannot append results. Accepted/completed identity state is
|
||||
released at the run boundary.
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import type { ServerToolResults } from './results';
|
||||
import { createServerToolResults } from './results';
|
||||
/**
|
||||
* OpenAI-compatible event handlers for agent streaming.
|
||||
*
|
||||
|
|
@ -12,6 +14,7 @@ import type {
|
|||
OpenAIResponseContext,
|
||||
ChatCompletionChunk,
|
||||
CompletionUsage,
|
||||
ServerToolResultFields,
|
||||
ToolCall,
|
||||
} from './types';
|
||||
import type { UsageMetadata } from '~/stream/interfaces/IJobStore';
|
||||
|
|
@ -63,6 +66,8 @@ export function writeSSE(writer: OpenAIWriter, data: ChatCompletionChunk | strin
|
|||
*/
|
||||
export interface OpenAIStreamTracker {
|
||||
/** Successful response completion, installed by the response handlers. */
|
||||
serverToolResults?: ServerToolResults;
|
||||
toolExecution?: { eagerToolExecution?: false };
|
||||
finishToolCalls?: () => void;
|
||||
abortToolCalls?: () => void;
|
||||
/** Whether any text content was emitted */
|
||||
|
|
@ -113,6 +118,8 @@ export function createOpenAIStreamTracker(): OpenAIStreamTracker {
|
|||
*/
|
||||
export interface OpenAIContentAggregator {
|
||||
/** Successful response completion, installed by the response handlers. */
|
||||
serverToolResults?: ServerToolResults;
|
||||
toolExecution?: { eagerToolExecution?: false };
|
||||
finishToolCalls?: () => void;
|
||||
abortToolCalls?: () => void;
|
||||
/** Accumulated text chunks */
|
||||
|
|
@ -162,6 +169,7 @@ export function createOpenAIContentAggregator(): OpenAIContentAggregator {
|
|||
|
||||
/** Legacy stream configuration; retained for existing package consumers. */
|
||||
export interface OpenAIStreamHandlerConfig {
|
||||
include?: 'server_tool_results'[];
|
||||
signal?: AbortSignal;
|
||||
res: OpenAIWriter;
|
||||
context: OpenAIResponseContext;
|
||||
|
|
@ -766,29 +774,78 @@ export class OpenAIReasoningDeltaHandler implements EventHandler {
|
|||
* Create all handlers for OpenAI streaming format
|
||||
*/
|
||||
export interface OpenAIAggregationHandlerConfig {
|
||||
include?: 'server_tool_results'[];
|
||||
aggregator: OpenAIContentAggregator;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
type OpenAIContentHandlerConfig = OpenAIContentStreamConfig | OpenAIAggregationHandlerConfig;
|
||||
|
||||
type ChatCompletionToolProjectionConfig = {
|
||||
include?: 'server_tool_results'[];
|
||||
signal?: AbortSignal;
|
||||
} & (
|
||||
| {
|
||||
streaming: true;
|
||||
tracker: OpenAIStreamTracker;
|
||||
writer: OpenAIWriter;
|
||||
context: OpenAIResponseContext;
|
||||
aggregator?: OpenAIContentAggregator;
|
||||
}
|
||||
| {
|
||||
streaming: false;
|
||||
aggregator: OpenAIContentAggregator;
|
||||
tracker?: OpenAIStreamTracker;
|
||||
writer?: OpenAIWriter;
|
||||
context?: OpenAIResponseContext;
|
||||
}
|
||||
);
|
||||
|
||||
/** Both HTTP hosts share selection, execution policy, framing and terminal ownership. */
|
||||
export function createChatCompletionToolProjection(
|
||||
config: ChatCompletionToolProjectionConfig,
|
||||
): ReturnType<typeof createAcceptedToolCallStream> & {
|
||||
runOptions: { eagerToolExecution?: false };
|
||||
response(): ServerToolResultFields | undefined;
|
||||
} {
|
||||
const target = config.streaming ? config.tracker : config.aggregator;
|
||||
const emit = config.streaming
|
||||
? (delta: ChatCompletionChunkChoice['delta']) =>
|
||||
writeSSE(config.writer, createChunk(config.context, delta))
|
||||
: undefined;
|
||||
const accepted = createAcceptedToolCallStream({
|
||||
signal: config.signal,
|
||||
toolCalls: target.toolCalls,
|
||||
emit,
|
||||
});
|
||||
const results = config.include?.includes('server_tool_results')
|
||||
? createServerToolResults({ signal: config.signal, emit })
|
||||
: undefined;
|
||||
const projection = results?.wrap(accepted) ?? accepted;
|
||||
// Opt-in observation needs accepted, processed ToolNode completions. Eager
|
||||
// prestart emits before acceptance and can reuse a failed provider attempt.
|
||||
const runOptions = results ? { eagerToolExecution: false as const } : {};
|
||||
target.serverToolResults = results;
|
||||
target.toolExecution = runOptions;
|
||||
target.finishToolCalls = projection.finish;
|
||||
target.abortToolCalls = projection.abort;
|
||||
return { ...projection, runOptions, response: () => results?.response() };
|
||||
}
|
||||
|
||||
export function createOpenAIHandlers(
|
||||
config: OpenAIContentHandlerConfig,
|
||||
toolExecuteOptions?: ToolExecuteOptions,
|
||||
): Record<string, EventHandler> {
|
||||
const target = 'aggregator' in config ? config.aggregator : config.tracker;
|
||||
const toolCallProjection = createAcceptedToolCallStream({
|
||||
signal: config.signal,
|
||||
toolCalls: target.toolCalls,
|
||||
emit:
|
||||
'aggregator' in config
|
||||
? undefined
|
||||
: (delta) => writeSSE(getStreamWriter(config), createChunk(config.context, delta)),
|
||||
});
|
||||
target.finishToolCalls = toolCallProjection.finish;
|
||||
target.abortToolCalls = toolCallProjection.abort;
|
||||
const projection =
|
||||
'aggregator' in config
|
||||
? createChatCompletionToolProjection({ ...config, streaming: false })
|
||||
: createChatCompletionToolProjection({
|
||||
...config,
|
||||
streaming: true,
|
||||
writer: getStreamWriter(config),
|
||||
});
|
||||
const handlers: Record<string, EventHandler> = {
|
||||
...toolCallProjection.handlers,
|
||||
...projection.handlers,
|
||||
[GraphEvents.ON_MESSAGE_DELTA]: new OpenAIMessageDeltaHandler(config),
|
||||
[GraphEvents.CHAT_MODEL_END]: new OpenAIModelEndHandler(config),
|
||||
[GraphEvents.CHAT_MODEL_STREAM]: new OpenAIChatModelStreamHandler(),
|
||||
|
|
@ -837,7 +894,12 @@ export function sendFinalChunk(
|
|||
};
|
||||
}
|
||||
|
||||
const finalChunk = createChunk(context, {}, reason, usage);
|
||||
const finalChunk = createChunk(
|
||||
context,
|
||||
tracker.serverToolResults?.incomplete ? { tool_results_incomplete: true } : {},
|
||||
reason,
|
||||
usage,
|
||||
);
|
||||
writeSSE(writer, finalChunk);
|
||||
|
||||
// Send [DONE] marker
|
||||
|
|
|
|||
|
|
@ -50,3 +50,5 @@ export {
|
|||
type ChatCompletionValidationSuccess,
|
||||
type ChatCompletionValidationFailure,
|
||||
} from './service';
|
||||
|
||||
export { createServerToolResults } from './results';
|
||||
|
|
|
|||
301
packages/api/src/agents/openai/results.spec.ts
Normal file
301
packages/api/src/agents/openai/results.spec.ts
Normal file
|
|
@ -0,0 +1,301 @@
|
|||
import { createOpenAIToolCallStream } from '@librechat/agents/openai';
|
||||
import { createServerToolResults, validateChatIncludes } from './results';
|
||||
import { createOpenAIStreamTracker, sendFinalChunk } from './handlers';
|
||||
import { buildNonStreamingResponse } from './service';
|
||||
|
||||
function fixture(options: Parameters<typeof createServerToolResults>[0] = {}) {
|
||||
const collector = createServerToolResults(options);
|
||||
const calls = new Map();
|
||||
const delegate = createOpenAIToolCallStream({ toolCalls: calls });
|
||||
const projection = collector.wrap(delegate);
|
||||
const accept = async (agent = 'a', id = 'call', disposition = 'sdk') => {
|
||||
await projection.handlers.on_model_response.handle('on_model_response', {
|
||||
type: 'model_response',
|
||||
id: `accepted-${agent}-${id}`,
|
||||
agentId: agent,
|
||||
toolCalls: [{ id, name: 'lookup', args: {} }],
|
||||
toolCallDispositions: [disposition],
|
||||
invalidToolCalls: [],
|
||||
} as never);
|
||||
};
|
||||
const complete = async (
|
||||
agent = 'a',
|
||||
id = 'call',
|
||||
output = 'protected result',
|
||||
step = `step-${agent}-${id}`,
|
||||
) => {
|
||||
await projection.handlers.on_run_step_completed.handle(
|
||||
'on_run_step_completed',
|
||||
{
|
||||
result: {
|
||||
id: step,
|
||||
type: 'tool_call',
|
||||
tool_call: {
|
||||
id,
|
||||
name: 'lookup',
|
||||
args: '{}',
|
||||
output,
|
||||
artifact: { private: 'secret' },
|
||||
},
|
||||
},
|
||||
} as never,
|
||||
{ agentId: agent } as never,
|
||||
);
|
||||
};
|
||||
return { collector, calls, delegate, projection, accept, complete };
|
||||
}
|
||||
|
||||
describe('server tool result projection', () => {
|
||||
it('validates an explicit extension selector', () => {
|
||||
for (const value of [
|
||||
undefined,
|
||||
[],
|
||||
['server_tool_results'],
|
||||
['server_tool_results', 'server_tool_results'],
|
||||
])
|
||||
expect(validateChatIncludes(value)).toBeUndefined();
|
||||
for (const value of [null, true, 'server_tool_results', [3], ['unknown']])
|
||||
expect(validateChatIncludes(value)).toEqual(expect.any(String));
|
||||
});
|
||||
|
||||
it('correlates SDK completions, deduplicates and never changes client execution requests', async () => {
|
||||
const f = fixture();
|
||||
await f.accept();
|
||||
await f.accept('a', 'client', 'client');
|
||||
await f.accept('a', 'provider', 'provider');
|
||||
await f.complete('a', 'client');
|
||||
await f.complete('a', 'provider');
|
||||
await f.complete('a', 'unknown');
|
||||
await f.complete();
|
||||
await f.complete();
|
||||
await f.accept();
|
||||
await f.complete();
|
||||
expect(f.collector.response()).toEqual({
|
||||
tool_results: [
|
||||
{ call_id: 'call', name: 'lookup', arguments: '{}', output: 'protected result' },
|
||||
],
|
||||
});
|
||||
expect(JSON.stringify(f.collector.response())).not.toContain('secret');
|
||||
// The final accepted internal-only turn supersedes pending client requests as usual.
|
||||
expect(f.calls.size).toBe(0);
|
||||
});
|
||||
|
||||
it('keeps parallel calls and identical IDs from different agents separate', async () => {
|
||||
const f = fixture();
|
||||
await f.accept('a', 'same');
|
||||
await f.accept('b', 'same');
|
||||
await Promise.all([f.complete('b', 'same', 'B'), f.complete('a', 'same', 'A')]);
|
||||
expect(f.collector.response().tool_results.map((result) => result.output)).toEqual(['B', 'A']);
|
||||
});
|
||||
|
||||
it('refuses ambiguous ownerless completions and unaccepted identities', async () => {
|
||||
const f = fixture();
|
||||
await f.accept('a', 'same');
|
||||
await f.accept('b', 'same');
|
||||
await f.projection.handlers.on_run_step_completed.handle('on_run_step_completed', {
|
||||
result: {
|
||||
id: 'step',
|
||||
type: 'tool_call',
|
||||
tool_call: { id: 'same', name: 'lookup', args: '{}', output: 'private sibling' },
|
||||
},
|
||||
});
|
||||
await f.complete('foreign', 'same', 'foreign result');
|
||||
expect(f.collector.response()).toEqual({ tool_results: [], tool_results_incomplete: true });
|
||||
});
|
||||
|
||||
it('never reattributes an ownerless duplicate after its sibling completed', async () => {
|
||||
const f = fixture();
|
||||
await f.accept('a', 'same');
|
||||
await f.accept('b', 'same');
|
||||
await f.complete('a', 'same', 'A', 'step-a');
|
||||
await f.projection.handlers.on_run_step_completed.handle('on_run_step_completed', {
|
||||
result: {
|
||||
id: 'step-a',
|
||||
type: 'tool_call',
|
||||
tool_call: { id: 'same', name: 'lookup', args: '{}', output: 'A' },
|
||||
},
|
||||
});
|
||||
await f.complete('b', 'same', 'B');
|
||||
expect(f.collector.response().tool_results.map((result) => result.output)).toEqual(['A', 'B']);
|
||||
expect(f.collector.incomplete).toBe(true);
|
||||
});
|
||||
|
||||
it.each(['provider', 'client'])(
|
||||
'never attributes an ownerless %s sibling to a server execution',
|
||||
async (disposition) => {
|
||||
const f = fixture();
|
||||
await f.accept('a', 'same', 'sdk');
|
||||
await f.accept('b', 'same', disposition);
|
||||
await f.projection.handlers.on_run_step_completed.handle('on_run_step_completed', {
|
||||
result: {
|
||||
id: 'step-b',
|
||||
type: 'tool_call',
|
||||
tool_call: { id: 'same', name: 'lookup', args: '{}', output: 'private B' },
|
||||
},
|
||||
});
|
||||
await f.complete('a', 'same', 'A');
|
||||
expect(f.collector.response().tool_results.map((result) => result.output)).toEqual(['A']);
|
||||
expect(f.collector.incomplete).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it('deduplicates model acceptance within its execution owner', async () => {
|
||||
const f = fixture();
|
||||
for (const agentId of ['a', 'b']) {
|
||||
await f.projection.handlers.on_model_response.handle('on_model_response', {
|
||||
type: 'model_response',
|
||||
id: 'same-acceptance',
|
||||
agentId,
|
||||
toolCalls: [{ id: 'same', name: 'lookup', args: {} }],
|
||||
toolCallDispositions: ['sdk'],
|
||||
invalidToolCalls: [],
|
||||
});
|
||||
}
|
||||
await f.complete('a', 'same', 'A');
|
||||
await f.complete('b', 'same', 'B');
|
||||
expect(f.collector.response().tool_results.map((result) => result.output)).toEqual(['A', 'B']);
|
||||
});
|
||||
|
||||
it('fences an oversized completed output without serializing artifacts or failing the graph', async () => {
|
||||
const f = fixture({ maxBytes: 100 });
|
||||
await f.accept();
|
||||
await f.complete('a', 'call', 'x'.repeat(200));
|
||||
await f.complete('a', 'call', 'late');
|
||||
expect(() => f.projection.finish()).not.toThrow();
|
||||
expect(f.collector.response()).toEqual({ tool_results: [], tool_results_incomplete: true });
|
||||
});
|
||||
|
||||
it('emits complete indexed records once and fences results on cancellation', async () => {
|
||||
const controller = new AbortController();
|
||||
const emit = jest.fn();
|
||||
const f = fixture({ signal: controller.signal, emit });
|
||||
await f.accept();
|
||||
await f.complete();
|
||||
controller.abort();
|
||||
await f.complete('a', 'late');
|
||||
expect(emit).toHaveBeenCalledTimes(1);
|
||||
expect(emit.mock.calls[0][0].tool_results[0]).toEqual(
|
||||
expect.objectContaining({ index: 0, output: 'protected result' }),
|
||||
);
|
||||
expect(f.collector.response().tool_results).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([{ maxCalls: 1 }, { maxBytes: 32 }])(
|
||||
'marks overflow without failing execution (%j)',
|
||||
async (limits) => {
|
||||
const emit = jest.fn();
|
||||
const f = fixture({ ...limits, emit });
|
||||
await f.accept('a', 'one');
|
||||
await f.complete('a', 'one', 'first');
|
||||
await f.accept('a', 'two');
|
||||
await f.complete('a', 'two', 'second');
|
||||
expect(() => f.projection.finish()).not.toThrow();
|
||||
expect(f.collector.response().tool_results_incomplete).toBe(true);
|
||||
expect(f.collector.response().tool_results.length).toBeLessThan(2);
|
||||
},
|
||||
);
|
||||
|
||||
it('reports the same overflow flag in terminal SSE and JSON without changing stop', async () => {
|
||||
const f = fixture({ maxCalls: 1 });
|
||||
await f.accept('a', 'one');
|
||||
await f.complete('a', 'one');
|
||||
await f.accept('a', 'two');
|
||||
const tracker = createOpenAIStreamTracker();
|
||||
tracker.serverToolResults = f.collector;
|
||||
tracker.finishToolCalls = f.projection.finish;
|
||||
const frames: string[] = [];
|
||||
const context = { requestId: 'test', created: 1, model: 'fixture' };
|
||||
sendFinalChunk({
|
||||
tracker,
|
||||
context,
|
||||
writer: {
|
||||
write: (frame) => {
|
||||
frames.push(frame);
|
||||
},
|
||||
},
|
||||
});
|
||||
const terminal = JSON.parse(frames[frames.length - 2].slice(6));
|
||||
expect(terminal.choices[0].delta).toEqual({ tool_results_incomplete: true });
|
||||
expect(terminal.choices[0].finish_reason).toBe('stop');
|
||||
const response = buildNonStreamingResponse(
|
||||
context,
|
||||
'done',
|
||||
'',
|
||||
new Map(),
|
||||
{ prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
|
||||
true,
|
||||
f.collector.response(),
|
||||
);
|
||||
expect(response.choices[0].message.tool_results_incomplete).toBe(true);
|
||||
expect(response.choices[0].message.tool_results).toHaveLength(1);
|
||||
expect(response.choices[0].finish_reason).toBe('stop');
|
||||
});
|
||||
|
||||
it('retains successful output through the SDK lifecycle cleanup', async () => {
|
||||
const f = fixture();
|
||||
await f.accept();
|
||||
await f.complete();
|
||||
f.projection.finish();
|
||||
f.projection.abort();
|
||||
expect(f.collector.response().tool_results).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('discards unpublished results when cancellation has no following event', async () => {
|
||||
const controller = new AbortController();
|
||||
const f = fixture({ signal: controller.signal });
|
||||
await f.accept();
|
||||
await f.complete();
|
||||
controller.abort();
|
||||
expect(() => f.projection.finish()).toThrow(expect.objectContaining({ name: 'AbortError' }));
|
||||
f.projection.abort();
|
||||
expect(f.collector.response()).toEqual({ tool_results: [] });
|
||||
});
|
||||
|
||||
it('does not seal success when the delegate cancels during finish', async () => {
|
||||
const controller = new AbortController();
|
||||
const f = fixture({ signal: controller.signal });
|
||||
jest.spyOn(f.delegate, 'finish').mockImplementation(() => controller.abort());
|
||||
await f.accept();
|
||||
await f.complete();
|
||||
expect(() => f.projection.finish()).toThrow(expect.objectContaining({ name: 'AbortError' }));
|
||||
f.projection.abort();
|
||||
expect(f.collector.response()).toEqual({ tool_results: [] });
|
||||
});
|
||||
|
||||
it('does not finalize after the result sink cancels reentrantly', async () => {
|
||||
const controller = new AbortController();
|
||||
const emit = jest.fn(() => controller.abort());
|
||||
const f = fixture({ signal: controller.signal, emit });
|
||||
const finish = jest.spyOn(f.delegate, 'finish');
|
||||
await f.accept();
|
||||
await f.complete();
|
||||
expect(() => f.projection.finish()).toThrow(expect.objectContaining({ name: 'AbortError' }));
|
||||
f.projection.abort();
|
||||
expect(emit).toHaveBeenCalledTimes(1);
|
||||
expect(finish).not.toHaveBeenCalled();
|
||||
expect(f.collector.response()).toEqual({ tool_results: [] });
|
||||
});
|
||||
|
||||
it('seals a successful finish once before normal finally cleanup', async () => {
|
||||
const controller = new AbortController();
|
||||
const f = fixture({ signal: controller.signal });
|
||||
const finish = jest.spyOn(f.delegate, 'finish');
|
||||
await f.accept();
|
||||
await f.complete();
|
||||
f.projection.finish();
|
||||
f.projection.finish();
|
||||
controller.abort();
|
||||
f.projection.abort();
|
||||
expect(finish).toHaveBeenCalledTimes(1);
|
||||
expect(f.collector.response().tool_results).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('discards buffered results and ignores late events after abort', async () => {
|
||||
const f = fixture();
|
||||
await f.accept();
|
||||
await f.complete();
|
||||
f.projection.abort();
|
||||
await f.complete();
|
||||
expect(f.collector.response()).toEqual({ tool_results: [] });
|
||||
});
|
||||
});
|
||||
227
packages/api/src/agents/openai/results.ts
Normal file
227
packages/api/src/agents/openai/results.ts
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
import type { EventHandler } from '@librechat/agents';
|
||||
import type { ServerToolResult, ChatCompletionChunkChoice, ServerToolResultFields } from './types';
|
||||
|
||||
/** Public-payload safety guards, separate from tool execution limits. Exhaustion
|
||||
* stops observation, never the graph. Match the SDK client-call projection's
|
||||
* 1024-call/4-MiB envelope; hosts may inject smaller budgets for their transport. */
|
||||
const MAX_RESULTS = 1024;
|
||||
const MAX_BYTES = 4 * 1024 * 1024;
|
||||
|
||||
type Projection = {
|
||||
handlers: Record<string, EventHandler>;
|
||||
finish: () => void;
|
||||
abort: () => void;
|
||||
};
|
||||
|
||||
type Options = {
|
||||
signal?: AbortSignal;
|
||||
emit?: (delta: ChatCompletionChunkChoice['delta']) => void;
|
||||
maxCalls?: number;
|
||||
maxBytes?: number;
|
||||
};
|
||||
|
||||
function record(value: unknown): value is Record<string, unknown> {
|
||||
return value != null && typeof value === 'object' && !Array.isArray(value);
|
||||
}
|
||||
|
||||
export function validateChatIncludes(value: unknown): string | undefined {
|
||||
if (value === undefined) return;
|
||||
if (!Array.isArray(value) || value.some((item) => item !== 'server_tool_results'))
|
||||
return 'include must be an array containing only server_tool_results';
|
||||
}
|
||||
|
||||
export interface ServerToolResults {
|
||||
readonly incomplete: boolean;
|
||||
response(): ServerToolResultFields;
|
||||
wrap<T extends Projection>(projection: T): T;
|
||||
}
|
||||
|
||||
/** Captures only SDK-accepted, server-executed completions. Raw tool-end content
|
||||
* and artifacts are deliberately outside this public projection. */
|
||||
export function createServerToolResults({
|
||||
signal,
|
||||
emit,
|
||||
maxCalls = MAX_RESULTS,
|
||||
maxBytes = MAX_BYTES,
|
||||
}: Options = {}): ServerToolResults {
|
||||
const pending = new Map<string, { name: string }>();
|
||||
const announcements = new Set<string>();
|
||||
const completed = new Set<string>();
|
||||
const results: ServerToolResult[] = [];
|
||||
let count = 0;
|
||||
let bytes = 0;
|
||||
let index = 0;
|
||||
let incomplete = false;
|
||||
let fenced = false;
|
||||
let closed = false;
|
||||
let finished = false;
|
||||
|
||||
const release = () => {
|
||||
pending.clear();
|
||||
announcements.clear();
|
||||
completed.clear();
|
||||
};
|
||||
const abort = () => {
|
||||
closed = true;
|
||||
release();
|
||||
if (!finished) {
|
||||
results.length = 0;
|
||||
incomplete = false;
|
||||
}
|
||||
};
|
||||
const active = () => {
|
||||
if (!closed && signal?.aborted) abort();
|
||||
return !closed && !fenced;
|
||||
};
|
||||
const reserve = (...values: string[]) => {
|
||||
let size = 0;
|
||||
for (const value of values) size += Buffer.byteLength(value, 'utf8');
|
||||
if (bytes + size > maxBytes) {
|
||||
incomplete = fenced = true;
|
||||
release();
|
||||
return false;
|
||||
}
|
||||
bytes += size;
|
||||
return true;
|
||||
};
|
||||
const accept = (data: unknown) => {
|
||||
if (
|
||||
!active() ||
|
||||
!record(data) ||
|
||||
data.type !== 'model_response' ||
|
||||
typeof data.agentId !== 'string' ||
|
||||
typeof data.id !== 'string' ||
|
||||
!data.id ||
|
||||
!Array.isArray(data.toolCalls) ||
|
||||
!Array.isArray(data.toolCallDispositions)
|
||||
)
|
||||
return;
|
||||
if (data.agentId.length + data.id.length > maxBytes) {
|
||||
incomplete = fenced = true;
|
||||
release();
|
||||
return;
|
||||
}
|
||||
const announcement = JSON.stringify([data.agentId, data.id]);
|
||||
if (announcements.has(announcement)) return;
|
||||
const dispositions = data.toolCallDispositions;
|
||||
if (dispositions.length !== data.toolCalls.length) return;
|
||||
for (let i = 0; i < data.toolCalls.length; i++) {
|
||||
if (dispositions[i] !== 'sdk') continue;
|
||||
const call: unknown = data.toolCalls[i];
|
||||
if (
|
||||
!record(call) ||
|
||||
typeof call.id !== 'string' ||
|
||||
!call.id ||
|
||||
typeof call.name !== 'string' ||
|
||||
!call.name
|
||||
)
|
||||
continue;
|
||||
if (count >= maxCalls) {
|
||||
incomplete = fenced = true;
|
||||
release();
|
||||
return;
|
||||
}
|
||||
if (!reserve(data.agentId, call.id, call.name)) return;
|
||||
if (!announcements.has(announcement)) {
|
||||
if (!reserve(data.id)) return;
|
||||
announcements.add(announcement);
|
||||
}
|
||||
const key = JSON.stringify([data.agentId, call.id]);
|
||||
pending.set(key, { name: call.name });
|
||||
count++;
|
||||
}
|
||||
};
|
||||
const complete = (data: unknown, metadata: unknown) => {
|
||||
if (!active() || !record(data) || !record(data.result)) return;
|
||||
const step = data.result;
|
||||
const call = step.tool_call;
|
||||
if (
|
||||
step.type !== 'tool_call' ||
|
||||
typeof step.id !== 'string' ||
|
||||
!step.id ||
|
||||
!record(call) ||
|
||||
typeof call.id !== 'string' ||
|
||||
!call.id ||
|
||||
typeof call.args !== 'string' ||
|
||||
typeof call.output !== 'string'
|
||||
)
|
||||
return;
|
||||
// SDK 4.2.10 ToolNode.withToolScope stamps this execution owner on
|
||||
// completion metadata. Never infer an owner from a pending call ID:
|
||||
// provider/client siblings can reuse that ID without entering this collector.
|
||||
const owner = record(metadata) ? metadata.agentId : undefined;
|
||||
if (typeof owner !== 'string' || !owner) {
|
||||
incomplete = true;
|
||||
return;
|
||||
}
|
||||
if (call.id.length > maxBytes || step.id.length > maxBytes || owner.length > maxBytes) return;
|
||||
const key = JSON.stringify([owner, call.id]);
|
||||
const accepted = pending.get(key);
|
||||
if (!accepted || call.name !== accepted.name) return;
|
||||
const identity = JSON.stringify([key, step.id]);
|
||||
if (completed.has(identity)) return;
|
||||
if (!reserve(step.id, call.args, call.output)) return;
|
||||
completed.add(identity);
|
||||
pending.delete(key);
|
||||
const result: ServerToolResult = {
|
||||
call_id: call.id,
|
||||
name: accepted.name,
|
||||
arguments: call.args,
|
||||
output: call.output,
|
||||
};
|
||||
if (emit) emit({ tool_results: [{ index: index++, ...result }] });
|
||||
else results.push(result);
|
||||
};
|
||||
return {
|
||||
get incomplete(): boolean {
|
||||
return incomplete;
|
||||
},
|
||||
response(): ServerToolResultFields {
|
||||
return {
|
||||
tool_results: results,
|
||||
...(incomplete && { tool_results_incomplete: true as const }),
|
||||
};
|
||||
},
|
||||
wrap<T extends Projection>(projection: T): T {
|
||||
const accepted = projection.handlers.on_model_response;
|
||||
return {
|
||||
...projection,
|
||||
handlers: {
|
||||
...projection.handlers,
|
||||
on_model_response: {
|
||||
async handle(event, data, metadata, graph) {
|
||||
await accepted.handle(event, data, metadata, graph);
|
||||
if (event === 'on_model_response') accept(data);
|
||||
},
|
||||
},
|
||||
on_run_step_completed: {
|
||||
handle(event, data, metadata) {
|
||||
if (event === 'on_run_step_completed') complete(data, metadata);
|
||||
},
|
||||
},
|
||||
},
|
||||
finish() {
|
||||
if (finished) return;
|
||||
const check = () => {
|
||||
if (!signal?.aborted) return;
|
||||
abort();
|
||||
const error = new Error('Agent response aborted');
|
||||
error.name = 'AbortError';
|
||||
throw error;
|
||||
};
|
||||
check();
|
||||
if (closed) return;
|
||||
projection.finish();
|
||||
check();
|
||||
finished = true;
|
||||
closed = true;
|
||||
release();
|
||||
},
|
||||
abort() {
|
||||
abort();
|
||||
projection.abort();
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -6,7 +6,7 @@ import type { CallbackHandlerMethods } from '@langchain/core/callbacks/base';
|
|||
import type { FiltersConfig } from 'librechat-data-provider';
|
||||
import type { ChatCompletionDependencies } from './service';
|
||||
import type { ChatCompletionChunk } from './types';
|
||||
import { createAgentChatCompletion } from './service';
|
||||
import { createAgentChatCompletion, validateRequest } from './service';
|
||||
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
logger: {
|
||||
|
|
@ -150,6 +150,85 @@ describe('createAgentChatCompletion - MCP permission user propagation', () => {
|
|||
},
|
||||
);
|
||||
|
||||
it.each([true, false])(
|
||||
'returns opted-in server results in the packaged service (stream=%s)',
|
||||
async (stream) => {
|
||||
const req = createMockReq(
|
||||
{ id: 'user' },
|
||||
{
|
||||
model: 'agent_test',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
stream,
|
||||
include: ['server_tool_results'],
|
||||
},
|
||||
);
|
||||
const res = createMockRes();
|
||||
processStream.mockImplementationOnce(async () => {
|
||||
const { customHandlers: handlers } = createRun.mock.calls[0][0] as Parameters<
|
||||
NonNullable<ChatCompletionDependencies['createRun']>
|
||||
>[0];
|
||||
await handlers.on_model_response.handle('on_model_response', {
|
||||
type: 'model_response',
|
||||
id: 'accepted-server',
|
||||
agentId: 'agent_test',
|
||||
toolCalls: [{ id: 'server', name: 'lookup', args: {} }],
|
||||
toolCallDispositions: ['sdk'],
|
||||
invalidToolCalls: [],
|
||||
});
|
||||
await handlers.on_run_step_completed.handle(
|
||||
'on_run_step_completed',
|
||||
{
|
||||
result: {
|
||||
id: 'step',
|
||||
type: 'tool_call',
|
||||
tool_call: { id: 'server', name: 'lookup', args: '{}', output: 'filtered result' },
|
||||
},
|
||||
},
|
||||
{ agentId: 'agent_test' },
|
||||
);
|
||||
});
|
||||
await createAgentChatCompletion(req, res, deps);
|
||||
expect(createRun.mock.calls[0][0]).toHaveProperty('eagerToolExecution', false);
|
||||
const expected = {
|
||||
call_id: 'server',
|
||||
name: 'lookup',
|
||||
arguments: '{}',
|
||||
output: 'filtered result',
|
||||
};
|
||||
if (stream) {
|
||||
const chunks: ChatCompletionChunk[] = (res.write as jest.Mock).mock.calls
|
||||
.map(([frame]: [string]) => frame)
|
||||
.filter((frame) => frame !== 'data: [DONE]\n\n')
|
||||
.map((frame) => JSON.parse(frame.slice(6)));
|
||||
expect(chunks.flatMap((chunk) => chunk.choices[0].delta.tool_results ?? [])).toEqual([
|
||||
{ index: 0, ...expected },
|
||||
]);
|
||||
expect(chunks.flatMap((chunk) => chunk.choices[0].delta.tool_calls ?? [])).toEqual([]);
|
||||
expect(chunks[chunks.length - 1]?.choices[0].finish_reason).toBe('stop');
|
||||
} else {
|
||||
const response = getResponseMock(res, 'json').mock.calls[0][0];
|
||||
expect(response.choices[0].message.tool_results).toEqual([expected]);
|
||||
expect(response.choices[0].message.tool_calls).toBeUndefined();
|
||||
expect(response.choices[0].finish_reason).toBe('stop');
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each([null, true, 'server_tool_results', ['unknown'], [3]])(
|
||||
'rejects malformed include before execution (%j)',
|
||||
async (include) => {
|
||||
const req = createMockReq(
|
||||
{ id: 'user' },
|
||||
{ model: 'agent_test', messages: [{ role: 'user', content: 'hi' }], include },
|
||||
);
|
||||
const res = createMockRes();
|
||||
expect(validateRequest(req.body).valid).toBe(false);
|
||||
await createAgentChatCompletion(req, res, deps);
|
||||
expect(getResponseMock(res, 'status')).toHaveBeenCalledWith(400);
|
||||
expect(createRun).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it.each([true, false])(
|
||||
'keeps graph and provider-owned calls internal (stream=%s)',
|
||||
async (stream) => {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import type { LocatorTraversalReporter } from '../../protection/diagnostics';
|
||||
import type { SdkModelHost } from '../remoteSdk/builder';
|
||||
import type { ServerToolResultFields } from './types';
|
||||
import { validateChatIncludes } from './results';
|
||||
/**
|
||||
* OpenAI-compatible chat completions service for agents.
|
||||
*
|
||||
|
|
@ -301,6 +303,7 @@ type CreateRunAppConfig = Pick<
|
|||
|
||||
type CreateRunFn = (params: {
|
||||
sdkModelHost?: SdkModelHost;
|
||||
eagerToolExecution?: boolean;
|
||||
modelCallbacks?: ReturnType<typeof createApiModelCallbacks>;
|
||||
agents: unknown[];
|
||||
messages: unknown[];
|
||||
|
|
@ -523,6 +526,8 @@ export function validateRequest(body: unknown): ChatCompletionValidationResult {
|
|||
}
|
||||
|
||||
const request = body as Record<string, unknown>;
|
||||
const includeError = validateChatIncludes(request.include);
|
||||
if (includeError) return { valid: false, error: includeError };
|
||||
|
||||
if (!request.model || typeof request.model !== 'string') {
|
||||
return { valid: false, error: 'model (agent_id) is required' };
|
||||
|
|
@ -603,6 +608,7 @@ export function buildNonStreamingResponse(
|
|||
usage: CompletionUsage,
|
||||
/** True when the map contains only accepted client-owned calls, not legacy run-step history. */
|
||||
acceptedToolCallsOnly = false,
|
||||
results?: ServerToolResultFields,
|
||||
): ChatCompletionResponse {
|
||||
const toolCallsArray = Array.from(toolCalls.values());
|
||||
const finishReason =
|
||||
|
|
@ -619,6 +625,7 @@ export function buildNonStreamingResponse(
|
|||
message: {
|
||||
role: 'assistant',
|
||||
content: text || null,
|
||||
...results,
|
||||
...(reasoning && { reasoning }),
|
||||
...(toolCallsArray.length > 0 && { tool_calls: toolCallsArray }),
|
||||
},
|
||||
|
|
@ -872,9 +879,10 @@ export async function createAgentChatCompletion(
|
|||
// Create event handlers
|
||||
const eventHandlers = createOpenAIHandlers(
|
||||
handlerConfig
|
||||
? { ...handlerConfig, signal: abortController.signal }
|
||||
? { ...handlerConfig, signal: abortController.signal, include: request.include }
|
||||
: {
|
||||
aggregator: aggregator!,
|
||||
include: request.include,
|
||||
signal: abortController.signal,
|
||||
},
|
||||
deps.toolExecuteOptions == null
|
||||
|
|
@ -886,6 +894,8 @@ export async function createAgentChatCompletion(
|
|||
},
|
||||
);
|
||||
|
||||
const target = tracker ?? aggregator!;
|
||||
|
||||
// Convert messages to internal format
|
||||
const messages = convertMessages(request.messages);
|
||||
|
||||
|
|
@ -903,6 +913,7 @@ export async function createAgentChatCompletion(
|
|||
*/
|
||||
const safeUser: Record<string, unknown> = { ...createSafeUser(reqUser), id: userId };
|
||||
const run = await deps.createRun({
|
||||
...target.toolExecution,
|
||||
sdkModelHost: deps.sdkModelHost,
|
||||
modelCallbacks: createApiModelCallbacks({
|
||||
appConfig: deps.appConfig,
|
||||
|
|
@ -922,7 +933,6 @@ export async function createAgentChatCompletion(
|
|||
});
|
||||
|
||||
if (run) {
|
||||
const target = tracker ?? aggregator!;
|
||||
await completeOpenAIToolCalls(
|
||||
{
|
||||
finish: () => target.finishToolCalls?.(),
|
||||
|
|
@ -982,6 +992,7 @@ export async function createAgentChatCompletion(
|
|||
aggregator.toolCalls,
|
||||
usage,
|
||||
true,
|
||||
aggregator.serverToolResults?.response(),
|
||||
);
|
||||
res.json(response);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import {
|
|||
Run,
|
||||
} from '@librechat/agents';
|
||||
import type { BaseMessage } from '@langchain/core/messages';
|
||||
import type { EventHandler } from '@librechat/agents';
|
||||
import type { Response } from 'express';
|
||||
import type { ChatCompletionChunk, ToolCall } from './types';
|
||||
import {
|
||||
|
|
@ -264,75 +265,358 @@ describe('tool-call projection with real SDK graph dispatch', () => {
|
|||
'{"i":1}',
|
||||
]);
|
||||
});
|
||||
it('keeps tools executed by the SDK out of the final OpenAI response', async () => {
|
||||
const frames: string[] = [];
|
||||
const tracker = createOpenAIStreamTracker();
|
||||
const context = { requestId: 'accepted', created: 1, model: 'fixture' };
|
||||
const res: Response = Object.create(response);
|
||||
jest.spyOn(res, 'write').mockImplementation((frame) => {
|
||||
frames.push(String(frame));
|
||||
return true;
|
||||
});
|
||||
const config = { tracker, context, res };
|
||||
const executed: string[] = [];
|
||||
const handlers = createOpenAIHandlers(config);
|
||||
const run = await Run.create({
|
||||
runId: 'accepted-host-integration',
|
||||
skipCleanup: true,
|
||||
tokenCounter: () => 1,
|
||||
graphConfig: {
|
||||
type: 'standard',
|
||||
maxContextTokens: 100_000,
|
||||
llmConfig: {
|
||||
provider: Providers.OPENAI,
|
||||
streaming: false,
|
||||
streamUsage: false,
|
||||
it.each([false, true])(
|
||||
'observes SDK tool results only when opted in (include=%s)',
|
||||
async (observe) => {
|
||||
const frames: string[] = [];
|
||||
const tracker = createOpenAIStreamTracker();
|
||||
const context = { requestId: 'accepted', created: 1, model: 'fixture' };
|
||||
const res: Response = Object.create(response);
|
||||
jest.spyOn(res, 'write').mockImplementation((frame) => {
|
||||
frames.push(String(frame));
|
||||
return true;
|
||||
});
|
||||
const config = {
|
||||
tracker,
|
||||
context,
|
||||
res,
|
||||
include: observe ? ['server_tool_results' as const] : undefined,
|
||||
};
|
||||
const executed: string[] = [];
|
||||
const handlers = createOpenAIHandlers(config);
|
||||
const owners: unknown[] = [];
|
||||
if (observe) {
|
||||
const complete = handlers.on_run_step_completed;
|
||||
handlers.on_run_step_completed = {
|
||||
handle(event, data, metadata, graph) {
|
||||
owners.push(metadata?.agentId);
|
||||
return complete.handle(event, data, metadata, graph);
|
||||
},
|
||||
};
|
||||
}
|
||||
const run = await Run.create({
|
||||
runId: 'accepted-host-integration',
|
||||
skipCleanup: true,
|
||||
tokenCounter: () => 1,
|
||||
graphConfig: {
|
||||
type: 'standard',
|
||||
maxContextTokens: 100_000,
|
||||
llmConfig: {
|
||||
provider: Providers.OPENAI,
|
||||
streaming: false,
|
||||
streamUsage: false,
|
||||
},
|
||||
tools: [
|
||||
tool(
|
||||
async ({ city }) => {
|
||||
executed.push(city);
|
||||
return 'sunny';
|
||||
},
|
||||
{
|
||||
name: 'lookup',
|
||||
description: 'Weather',
|
||||
schema: z.object({ city: z.string() }),
|
||||
},
|
||||
),
|
||||
],
|
||||
},
|
||||
customHandlers: handlers,
|
||||
});
|
||||
if (run.Graph == null) throw new Error('Missing graph');
|
||||
let calls = 0;
|
||||
run.Graph.overrideModel = {
|
||||
invoke: async (messages: BaseMessage[]): Promise<AIMessageChunk> => {
|
||||
calls++;
|
||||
return messages.some((message) => message.getType() === 'tool')
|
||||
? new AIMessageChunk('done')
|
||||
: new AIMessageChunk({
|
||||
content: '',
|
||||
tool_calls: [{ id: 'internal', name: 'lookup', args: { city: 'Paris' } }],
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
await run.processStream(
|
||||
{ messages: [new HumanMessage('weather')] },
|
||||
{ configurable: { thread_id: 'accepted-host-integration' }, version: 'v2' },
|
||||
);
|
||||
sendFinalChunk(config);
|
||||
|
||||
expect(calls).toBe(2);
|
||||
expect(executed).toEqual(['Paris']);
|
||||
if (observe) expect(owners).toEqual([expect.any(String)]);
|
||||
expect(tracker.toolCalls.size).toBe(0);
|
||||
const chunks: ChatCompletionChunk[] = frames
|
||||
.filter((frame) => frame !== 'data: [DONE]\n\n')
|
||||
.map((frame) => JSON.parse(frame.slice(6)));
|
||||
expect(chunks.flatMap((chunk) => chunk.choices[0].delta.tool_calls ?? [])).toEqual([]);
|
||||
expect(chunks.some((chunk) => chunk.choices[0].delta.content === 'done')).toBe(true);
|
||||
expect(chunks.flatMap((chunk) => chunk.choices[0].delta.tool_results ?? [])).toEqual(
|
||||
observe
|
||||
? [
|
||||
{
|
||||
index: 0,
|
||||
call_id: 'internal',
|
||||
name: 'lookup',
|
||||
arguments: '{"city":"Paris"}',
|
||||
output: 'sunny',
|
||||
},
|
||||
]
|
||||
: [],
|
||||
);
|
||||
expect(chunks[chunks.length - 1].choices[0].finish_reason).toBe('stop');
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ observe: true, fallback: undefined },
|
||||
{ observe: true, fallback: 'Paris' },
|
||||
{ observe: true, fallback: 'Berlin' },
|
||||
{ observe: false, fallback: undefined },
|
||||
])('observes accepted results without eager speculation (%j)', async ({ observe, fallback }) => {
|
||||
const frames: string[] = [];
|
||||
const events: string[] = [];
|
||||
const owners: unknown[] = [];
|
||||
const tracker = createOpenAIStreamTracker();
|
||||
const config = {
|
||||
tracker,
|
||||
context: { requestId: 'eager', created: 1, model: 'fixture' },
|
||||
writer: {
|
||||
write: (frame: string) => {
|
||||
frames.push(frame);
|
||||
},
|
||||
tools: [
|
||||
tool(
|
||||
async ({ city }) => {
|
||||
executed.push(city);
|
||||
return 'sunny';
|
||||
},
|
||||
{
|
||||
name: 'lookup',
|
||||
description: 'Weather',
|
||||
schema: z.object({ city: z.string() }),
|
||||
},
|
||||
),
|
||||
],
|
||||
},
|
||||
customHandlers: handlers,
|
||||
include: observe ? ['server_tool_results' as const] : undefined,
|
||||
};
|
||||
const handlers: Record<string, EventHandler> = createOpenAIHandlers(config);
|
||||
let completed!: () => void;
|
||||
const completion = new Promise<void>((resolve) => {
|
||||
completed = resolve;
|
||||
});
|
||||
if (run.Graph == null) throw new Error('Missing graph');
|
||||
let calls = 0;
|
||||
run.Graph.overrideModel = {
|
||||
invoke: async (messages: BaseMessage[]): Promise<AIMessageChunk> => {
|
||||
calls++;
|
||||
return messages.some((message) => message.getType() === 'tool')
|
||||
? new AIMessageChunk('done')
|
||||
: new AIMessageChunk({
|
||||
content: '',
|
||||
tool_calls: [{ id: 'internal', name: 'lookup', args: { city: 'Paris' } }],
|
||||
});
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
const deadline = new Promise<never>((_resolve, reject) => {
|
||||
timeout = setTimeout(() => reject(new Error('Eager completion was not dispatched')), 3000);
|
||||
});
|
||||
for (const name of ['on_model_response', 'on_run_step_completed'] as const) {
|
||||
const delegate = handlers[name];
|
||||
handlers[name] = {
|
||||
async handle(event, data, metadata, graph) {
|
||||
events.push(name);
|
||||
if (name === 'on_run_step_completed') {
|
||||
owners.push(metadata?.agentId);
|
||||
if (
|
||||
data == null ||
|
||||
!('result' in data) ||
|
||||
data.result == null ||
|
||||
typeof data.result !== 'object' ||
|
||||
!('id' in data.result) ||
|
||||
typeof data.result.id !== 'string'
|
||||
)
|
||||
throw new Error('Expected SDK completion');
|
||||
if (!observe) expect(data.result).toEqual(expect.objectContaining({ eager: true }));
|
||||
const step = graph?.getRunStep(data.result.id);
|
||||
expect(step?.stepDetails).toEqual(
|
||||
expect.objectContaining({
|
||||
tool_calls: expect.arrayContaining([expect.objectContaining({ id: 'eager' })]),
|
||||
}),
|
||||
);
|
||||
}
|
||||
await delegate?.handle(event, data, metadata, graph);
|
||||
if (name === 'on_run_step_completed') completed();
|
||||
},
|
||||
};
|
||||
}
|
||||
const fallbackCalls: unknown[] = [];
|
||||
const transport = jest.fn(async () => {
|
||||
const chunks = [
|
||||
{
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
delta: {
|
||||
role: 'assistant',
|
||||
content: '',
|
||||
tool_calls: [
|
||||
{
|
||||
index: 0,
|
||||
id: 'eager',
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'lookup',
|
||||
arguments: JSON.stringify({ city: fallback }),
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
finish_reason: null,
|
||||
},
|
||||
],
|
||||
},
|
||||
{ choices: [{ index: 0, delta: {}, finish_reason: 'tool_calls' }] },
|
||||
];
|
||||
const payload =
|
||||
chunks
|
||||
.map(
|
||||
(chunk) =>
|
||||
'data: ' +
|
||||
JSON.stringify({
|
||||
id: 'fallback-response',
|
||||
object: 'chat.completion.chunk',
|
||||
created: 1,
|
||||
model: 'fallback',
|
||||
...chunk,
|
||||
}) +
|
||||
'\n\n',
|
||||
)
|
||||
.join('') + 'data: [DONE]\n\n';
|
||||
return new globalThis.Response(payload, {
|
||||
status: 200,
|
||||
headers: { 'content-type': 'text/event-stream' },
|
||||
});
|
||||
});
|
||||
const acceptance = handlers.on_model_response;
|
||||
handlers.on_model_response = {
|
||||
async handle(event, data, metadata, graph) {
|
||||
if (data != null && 'toolCallDispositions' in data && data.toolCalls.length > 0) {
|
||||
fallbackCalls.push(data.toolCalls[0].args);
|
||||
// The SDK registry retains the failed primary step even when accepted fallback args differ.
|
||||
const step = graph?.getRunStep(graph.toolCallStepIds.get('eager') ?? '');
|
||||
expect(step?.stepDetails).toEqual(
|
||||
expect.objectContaining({
|
||||
tool_calls: expect.arrayContaining([
|
||||
expect.objectContaining({ args: { city: 'Paris' } }),
|
||||
]),
|
||||
}),
|
||||
);
|
||||
}
|
||||
await acceptance.handle(event, data, metadata, graph);
|
||||
},
|
||||
};
|
||||
|
||||
await run.processStream(
|
||||
{ messages: [new HumanMessage('weather')] },
|
||||
{ configurable: { thread_id: 'accepted-host-integration' }, version: 'v2' },
|
||||
);
|
||||
sendFinalChunk(config);
|
||||
|
||||
expect(calls).toBe(2);
|
||||
expect(executed).toEqual(['Paris']);
|
||||
expect(tracker.toolCalls.size).toBe(0);
|
||||
const chunks: ChatCompletionChunk[] = frames
|
||||
.filter((frame) => frame !== 'data: [DONE]\n\n')
|
||||
.map((frame) => JSON.parse(frame.slice(6)));
|
||||
expect(chunks.flatMap((chunk) => chunk.choices[0].delta.tool_calls ?? [])).toEqual([]);
|
||||
expect(chunks.some((chunk) => chunk.choices[0].delta.content === 'done')).toBe(true);
|
||||
expect(chunks[chunks.length - 1].choices[0].finish_reason).toBe('stop');
|
||||
const executed: string[] = [];
|
||||
handlers.on_tool_execute = {
|
||||
handle(_event, data) {
|
||||
if (data == null || !('resolve' in data))
|
||||
throw new Error('Expected SDK tool execution request');
|
||||
events.push('execute');
|
||||
data.resolve(
|
||||
data.toolCalls.map((call) => {
|
||||
executed.push(call.id);
|
||||
return { toolCallId: call.id, status: 'success', content: 'protected result' };
|
||||
}),
|
||||
);
|
||||
},
|
||||
};
|
||||
try {
|
||||
const run = await Run.create({
|
||||
runId: 'eager-host-integration',
|
||||
skipCleanup: true,
|
||||
tokenCounter: () => 1,
|
||||
eagerEventToolExecution: { enabled: tracker.toolExecution?.eagerToolExecution ?? true },
|
||||
graphConfig: {
|
||||
type: 'standard',
|
||||
maxContextTokens: 100_000,
|
||||
llmConfig: {
|
||||
provider: Providers.OPENAI,
|
||||
streaming: true,
|
||||
streamUsage: false,
|
||||
fallbacks: fallback
|
||||
? [
|
||||
{
|
||||
provider: Providers.OPENAI,
|
||||
clientOptions: {
|
||||
apiKey: 'fixture',
|
||||
model: 'fallback',
|
||||
streaming: true,
|
||||
maxRetries: 0,
|
||||
configuration: { fetch: transport },
|
||||
},
|
||||
},
|
||||
]
|
||||
: undefined,
|
||||
},
|
||||
tools: [],
|
||||
toolDefinitions: [
|
||||
{
|
||||
name: 'lookup',
|
||||
description: 'Fixture',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: { city: { type: 'string' } },
|
||||
required: ['city'],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
customHandlers: handlers,
|
||||
});
|
||||
if (run.Graph == null) throw new Error('Missing graph');
|
||||
run.Graph.overrideModel = {
|
||||
invoke: async () => {
|
||||
throw new Error('Expected streaming provider invocation');
|
||||
},
|
||||
stream: async (messages: BaseMessage[]) => {
|
||||
return (async function* () {
|
||||
if (messages.some((message) => message.getType() === 'tool')) {
|
||||
yield new AIMessageChunk('done');
|
||||
return;
|
||||
}
|
||||
yield new AIMessageChunk({
|
||||
content: '',
|
||||
tool_call_chunks: [
|
||||
{
|
||||
id: 'eager',
|
||||
name: 'lookup',
|
||||
args: '{"city":"Paris"}',
|
||||
index: 0,
|
||||
type: 'tool_call_chunk',
|
||||
},
|
||||
],
|
||||
response_metadata: { finish_reason: 'tool_calls' },
|
||||
});
|
||||
if (!observe) await Promise.race([completion, deadline]);
|
||||
if (fallback) throw new Error('Fixture primary failed after eager completion');
|
||||
})();
|
||||
},
|
||||
};
|
||||
await run.processStream(
|
||||
{ messages: [new HumanMessage('weather')] },
|
||||
{ configurable: { thread_id: 'eager-host-integration' }, version: 'v2' },
|
||||
);
|
||||
sendFinalChunk(config);
|
||||
if (observe) {
|
||||
expect(events.indexOf('on_model_response')).toBeLessThan(events.indexOf('execute'));
|
||||
expect(owners).toEqual(['default']);
|
||||
} else {
|
||||
expect(events.indexOf('on_run_step_completed')).toBeLessThan(
|
||||
events.indexOf('on_model_response'),
|
||||
);
|
||||
expect(owners).toEqual([undefined]);
|
||||
}
|
||||
expect(fallbackCalls).toEqual([{ city: fallback ?? 'Paris' }]);
|
||||
expect(transport).toHaveBeenCalledTimes(fallback ? 1 : 0);
|
||||
expect(executed).toEqual(['eager']);
|
||||
expect(tracker.toolCalls.size).toBe(0);
|
||||
const chunks: ChatCompletionChunk[] = frames
|
||||
.filter((frame) => frame !== 'data: [DONE]\n\n')
|
||||
.map((frame) => JSON.parse(frame.slice(6)));
|
||||
expect(chunks.flatMap((chunk) => chunk.choices[0].delta.tool_calls ?? [])).toEqual([]);
|
||||
expect(chunks.flatMap((chunk) => chunk.choices[0].delta.tool_results ?? [])).toEqual(
|
||||
observe
|
||||
? [
|
||||
{
|
||||
index: 0,
|
||||
call_id: 'eager',
|
||||
name: 'lookup',
|
||||
arguments: JSON.stringify({ city: fallback ?? 'Paris' }),
|
||||
output: 'protected result',
|
||||
},
|
||||
]
|
||||
: [],
|
||||
);
|
||||
expect(chunks[chunks.length - 1].choices[0].finish_reason).toBe('stop');
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
});
|
||||
|
||||
it('assembles the name before emitting when real SDK chunks split it', async () => {
|
||||
|
|
|
|||
|
|
@ -51,6 +51,8 @@ export interface ChatMessage {
|
|||
* OpenAI chat completion request
|
||||
*/
|
||||
export interface ChatCompletionRequest {
|
||||
/** LibreChat extension: observe completed server tools without requesting client execution. */
|
||||
include?: 'server_tool_results'[];
|
||||
/** Agent ID to invoke (maps to model in OpenAI spec) */
|
||||
model: string;
|
||||
/** Conversation messages */
|
||||
|
|
@ -100,6 +102,20 @@ export interface CompletionUsage {
|
|||
subagent?: CompletionUsageTotals;
|
||||
}
|
||||
|
||||
/** A completed server execution, distinct from an actionable client tool_call. */
|
||||
export interface ServerToolResult {
|
||||
call_id: string;
|
||||
name: string;
|
||||
arguments: string;
|
||||
output: string;
|
||||
}
|
||||
|
||||
export interface ServerToolResultFields {
|
||||
tool_results: ServerToolResult[];
|
||||
/** Projection budget exhausted: these records are not a complete audit log. */
|
||||
tool_results_incomplete?: true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Non-streaming choice
|
||||
*/
|
||||
|
|
@ -110,6 +126,8 @@ export interface ChatCompletionChoice {
|
|||
content: string | null;
|
||||
/** Reasoning/thinking content (OpenRouter convention) */
|
||||
reasoning?: string | null;
|
||||
tool_results?: ServerToolResult[];
|
||||
tool_results_incomplete?: true;
|
||||
tool_calls?: ToolCall[];
|
||||
};
|
||||
finish_reason: 'stop' | 'length' | 'tool_calls' | 'content_filter' | null;
|
||||
|
|
@ -138,6 +156,8 @@ export interface ChatCompletionChunkChoice {
|
|||
content?: string | null;
|
||||
/** Reasoning/thinking content (OpenRouter convention) */
|
||||
reasoning?: string | null;
|
||||
tool_results?: Array<ServerToolResult & { index: number }>;
|
||||
tool_results_incomplete?: true;
|
||||
tool_calls?: Array<{
|
||||
index: number;
|
||||
id?: string;
|
||||
|
|
|
|||
|
|
@ -2357,6 +2357,7 @@ export async function createRun({
|
|||
sdkResumeState,
|
||||
streaming = true,
|
||||
streamUsage = true,
|
||||
eagerToolExecution = true,
|
||||
}: {
|
||||
agents: RunAgent[];
|
||||
signal: AbortSignal;
|
||||
|
|
@ -2366,6 +2367,8 @@ export async function createRun({
|
|||
conversationId?: string;
|
||||
streaming?: boolean;
|
||||
streamUsage?: boolean;
|
||||
/** Host-owned execution policy; observed Chat results require post-acceptance execution. */
|
||||
eagerToolExecution?: boolean;
|
||||
requestBody?: t.RequestBody;
|
||||
codeApprovalMode?: CodeApprovalMode;
|
||||
user?: IUser;
|
||||
|
|
@ -3292,7 +3295,7 @@ export async function createRun({
|
|||
// `interrupt()` from its tool body, which must run inside the Pregel task
|
||||
// frame — a speculative eager execution could never pause the run.
|
||||
eagerEventToolExecution: {
|
||||
enabled: true,
|
||||
enabled: eagerToolExecution,
|
||||
excludeToolNames: [
|
||||
CREATE_FILE_TOOL_NAME,
|
||||
EDIT_FILE_TOOL_NAME,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue