diff --git a/api/server/controllers/agents/__tests__/openai.spec.js b/api/server/controllers/agents/__tests__/openai.spec.js index e115e0094c..05d7a92d12 100644 --- a/api/server/controllers/agents/__tests__/openai.spec.js +++ b/api/server/controllers/agents/__tests__/openai.spec.js @@ -180,6 +180,8 @@ jest.mock('~/server/services/ApiKeys/grants', () => ({ })); jest.mock('@librechat/api', () => ({ + createChatCompletionToolProjection: (...args) => + jest.requireActual('@librechat/api').createChatCompletionToolProjection(...args), getModelEndUsage: (...args) => jest.requireActual('@librechat/api').getModelEndUsage(...args), createApiModelCallbacks: (...args) => jest.requireActual('@librechat/api').createApiModelCallbacks(...args), @@ -635,7 +637,10 @@ describe('OpenAIChatCompletionController', () => { req.body.stream = streaming; api.validateRequest.mockReturnValueOnce({ request: req.body }); mockProcessStream.mockImplementationOnce(async () => { - const { customHandlers: handlers } = api.createRun.mock.calls.at(-1)[0]; + const { customHandlers: handlers, eagerToolExecution } = api.createRun.mock.calls.at(-1)[0]; + expect(eagerToolExecution).toBe( + req.body.include?.includes('server_tool_results') ? false : undefined, + ); expect(handlers.on_run_step).toBeUndefined(); expect(handlers.on_run_step_delta).toBeUndefined(); expect(handlers.on_run_step_completed).toBeUndefined(); @@ -679,6 +684,91 @@ describe('OpenAIChatCompletionController', () => { }, ); + it.each([true, false])( + 'includes completed server results without advertising execution requests (stream=%s)', + async (streaming) => { + const api = require('@librechat/api'); + const actual = jest.requireActual('@librechat/api'); + const names = [ + 'createChunk', + 'writeSSE', + 'sendFinalChunk', + 'buildNonStreamingResponse', + 'createOpenAIStreamTracker', + 'createOpenAIContentAggregator', + ]; + const originals = names.map((name) => [name, api[name].getMockImplementation()]); + for (const name of names) api[name].mockImplementation(actual[name]); + req.body.stream = streaming; + req.body.include = ['server_tool_results']; + let completion; + api.validateRequest.mockReturnValueOnce({ request: req.body }); + mockProcessStream.mockImplementationOnce(async () => { + const { customHandlers: handlers, eagerToolExecution } = api.createRun.mock.calls.at(-1)[0]; + expect(eagerToolExecution).toBe( + req.body.include?.includes('server_tool_results') ? false : undefined, + ); + await handlers.on_model_response.handle('on_model_response', { + type: 'model_response', + id: 'accepted-server', + agentId: 'agent-123', + toolCalls: [{ id: 'lookup-1', name: 'lookup', args: { city: 'Madrid' } }], + toolCallDispositions: ['sdk'], + invalidToolCalls: [], + }); + completion = handlers.on_run_step_completed; + await completion?.handle( + 'on_run_step_completed', + { + result: { + id: 'step-1', + type: 'tool_call', + tool_call: { + id: 'lookup-1', + name: 'lookup', + args: '{"city":"Madrid"}', + output: 'Sunny', + }, + }, + }, + { agentId: 'agent-123' }, + ); + await handlers.on_message_delta.handle('on_message_delta', { + delta: { content: [{ type: 'text', text: 'It is sunny.' }] }, + }); + }); + try { + await OpenAIChatCompletionController(req, res); + expect(completion).toBeDefined(); + const expected = { + call_id: 'lookup-1', + name: 'lookup', + arguments: '{"city":"Madrid"}', + output: 'Sunny', + }; + if (streaming) { + const frames = res.write.mock.calls + .map(([frame]) => frame) + .filter((frame) => frame !== 'data: [DONE]\n\n') + .map((frame) => JSON.parse(frame.slice(6))); + expect(frames.flatMap((frame) => frame.choices[0].delta.tool_results ?? [])).toEqual([ + { index: 0, ...expected }, + ]); + expect(frames.flatMap((frame) => frame.choices[0].delta.tool_calls ?? [])).toEqual([]); + expect(frames.at(-1).choices[0].finish_reason).toBe('stop'); + } else { + const choice = res.json.mock.calls[0][0].choices[0]; + expect(choice.message.tool_results).toEqual([expected]); + expect(choice.message.tool_calls).toBeUndefined(); + expect(choice.finish_reason).toBe('stop'); + } + } finally { + for (const [name, implementation] of originals) + api[name].mockImplementation(implementation); + } + }, + ); + it.each([true, false])( 'publishes accepted client calls after text and a model claim (stream=%s)', async (streaming) => { @@ -697,7 +787,10 @@ describe('OpenAIChatCompletionController', () => { req.body.stream = streaming; api.validateRequest.mockReturnValueOnce({ request: req.body }); mockProcessStream.mockImplementationOnce(async () => { - const { customHandlers: handlers } = api.createRun.mock.calls.at(-1)[0]; + const { customHandlers: handlers, eagerToolExecution } = api.createRun.mock.calls.at(-1)[0]; + expect(eagerToolExecution).toBe( + req.body.include?.includes('server_tool_results') ? false : undefined, + ); await handlers.on_model_response.handle('on_model_response', { type: 'model_response', id: 'discarded', @@ -1030,6 +1123,7 @@ describe('OpenAIChatCompletionController', () => { expect.anything(), mockCompletionUsage, true, + undefined, ); }); diff --git a/api/server/controllers/agents/openai.js b/api/server/controllers/agents/openai.js index aca53c54aa..8f22db1597 100644 --- a/api/server/controllers/agents/openai.js +++ b/api/server/controllers/agents/openai.js @@ -1,9 +1,6 @@ const { nanoid } = require('nanoid'); const { logger } = require('@librechat/data-schemas'); const { Callback, formatAgentMessages } = require('@librechat/agents'); -const { - createOpenAIToolCallStream: createAcceptedToolCallStream, -} = require('@librechat/agents/openai'); const { EModelEndpoint, ResourceType, @@ -17,6 +14,7 @@ const { createRemoteExecutionContext, getRemoteRunKeyId, createChunk, + createChatCompletionToolProjection, applyContextToAgent, buildRunToolSet, buildInitialToolSessions, @@ -1033,17 +1031,19 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { } }; - const toolCallStream = createAcceptedToolCallStream({ + const projection = createChatCompletionToolProjection({ + streaming: isStreaming, signal: execution.signal, - toolCalls: isStreaming ? tracker.toolCalls : aggregator.toolCalls, - ...(isStreaming && { - emit: (delta) => writeSSE(res, createChunk(context, delta)), - }), + tracker, + aggregator, + writer: res, + context, + include: request.include, }); // Event handlers for OpenAI-compatible streaming const handlers = { - ...toolCallStream.handlers, + ...projection.handlers, // Text content streaming on_message_delta: createHandler((data) => { const content = data?.delta?.content; @@ -1145,6 +1145,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { const initialSessions = buildInitialToolSessions({ agents: runAgents }); const run = await createRun({ + ...projection.runOptions, sdkModelHost: req?.app?.locals?.sdkModelHost, agents: runAgents, messages: formattedMessages, @@ -1191,7 +1192,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { version: 'v2', }; - await completeOpenAIToolCalls(toolCallStream, async () => { + await completeOpenAIToolCalls(projection, async () => { await run.processStream({ messages: formattedMessages }, config, { callbacks: { [Callback.TOOL_ERROR]: (graph, error, toolId) => { @@ -1277,6 +1278,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { aggregator.toolCalls, usage, true, + projection.response(), ); res.json(response); logger.debug( diff --git a/e2e/config/librechat.e2e.yaml b/e2e/config/librechat.e2e.yaml index 3578623298..c907352cad 100644 --- a/e2e/config/librechat.e2e.yaml +++ b/e2e/config/librechat.e2e.yaml @@ -106,7 +106,9 @@ mcpServers: timeout: 30000 e2e-http: type: streamable-http - url: http://127.0.0.1:8765/mcp + url: &e2e-http-url http://127.0.0.1:8765/mcp + # The fixture is explicitly anonymous; enrollment and execution bind the same auth descriptor. + requiresOAuth: false title: E2E HTTP description: Local HTTP MCP fixture for allowlist-override e2e tests. timeout: 30000 @@ -183,6 +185,15 @@ endpoints: # Default capabilities plus run_in_background (off by default upstream) so the # background tool-call e2e can opt an MCP tool into detached dispatch. agents: + remoteApi: + integrations: + resources: + e2e-http: + url: *e2e-http-url + credentialMode: anonymous + issuer: null + audience: null + scopes: [] sdkHostFiles: {} sdkRuntimes: remote-example: diff --git a/e2e/setup/fake-mcp-http-server.js b/e2e/setup/fake-mcp-http-server.js index 93003e1718..ac1d467e22 100644 --- a/e2e/setup/fake-mcp-http-server.js +++ b/e2e/setup/fake-mcp-http-server.js @@ -14,6 +14,7 @@ */ const http = require('http'); +const { z } = require('zod'); const { randomUUID } = require('crypto'); const { keepIdleConnections } = require('./keepalive'); const { McpServer } = require('@modelcontextprotocol/sdk/server/mcp.js'); @@ -36,6 +37,17 @@ function createMcpServer() { content: [{ type: 'text', text: 'E2E HTTP MCP pong' }], }), ); + // URL-backed tool result fixture: delegated API-key grants require a declared recipient. + server.registerTool( + 'remember_fact', + { + description: 'Records a deterministic fact for API tool result tests.', + inputSchema: { fact: z.string().optional() }, + }, + async ({ fact = 'LibreChat MCP e2e fact' }) => ({ + content: [{ type: 'text', text: `E2E MCP memory noted: ${fact}` }], + }), + ); return server; } diff --git a/e2e/specs/mock/scenarios/chat-tool-results.spec.ts b/e2e/specs/mock/scenarios/chat-tool-results.spec.ts new file mode 100644 index 0000000000..ae53e75166 --- /dev/null +++ b/e2e/specs/mock/scenarios/chat-tool-results.spec.ts @@ -0,0 +1,206 @@ +import { randomUUID } from 'node:crypto'; +import { expect, test } from '@playwright/test'; +import type { APIRequestContext } from '@playwright/test'; +import { loginAdmin, restoreRuntimeFilters, setRuntimeFilters } from '../content-filters.helpers'; + +type Result = { call_id: string; name: string; arguments: string; output: string; index?: number }; + +async function remote(request: APIRequestContext) { + const token = await loginAdmin(request); + const headers = { Authorization: `Bearer ${token}` }; + const profile = await request.get('/api/user', { headers }); + expect(profile.status()).toBe(200); + const user = await profile.json(); + const userId = user.id ?? user._id; + const origin = `http://127.0.0.1:${process.env.E2E_MCP_HTTP_PORT || '8765'}`; + const overridePath = `/api/admin/config/user/${userId}`; + const created: { agentId?: string; keyId?: string } = {}; + const cleanup = async () => { + if (created.keyId) await request.delete(`/api/api-keys/${created.keyId}`, { headers }); + if (created.agentId) await request.delete(`/api/agents/${created.agentId}`, { headers }); + const cleared = await request.delete(overridePath, { headers }); + expect([200, 204, 404]).toContain(cleared.status()); + }; + try { + const configured = await request.put(overridePath, { + headers, + data: { + overrides: { + mcpSettings: { allowedDomains: [origin] }, + }, + }, + }); + expect(configured.ok(), await configured.text()).toBe(true); + await expect + .poll( + async () => { + const initialized = await request.post('/api/mcp/e2e-http/reinitialize', { headers }); + return initialized.ok() && (await initialized.json()).success === true; + }, + { timeout: 30000, intervals: [500, 1000] }, + ) + .toBe(true); + const agent = await request.post('/api/agents', { + headers, + data: { + name: `Tool results ${randomUUID().slice(0, 8)}`, + provider: 'Mock Provider A', + model: 'mock-model-a', + model_parameters: {}, + tools: ['remember_fact_mcp_e2e-http'], + }, + }); + expect(agent.status(), await agent.text()).toBe(201); + const agentId = (await agent.json()).id; + created.agentId = agentId; + const key = await request.post('/api/api-keys', { + headers, + data: { name: 'tool result contract' }, + }); + expect(key.status()).toBe(201); + const credentials = await key.json(); + created.keyId = credentials.id; + // The API key owns a narrow per-agent grant; observation must not bypass MCP authorization. + const consentPath = `/api/api-keys/${credentials.id}/agents/${agentId}/mcp-consent`; + const offered = await request.get(consentPath, { headers }); + expect(offered.status(), await offered.text()).toBe(200); + const view = await offered.json(); + expect(view.offer).not.toBeNull(); + const target = view.targets.find( + (entry: { resource: { serverName: string } }) => entry.resource.serverName === 'e2e-http', + ); + expect(target).toBeDefined(); + const selected = target.permittedTools.find( + (entry: { agentId: string }) => entry.agentId === agentId, + ); + const tools = selected.tools.filter( + (name: string) => name === 'remember_fact' || name === 'remember_fact_mcp_e2e-http', + ); + expect(tools).toHaveLength(1); + const granted = await request.post(consentPath, { + headers, + data: { + offerDigest: view.offer.digest, + expectedRevision: view.revision, + lifetimeHours: 1, + allow: { 'e2e-http': tools }, + }, + }); + expect(granted.status(), await granted.text()).toBe(200); + + return { + token, + agentId, + apiKey: credentials.key as string, + cleanup, + }; + } catch (error) { + await cleanup(); + throw error; + } +} + +async function execute( + request: APIRequestContext, + agent: Awaited>, + stream: boolean, + include: boolean, +) { + const label = randomUUID().replace(/-/g, '').slice(0, 10); + const response = await request.post('/api/agents/v1/chat/completions', { + headers: { Authorization: `Bearer ${agent.apiKey}` }, + data: { + model: agent.agentId, + messages: [{ role: 'user', content: `E2E_ACTIVITY_REPLY:${label}` }], + stream, + ...(include && { include: ['server_tool_results'] }), + }, + timeout: 60000, + }); + const text = await response.text(); + expect(response.status(), text).toBe(200); + if (!stream) { + const choice = JSON.parse(text).choices[0]; + expect(choice.finish_reason).toBe('stop'); + expect(choice.message.content).toContain(`E2E activity reply done ${label}`); + expect(choice.message.tool_calls).toBeUndefined(); + if (!include) expect(choice.message.tool_results).toBeUndefined(); + return { label, results: (choice.message.tool_results ?? []) as Result[], text }; + } + const frames = text + .split('\n\n') + .map((frame) => frame.trim()) + .filter((frame) => frame.startsWith('data: ') && frame !== 'data: [DONE]') + .map((frame) => JSON.parse(frame.slice(6))); + expect(frames[frames.length - 1].choices[0].finish_reason).toBe('stop'); + expect(frames.map((frame) => frame.choices[0].delta.content ?? '').join('')).toContain( + `E2E activity reply done ${label}`, + ); + expect(frames.flatMap((frame) => frame.choices[0].delta.tool_calls ?? [])).toEqual([]); + return { + label, + results: frames.flatMap((frame) => frame.choices[0].delta.tool_results ?? []) as Result[], + text, + }; +} + +test('Chat tool result observation preserves execution ownership in JSON and SSE @scenario:chat-server-tool-results', async ({ + playwright, + baseURL, +}) => { + test.setTimeout(180000); + const request = await playwright.request.newContext({ baseURL }); + const agent = await remote(request); + try { + for (const stream of [false, true]) + for (const include of [false, true]) { + const { label, results } = await execute(request, agent, stream, include); + expect(results).toHaveLength(include ? 2 : 0); + if (!include) continue; + expect(new Set(results.map((result) => result.call_id)).size).toBe(2); + for (const result of results) { + expect(result.name).toBe('remember_fact_mcp_e2e-http'); + expect(JSON.parse(result.arguments).fact).toContain(label); + expect(result.output).toContain('E2E MCP memory noted:'); + } + if (stream) expect(results.map((result) => result.index)).toEqual([0, 1]); + } + } finally { + await agent.cleanup(); + await request.dispose(); + } +}); + +test('Chat observed results retain the tool-output protection policy @scenario:chat-tool-results-protected', async ({ + playwright, + baseURL, +}) => { + test.setTimeout(120000); + const request = await playwright.request.newContext({ baseURL }); + const agent = await remote(request); + try { + await setRuntimeFilters(request, agent.token, { + toolArguments: { + pii: { + fields: ['output'], + starterPatterns: [], + customPatterns: [ + { id: 'tool-result-probe', label: 'Tool result probe', regex: 'E2E MCP memory noted:' }, + ], + }, + }, + }); + for (const stream of [false, true]) { + const { results, text } = await execute(request, agent, stream, true); + expect(results).toHaveLength(2); + expect(text).not.toContain('E2E MCP memory noted:'); + expect(text).not.toContain('mcp_reauth_required'); + expect(results.every((result) => /blocked/i.test(result.output))).toBe(true); + expect(results.every((result) => result.output.length > 0)).toBe(true); + } + } finally { + await restoreRuntimeFilters(request, agent.token); + await agent.cleanup(); + await request.dispose(); + } +}); diff --git a/packages/api/src/agents/__tests__/run-codeTools.test.ts b/packages/api/src/agents/__tests__/run-codeTools.test.ts index fe2bc395e1..b851865290 100644 --- a/packages/api/src/agents/__tests__/run-codeTools.test.ts +++ b/packages/api/src/agents/__tests__/run-codeTools.test.ts @@ -522,6 +522,20 @@ describe('createRun code-tool eager/session wiring', () => { ); }); + it('lets observed Chat results defer tool execution until model acceptance', async () => { + await createRun({ + agents: [makeAgent()] as never, + signal: new AbortController().signal, + eagerToolExecution: false, + }); + expect((Run.create as jest.Mock).mock.calls[0][0].eagerEventToolExecution).toEqual( + expect.objectContaining({ + enabled: false, + excludeToolNames: expect.arrayContaining(['create_file', 'execute_code']), + }), + ); + }); + it('declares create_file/edit_file/read_file as code-session participants', async () => { const runConfig = await captureRunConfig(); expect(runConfig.codeSessionToolNames).toEqual( diff --git a/packages/api/src/agents/openai/README.md b/packages/api/src/agents/openai/README.md index d9cb6b293b..3b56783dd5 100644 --- a/packages/api/src/agents/openai/README.md +++ b/packages/api/src/agents/openai/README.md @@ -1,6 +1,6 @@ # Agents API response projection -This module projects an **agent run**, which can include several model invocations and server-executed tools, onto an OpenAI-compatible response. It is not a transparent proxy for a single model completion. Only explicitly client-owned calls in graph-accepted model responses may become outward `tool_calls`; SDK- and provider-executed calls stay inside the run. Binding caller-supplied tool schemas and client execution remains a separate feature. +This module projects an **agent run**, which can include several model invocations and server-executed tools, onto an OpenAI-compatible response. It is not a transparent proxy for a single model completion. Only explicitly client-owned calls in graph-accepted model responses may become outward `tool_calls`; SDK- and provider-executed calls never become client execution requests. Completed SDK executions can be observed separately with the opt-in extension below. Binding caller-supplied tool schemas and client execution remains a separate feature. ## Owners and boundaries @@ -69,3 +69,64 @@ The SDK projector accepts only client-disposition calls from graph-accepted resp - `service.spec.ts` and the Express controller spec: both response modes, actual serializers, internal-call suppression, accepted client calls, claim discard and failure behavior. These tests do not run a live provider, a distributed multi-agent deployment or a persisted resume. The graph test executes a local tool but does not exercise the mounted HTTP server. There is no new database, cache, authorization policy or migration. Code and built packages must deploy together. + +## Opt-in server execution results + +Both Chat hosts accept the LibreChat extension `include: ["server_tool_results"]`. +It adds completed execution records to `choices[0].message.tool_results` in JSON, +or `choices[0].delta.tool_results` in SSE. A record is `{ call_id, name, arguments, +output }`, all strings; streamed records also carry a stable zero-based completion +`index`. JSON includes an empty array if no results are observed. Omitted `include` +or `include: []` omits the fields. Unknown/malformed selectors return 400 before +execution; duplicate supported selectors are harmless. + +```text +accepted model response + sdk disposition (owner/call identity) + -> SDK ToolNode processed completion + authoritative metadata.agentId + -> run-local bounded observation projector + -> JSON tool_results / indexed SSE delta.tool_results +``` + +For this selector, both Chat hosts disable the SDK's speculative eager tool +prestart. Tools execute after model acceptance, so the observer receives final +processed ToolNode results rather than trying to associate a failed provider +attempt's early output with an accepted fallback. Normal Chat requests and the +Responses API retain their existing execution policy. Custom `createRun` adapters +must forward the supplied `eagerToolExecution: false` option. + +This trades tool/model-stream overlap for authoritative result observation: +opted-in tools start after the model finishes its accepted call, and failed +provider attempts do not speculatively execute them. Tool latency may therefore +be higher for opted-in requests; there is no production latency estimate. + +The collector never changes execution requests, tool ownership, `finish_reason`, +authorization, billing or artifact callbacks. It consumes only accepted SDK-owned +calls with matching completed execution owner, name and call ID. Pinned SDK 4.2.10 +`ToolNode.withToolScope` stamps `metadata.agentId`; `Run` forwards it unchanged. +Missing owner metadata fails closed with an incomplete flag: an ownerless result +cannot be inferred safely from a pending ID when provider/client siblings reuse +that ID. Acceptance deduplication is also scoped by execution owner. + +Only processed string arguments/results are exposed; no raw ToolMessage, artifact +or graph-state serialization is performed. Existing tool-output protection and +SDK truncation remain authoritative. Provider-hosted tools without SDK completion +and unfinished/client-executed calls have no invented records. These observations +are distinct from the actionable client `tool_calls` projection above. Responses +API output, persistence and retrieval are unchanged; this selector applies only +to Chat Completions and is not an Open Responses requirement. + +Projection-only safety guards bound capture to 1024 accepted server calls and +4 MiB of identity/argument/output UTF-8 text per run, matching the SDK client-call +projection envelope. They are independent of tool execution limits; exhaustion +fences additional observation while preserving completed records and normal graph +success. Smaller guards can be injected into the collector for constrained hosts +and focused tests. Overflow or unattributable ownership adds +`message.tool_results_incomplete: true` in JSON or +`delta.tool_results_incomplete: true` on the terminal SSE chunk. Such records are +not a complete audit log. Partial SSE records also do not imply run success if the +usual execution error follows. No durable Chat storage is introduced. + +Successful records survive the existing `completeOpenAIToolCalls` finally-abort +cleanup after `finish`. Failure/cancellation clears unpublished records; late and +duplicate completions cannot append results. Accepted/completed identity state is +released at the run boundary. diff --git a/packages/api/src/agents/openai/handlers.ts b/packages/api/src/agents/openai/handlers.ts index 01c9b500dd..dbe513fdce 100644 --- a/packages/api/src/agents/openai/handlers.ts +++ b/packages/api/src/agents/openai/handlers.ts @@ -1,3 +1,5 @@ +import type { ServerToolResults } from './results'; +import { createServerToolResults } from './results'; /** * OpenAI-compatible event handlers for agent streaming. * @@ -12,6 +14,7 @@ import type { OpenAIResponseContext, ChatCompletionChunk, CompletionUsage, + ServerToolResultFields, ToolCall, } from './types'; import type { UsageMetadata } from '~/stream/interfaces/IJobStore'; @@ -63,6 +66,8 @@ export function writeSSE(writer: OpenAIWriter, data: ChatCompletionChunk | strin */ export interface OpenAIStreamTracker { /** Successful response completion, installed by the response handlers. */ + serverToolResults?: ServerToolResults; + toolExecution?: { eagerToolExecution?: false }; finishToolCalls?: () => void; abortToolCalls?: () => void; /** Whether any text content was emitted */ @@ -113,6 +118,8 @@ export function createOpenAIStreamTracker(): OpenAIStreamTracker { */ export interface OpenAIContentAggregator { /** Successful response completion, installed by the response handlers. */ + serverToolResults?: ServerToolResults; + toolExecution?: { eagerToolExecution?: false }; finishToolCalls?: () => void; abortToolCalls?: () => void; /** Accumulated text chunks */ @@ -162,6 +169,7 @@ export function createOpenAIContentAggregator(): OpenAIContentAggregator { /** Legacy stream configuration; retained for existing package consumers. */ export interface OpenAIStreamHandlerConfig { + include?: 'server_tool_results'[]; signal?: AbortSignal; res: OpenAIWriter; context: OpenAIResponseContext; @@ -766,29 +774,78 @@ export class OpenAIReasoningDeltaHandler implements EventHandler { * Create all handlers for OpenAI streaming format */ export interface OpenAIAggregationHandlerConfig { + include?: 'server_tool_results'[]; aggregator: OpenAIContentAggregator; signal?: AbortSignal; } type OpenAIContentHandlerConfig = OpenAIContentStreamConfig | OpenAIAggregationHandlerConfig; +type ChatCompletionToolProjectionConfig = { + include?: 'server_tool_results'[]; + signal?: AbortSignal; +} & ( + | { + streaming: true; + tracker: OpenAIStreamTracker; + writer: OpenAIWriter; + context: OpenAIResponseContext; + aggregator?: OpenAIContentAggregator; + } + | { + streaming: false; + aggregator: OpenAIContentAggregator; + tracker?: OpenAIStreamTracker; + writer?: OpenAIWriter; + context?: OpenAIResponseContext; + } +); + +/** Both HTTP hosts share selection, execution policy, framing and terminal ownership. */ +export function createChatCompletionToolProjection( + config: ChatCompletionToolProjectionConfig, +): ReturnType & { + runOptions: { eagerToolExecution?: false }; + response(): ServerToolResultFields | undefined; +} { + const target = config.streaming ? config.tracker : config.aggregator; + const emit = config.streaming + ? (delta: ChatCompletionChunkChoice['delta']) => + writeSSE(config.writer, createChunk(config.context, delta)) + : undefined; + const accepted = createAcceptedToolCallStream({ + signal: config.signal, + toolCalls: target.toolCalls, + emit, + }); + const results = config.include?.includes('server_tool_results') + ? createServerToolResults({ signal: config.signal, emit }) + : undefined; + const projection = results?.wrap(accepted) ?? accepted; + // Opt-in observation needs accepted, processed ToolNode completions. Eager + // prestart emits before acceptance and can reuse a failed provider attempt. + const runOptions = results ? { eagerToolExecution: false as const } : {}; + target.serverToolResults = results; + target.toolExecution = runOptions; + target.finishToolCalls = projection.finish; + target.abortToolCalls = projection.abort; + return { ...projection, runOptions, response: () => results?.response() }; +} + export function createOpenAIHandlers( config: OpenAIContentHandlerConfig, toolExecuteOptions?: ToolExecuteOptions, ): Record { - const target = 'aggregator' in config ? config.aggregator : config.tracker; - const toolCallProjection = createAcceptedToolCallStream({ - signal: config.signal, - toolCalls: target.toolCalls, - emit: - 'aggregator' in config - ? undefined - : (delta) => writeSSE(getStreamWriter(config), createChunk(config.context, delta)), - }); - target.finishToolCalls = toolCallProjection.finish; - target.abortToolCalls = toolCallProjection.abort; + const projection = + 'aggregator' in config + ? createChatCompletionToolProjection({ ...config, streaming: false }) + : createChatCompletionToolProjection({ + ...config, + streaming: true, + writer: getStreamWriter(config), + }); const handlers: Record = { - ...toolCallProjection.handlers, + ...projection.handlers, [GraphEvents.ON_MESSAGE_DELTA]: new OpenAIMessageDeltaHandler(config), [GraphEvents.CHAT_MODEL_END]: new OpenAIModelEndHandler(config), [GraphEvents.CHAT_MODEL_STREAM]: new OpenAIChatModelStreamHandler(), @@ -837,7 +894,12 @@ export function sendFinalChunk( }; } - const finalChunk = createChunk(context, {}, reason, usage); + const finalChunk = createChunk( + context, + tracker.serverToolResults?.incomplete ? { tool_results_incomplete: true } : {}, + reason, + usage, + ); writeSSE(writer, finalChunk); // Send [DONE] marker diff --git a/packages/api/src/agents/openai/index.ts b/packages/api/src/agents/openai/index.ts index 3a0a016108..ad475c69ba 100644 --- a/packages/api/src/agents/openai/index.ts +++ b/packages/api/src/agents/openai/index.ts @@ -50,3 +50,5 @@ export { type ChatCompletionValidationSuccess, type ChatCompletionValidationFailure, } from './service'; + +export { createServerToolResults } from './results'; diff --git a/packages/api/src/agents/openai/results.spec.ts b/packages/api/src/agents/openai/results.spec.ts new file mode 100644 index 0000000000..fdbf5726b1 --- /dev/null +++ b/packages/api/src/agents/openai/results.spec.ts @@ -0,0 +1,301 @@ +import { createOpenAIToolCallStream } from '@librechat/agents/openai'; +import { createServerToolResults, validateChatIncludes } from './results'; +import { createOpenAIStreamTracker, sendFinalChunk } from './handlers'; +import { buildNonStreamingResponse } from './service'; + +function fixture(options: Parameters[0] = {}) { + const collector = createServerToolResults(options); + const calls = new Map(); + const delegate = createOpenAIToolCallStream({ toolCalls: calls }); + const projection = collector.wrap(delegate); + const accept = async (agent = 'a', id = 'call', disposition = 'sdk') => { + await projection.handlers.on_model_response.handle('on_model_response', { + type: 'model_response', + id: `accepted-${agent}-${id}`, + agentId: agent, + toolCalls: [{ id, name: 'lookup', args: {} }], + toolCallDispositions: [disposition], + invalidToolCalls: [], + } as never); + }; + const complete = async ( + agent = 'a', + id = 'call', + output = 'protected result', + step = `step-${agent}-${id}`, + ) => { + await projection.handlers.on_run_step_completed.handle( + 'on_run_step_completed', + { + result: { + id: step, + type: 'tool_call', + tool_call: { + id, + name: 'lookup', + args: '{}', + output, + artifact: { private: 'secret' }, + }, + }, + } as never, + { agentId: agent } as never, + ); + }; + return { collector, calls, delegate, projection, accept, complete }; +} + +describe('server tool result projection', () => { + it('validates an explicit extension selector', () => { + for (const value of [ + undefined, + [], + ['server_tool_results'], + ['server_tool_results', 'server_tool_results'], + ]) + expect(validateChatIncludes(value)).toBeUndefined(); + for (const value of [null, true, 'server_tool_results', [3], ['unknown']]) + expect(validateChatIncludes(value)).toEqual(expect.any(String)); + }); + + it('correlates SDK completions, deduplicates and never changes client execution requests', async () => { + const f = fixture(); + await f.accept(); + await f.accept('a', 'client', 'client'); + await f.accept('a', 'provider', 'provider'); + await f.complete('a', 'client'); + await f.complete('a', 'provider'); + await f.complete('a', 'unknown'); + await f.complete(); + await f.complete(); + await f.accept(); + await f.complete(); + expect(f.collector.response()).toEqual({ + tool_results: [ + { call_id: 'call', name: 'lookup', arguments: '{}', output: 'protected result' }, + ], + }); + expect(JSON.stringify(f.collector.response())).not.toContain('secret'); + // The final accepted internal-only turn supersedes pending client requests as usual. + expect(f.calls.size).toBe(0); + }); + + it('keeps parallel calls and identical IDs from different agents separate', async () => { + const f = fixture(); + await f.accept('a', 'same'); + await f.accept('b', 'same'); + await Promise.all([f.complete('b', 'same', 'B'), f.complete('a', 'same', 'A')]); + expect(f.collector.response().tool_results.map((result) => result.output)).toEqual(['B', 'A']); + }); + + it('refuses ambiguous ownerless completions and unaccepted identities', async () => { + const f = fixture(); + await f.accept('a', 'same'); + await f.accept('b', 'same'); + await f.projection.handlers.on_run_step_completed.handle('on_run_step_completed', { + result: { + id: 'step', + type: 'tool_call', + tool_call: { id: 'same', name: 'lookup', args: '{}', output: 'private sibling' }, + }, + }); + await f.complete('foreign', 'same', 'foreign result'); + expect(f.collector.response()).toEqual({ tool_results: [], tool_results_incomplete: true }); + }); + + it('never reattributes an ownerless duplicate after its sibling completed', async () => { + const f = fixture(); + await f.accept('a', 'same'); + await f.accept('b', 'same'); + await f.complete('a', 'same', 'A', 'step-a'); + await f.projection.handlers.on_run_step_completed.handle('on_run_step_completed', { + result: { + id: 'step-a', + type: 'tool_call', + tool_call: { id: 'same', name: 'lookup', args: '{}', output: 'A' }, + }, + }); + await f.complete('b', 'same', 'B'); + expect(f.collector.response().tool_results.map((result) => result.output)).toEqual(['A', 'B']); + expect(f.collector.incomplete).toBe(true); + }); + + it.each(['provider', 'client'])( + 'never attributes an ownerless %s sibling to a server execution', + async (disposition) => { + const f = fixture(); + await f.accept('a', 'same', 'sdk'); + await f.accept('b', 'same', disposition); + await f.projection.handlers.on_run_step_completed.handle('on_run_step_completed', { + result: { + id: 'step-b', + type: 'tool_call', + tool_call: { id: 'same', name: 'lookup', args: '{}', output: 'private B' }, + }, + }); + await f.complete('a', 'same', 'A'); + expect(f.collector.response().tool_results.map((result) => result.output)).toEqual(['A']); + expect(f.collector.incomplete).toBe(true); + }, + ); + + it('deduplicates model acceptance within its execution owner', async () => { + const f = fixture(); + for (const agentId of ['a', 'b']) { + await f.projection.handlers.on_model_response.handle('on_model_response', { + type: 'model_response', + id: 'same-acceptance', + agentId, + toolCalls: [{ id: 'same', name: 'lookup', args: {} }], + toolCallDispositions: ['sdk'], + invalidToolCalls: [], + }); + } + await f.complete('a', 'same', 'A'); + await f.complete('b', 'same', 'B'); + expect(f.collector.response().tool_results.map((result) => result.output)).toEqual(['A', 'B']); + }); + + it('fences an oversized completed output without serializing artifacts or failing the graph', async () => { + const f = fixture({ maxBytes: 100 }); + await f.accept(); + await f.complete('a', 'call', 'x'.repeat(200)); + await f.complete('a', 'call', 'late'); + expect(() => f.projection.finish()).not.toThrow(); + expect(f.collector.response()).toEqual({ tool_results: [], tool_results_incomplete: true }); + }); + + it('emits complete indexed records once and fences results on cancellation', async () => { + const controller = new AbortController(); + const emit = jest.fn(); + const f = fixture({ signal: controller.signal, emit }); + await f.accept(); + await f.complete(); + controller.abort(); + await f.complete('a', 'late'); + expect(emit).toHaveBeenCalledTimes(1); + expect(emit.mock.calls[0][0].tool_results[0]).toEqual( + expect.objectContaining({ index: 0, output: 'protected result' }), + ); + expect(f.collector.response().tool_results).toEqual([]); + }); + + it.each([{ maxCalls: 1 }, { maxBytes: 32 }])( + 'marks overflow without failing execution (%j)', + async (limits) => { + const emit = jest.fn(); + const f = fixture({ ...limits, emit }); + await f.accept('a', 'one'); + await f.complete('a', 'one', 'first'); + await f.accept('a', 'two'); + await f.complete('a', 'two', 'second'); + expect(() => f.projection.finish()).not.toThrow(); + expect(f.collector.response().tool_results_incomplete).toBe(true); + expect(f.collector.response().tool_results.length).toBeLessThan(2); + }, + ); + + it('reports the same overflow flag in terminal SSE and JSON without changing stop', async () => { + const f = fixture({ maxCalls: 1 }); + await f.accept('a', 'one'); + await f.complete('a', 'one'); + await f.accept('a', 'two'); + const tracker = createOpenAIStreamTracker(); + tracker.serverToolResults = f.collector; + tracker.finishToolCalls = f.projection.finish; + const frames: string[] = []; + const context = { requestId: 'test', created: 1, model: 'fixture' }; + sendFinalChunk({ + tracker, + context, + writer: { + write: (frame) => { + frames.push(frame); + }, + }, + }); + const terminal = JSON.parse(frames[frames.length - 2].slice(6)); + expect(terminal.choices[0].delta).toEqual({ tool_results_incomplete: true }); + expect(terminal.choices[0].finish_reason).toBe('stop'); + const response = buildNonStreamingResponse( + context, + 'done', + '', + new Map(), + { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, + true, + f.collector.response(), + ); + expect(response.choices[0].message.tool_results_incomplete).toBe(true); + expect(response.choices[0].message.tool_results).toHaveLength(1); + expect(response.choices[0].finish_reason).toBe('stop'); + }); + + it('retains successful output through the SDK lifecycle cleanup', async () => { + const f = fixture(); + await f.accept(); + await f.complete(); + f.projection.finish(); + f.projection.abort(); + expect(f.collector.response().tool_results).toHaveLength(1); + }); + + it('discards unpublished results when cancellation has no following event', async () => { + const controller = new AbortController(); + const f = fixture({ signal: controller.signal }); + await f.accept(); + await f.complete(); + controller.abort(); + expect(() => f.projection.finish()).toThrow(expect.objectContaining({ name: 'AbortError' })); + f.projection.abort(); + expect(f.collector.response()).toEqual({ tool_results: [] }); + }); + + it('does not seal success when the delegate cancels during finish', async () => { + const controller = new AbortController(); + const f = fixture({ signal: controller.signal }); + jest.spyOn(f.delegate, 'finish').mockImplementation(() => controller.abort()); + await f.accept(); + await f.complete(); + expect(() => f.projection.finish()).toThrow(expect.objectContaining({ name: 'AbortError' })); + f.projection.abort(); + expect(f.collector.response()).toEqual({ tool_results: [] }); + }); + + it('does not finalize after the result sink cancels reentrantly', async () => { + const controller = new AbortController(); + const emit = jest.fn(() => controller.abort()); + const f = fixture({ signal: controller.signal, emit }); + const finish = jest.spyOn(f.delegate, 'finish'); + await f.accept(); + await f.complete(); + expect(() => f.projection.finish()).toThrow(expect.objectContaining({ name: 'AbortError' })); + f.projection.abort(); + expect(emit).toHaveBeenCalledTimes(1); + expect(finish).not.toHaveBeenCalled(); + expect(f.collector.response()).toEqual({ tool_results: [] }); + }); + + it('seals a successful finish once before normal finally cleanup', async () => { + const controller = new AbortController(); + const f = fixture({ signal: controller.signal }); + const finish = jest.spyOn(f.delegate, 'finish'); + await f.accept(); + await f.complete(); + f.projection.finish(); + f.projection.finish(); + controller.abort(); + f.projection.abort(); + expect(finish).toHaveBeenCalledTimes(1); + expect(f.collector.response().tool_results).toHaveLength(1); + }); + + it('discards buffered results and ignores late events after abort', async () => { + const f = fixture(); + await f.accept(); + await f.complete(); + f.projection.abort(); + await f.complete(); + expect(f.collector.response()).toEqual({ tool_results: [] }); + }); +}); diff --git a/packages/api/src/agents/openai/results.ts b/packages/api/src/agents/openai/results.ts new file mode 100644 index 0000000000..46f53309ad --- /dev/null +++ b/packages/api/src/agents/openai/results.ts @@ -0,0 +1,227 @@ +import type { EventHandler } from '@librechat/agents'; +import type { ServerToolResult, ChatCompletionChunkChoice, ServerToolResultFields } from './types'; + +/** Public-payload safety guards, separate from tool execution limits. Exhaustion + * stops observation, never the graph. Match the SDK client-call projection's + * 1024-call/4-MiB envelope; hosts may inject smaller budgets for their transport. */ +const MAX_RESULTS = 1024; +const MAX_BYTES = 4 * 1024 * 1024; + +type Projection = { + handlers: Record; + finish: () => void; + abort: () => void; +}; + +type Options = { + signal?: AbortSignal; + emit?: (delta: ChatCompletionChunkChoice['delta']) => void; + maxCalls?: number; + maxBytes?: number; +}; + +function record(value: unknown): value is Record { + return value != null && typeof value === 'object' && !Array.isArray(value); +} + +export function validateChatIncludes(value: unknown): string | undefined { + if (value === undefined) return; + if (!Array.isArray(value) || value.some((item) => item !== 'server_tool_results')) + return 'include must be an array containing only server_tool_results'; +} + +export interface ServerToolResults { + readonly incomplete: boolean; + response(): ServerToolResultFields; + wrap(projection: T): T; +} + +/** Captures only SDK-accepted, server-executed completions. Raw tool-end content + * and artifacts are deliberately outside this public projection. */ +export function createServerToolResults({ + signal, + emit, + maxCalls = MAX_RESULTS, + maxBytes = MAX_BYTES, +}: Options = {}): ServerToolResults { + const pending = new Map(); + const announcements = new Set(); + const completed = new Set(); + const results: ServerToolResult[] = []; + let count = 0; + let bytes = 0; + let index = 0; + let incomplete = false; + let fenced = false; + let closed = false; + let finished = false; + + const release = () => { + pending.clear(); + announcements.clear(); + completed.clear(); + }; + const abort = () => { + closed = true; + release(); + if (!finished) { + results.length = 0; + incomplete = false; + } + }; + const active = () => { + if (!closed && signal?.aborted) abort(); + return !closed && !fenced; + }; + const reserve = (...values: string[]) => { + let size = 0; + for (const value of values) size += Buffer.byteLength(value, 'utf8'); + if (bytes + size > maxBytes) { + incomplete = fenced = true; + release(); + return false; + } + bytes += size; + return true; + }; + const accept = (data: unknown) => { + if ( + !active() || + !record(data) || + data.type !== 'model_response' || + typeof data.agentId !== 'string' || + typeof data.id !== 'string' || + !data.id || + !Array.isArray(data.toolCalls) || + !Array.isArray(data.toolCallDispositions) + ) + return; + if (data.agentId.length + data.id.length > maxBytes) { + incomplete = fenced = true; + release(); + return; + } + const announcement = JSON.stringify([data.agentId, data.id]); + if (announcements.has(announcement)) return; + const dispositions = data.toolCallDispositions; + if (dispositions.length !== data.toolCalls.length) return; + for (let i = 0; i < data.toolCalls.length; i++) { + if (dispositions[i] !== 'sdk') continue; + const call: unknown = data.toolCalls[i]; + if ( + !record(call) || + typeof call.id !== 'string' || + !call.id || + typeof call.name !== 'string' || + !call.name + ) + continue; + if (count >= maxCalls) { + incomplete = fenced = true; + release(); + return; + } + if (!reserve(data.agentId, call.id, call.name)) return; + if (!announcements.has(announcement)) { + if (!reserve(data.id)) return; + announcements.add(announcement); + } + const key = JSON.stringify([data.agentId, call.id]); + pending.set(key, { name: call.name }); + count++; + } + }; + const complete = (data: unknown, metadata: unknown) => { + if (!active() || !record(data) || !record(data.result)) return; + const step = data.result; + const call = step.tool_call; + if ( + step.type !== 'tool_call' || + typeof step.id !== 'string' || + !step.id || + !record(call) || + typeof call.id !== 'string' || + !call.id || + typeof call.args !== 'string' || + typeof call.output !== 'string' + ) + return; + // SDK 4.2.10 ToolNode.withToolScope stamps this execution owner on + // completion metadata. Never infer an owner from a pending call ID: + // provider/client siblings can reuse that ID without entering this collector. + const owner = record(metadata) ? metadata.agentId : undefined; + if (typeof owner !== 'string' || !owner) { + incomplete = true; + return; + } + if (call.id.length > maxBytes || step.id.length > maxBytes || owner.length > maxBytes) return; + const key = JSON.stringify([owner, call.id]); + const accepted = pending.get(key); + if (!accepted || call.name !== accepted.name) return; + const identity = JSON.stringify([key, step.id]); + if (completed.has(identity)) return; + if (!reserve(step.id, call.args, call.output)) return; + completed.add(identity); + pending.delete(key); + const result: ServerToolResult = { + call_id: call.id, + name: accepted.name, + arguments: call.args, + output: call.output, + }; + if (emit) emit({ tool_results: [{ index: index++, ...result }] }); + else results.push(result); + }; + return { + get incomplete(): boolean { + return incomplete; + }, + response(): ServerToolResultFields { + return { + tool_results: results, + ...(incomplete && { tool_results_incomplete: true as const }), + }; + }, + wrap(projection: T): T { + const accepted = projection.handlers.on_model_response; + return { + ...projection, + handlers: { + ...projection.handlers, + on_model_response: { + async handle(event, data, metadata, graph) { + await accepted.handle(event, data, metadata, graph); + if (event === 'on_model_response') accept(data); + }, + }, + on_run_step_completed: { + handle(event, data, metadata) { + if (event === 'on_run_step_completed') complete(data, metadata); + }, + }, + }, + finish() { + if (finished) return; + const check = () => { + if (!signal?.aborted) return; + abort(); + const error = new Error('Agent response aborted'); + error.name = 'AbortError'; + throw error; + }; + check(); + if (closed) return; + projection.finish(); + check(); + finished = true; + closed = true; + release(); + }, + abort() { + abort(); + projection.abort(); + }, + }; + }, + }; +} diff --git a/packages/api/src/agents/openai/service.spec.ts b/packages/api/src/agents/openai/service.spec.ts index 7d989c2952..b315e92d69 100644 --- a/packages/api/src/agents/openai/service.spec.ts +++ b/packages/api/src/agents/openai/service.spec.ts @@ -6,7 +6,7 @@ import type { CallbackHandlerMethods } from '@langchain/core/callbacks/base'; import type { FiltersConfig } from 'librechat-data-provider'; import type { ChatCompletionDependencies } from './service'; import type { ChatCompletionChunk } from './types'; -import { createAgentChatCompletion } from './service'; +import { createAgentChatCompletion, validateRequest } from './service'; jest.mock('@librechat/data-schemas', () => ({ logger: { @@ -150,6 +150,85 @@ describe('createAgentChatCompletion - MCP permission user propagation', () => { }, ); + it.each([true, false])( + 'returns opted-in server results in the packaged service (stream=%s)', + async (stream) => { + const req = createMockReq( + { id: 'user' }, + { + model: 'agent_test', + messages: [{ role: 'user', content: 'hi' }], + stream, + include: ['server_tool_results'], + }, + ); + const res = createMockRes(); + processStream.mockImplementationOnce(async () => { + const { customHandlers: handlers } = createRun.mock.calls[0][0] as Parameters< + NonNullable + >[0]; + await handlers.on_model_response.handle('on_model_response', { + type: 'model_response', + id: 'accepted-server', + agentId: 'agent_test', + toolCalls: [{ id: 'server', name: 'lookup', args: {} }], + toolCallDispositions: ['sdk'], + invalidToolCalls: [], + }); + await handlers.on_run_step_completed.handle( + 'on_run_step_completed', + { + result: { + id: 'step', + type: 'tool_call', + tool_call: { id: 'server', name: 'lookup', args: '{}', output: 'filtered result' }, + }, + }, + { agentId: 'agent_test' }, + ); + }); + await createAgentChatCompletion(req, res, deps); + expect(createRun.mock.calls[0][0]).toHaveProperty('eagerToolExecution', false); + const expected = { + call_id: 'server', + name: 'lookup', + arguments: '{}', + output: 'filtered result', + }; + if (stream) { + const chunks: ChatCompletionChunk[] = (res.write as jest.Mock).mock.calls + .map(([frame]: [string]) => frame) + .filter((frame) => frame !== 'data: [DONE]\n\n') + .map((frame) => JSON.parse(frame.slice(6))); + expect(chunks.flatMap((chunk) => chunk.choices[0].delta.tool_results ?? [])).toEqual([ + { index: 0, ...expected }, + ]); + expect(chunks.flatMap((chunk) => chunk.choices[0].delta.tool_calls ?? [])).toEqual([]); + expect(chunks[chunks.length - 1]?.choices[0].finish_reason).toBe('stop'); + } else { + const response = getResponseMock(res, 'json').mock.calls[0][0]; + expect(response.choices[0].message.tool_results).toEqual([expected]); + expect(response.choices[0].message.tool_calls).toBeUndefined(); + expect(response.choices[0].finish_reason).toBe('stop'); + } + }, + ); + + it.each([null, true, 'server_tool_results', ['unknown'], [3]])( + 'rejects malformed include before execution (%j)', + async (include) => { + const req = createMockReq( + { id: 'user' }, + { model: 'agent_test', messages: [{ role: 'user', content: 'hi' }], include }, + ); + const res = createMockRes(); + expect(validateRequest(req.body).valid).toBe(false); + await createAgentChatCompletion(req, res, deps); + expect(getResponseMock(res, 'status')).toHaveBeenCalledWith(400); + expect(createRun).not.toHaveBeenCalled(); + }, + ); + it.each([true, false])( 'keeps graph and provider-owned calls internal (stream=%s)', async (stream) => { diff --git a/packages/api/src/agents/openai/service.ts b/packages/api/src/agents/openai/service.ts index 5e2b38fb86..b85f1af049 100644 --- a/packages/api/src/agents/openai/service.ts +++ b/packages/api/src/agents/openai/service.ts @@ -1,5 +1,7 @@ import type { LocatorTraversalReporter } from '../../protection/diagnostics'; import type { SdkModelHost } from '../remoteSdk/builder'; +import type { ServerToolResultFields } from './types'; +import { validateChatIncludes } from './results'; /** * OpenAI-compatible chat completions service for agents. * @@ -301,6 +303,7 @@ type CreateRunAppConfig = Pick< type CreateRunFn = (params: { sdkModelHost?: SdkModelHost; + eagerToolExecution?: boolean; modelCallbacks?: ReturnType; agents: unknown[]; messages: unknown[]; @@ -523,6 +526,8 @@ export function validateRequest(body: unknown): ChatCompletionValidationResult { } const request = body as Record; + const includeError = validateChatIncludes(request.include); + if (includeError) return { valid: false, error: includeError }; if (!request.model || typeof request.model !== 'string') { return { valid: false, error: 'model (agent_id) is required' }; @@ -603,6 +608,7 @@ export function buildNonStreamingResponse( usage: CompletionUsage, /** True when the map contains only accepted client-owned calls, not legacy run-step history. */ acceptedToolCallsOnly = false, + results?: ServerToolResultFields, ): ChatCompletionResponse { const toolCallsArray = Array.from(toolCalls.values()); const finishReason = @@ -619,6 +625,7 @@ export function buildNonStreamingResponse( message: { role: 'assistant', content: text || null, + ...results, ...(reasoning && { reasoning }), ...(toolCallsArray.length > 0 && { tool_calls: toolCallsArray }), }, @@ -872,9 +879,10 @@ export async function createAgentChatCompletion( // Create event handlers const eventHandlers = createOpenAIHandlers( handlerConfig - ? { ...handlerConfig, signal: abortController.signal } + ? { ...handlerConfig, signal: abortController.signal, include: request.include } : { aggregator: aggregator!, + include: request.include, signal: abortController.signal, }, deps.toolExecuteOptions == null @@ -886,6 +894,8 @@ export async function createAgentChatCompletion( }, ); + const target = tracker ?? aggregator!; + // Convert messages to internal format const messages = convertMessages(request.messages); @@ -903,6 +913,7 @@ export async function createAgentChatCompletion( */ const safeUser: Record = { ...createSafeUser(reqUser), id: userId }; const run = await deps.createRun({ + ...target.toolExecution, sdkModelHost: deps.sdkModelHost, modelCallbacks: createApiModelCallbacks({ appConfig: deps.appConfig, @@ -922,7 +933,6 @@ export async function createAgentChatCompletion( }); if (run) { - const target = tracker ?? aggregator!; await completeOpenAIToolCalls( { finish: () => target.finishToolCalls?.(), @@ -982,6 +992,7 @@ export async function createAgentChatCompletion( aggregator.toolCalls, usage, true, + aggregator.serverToolResults?.response(), ); res.json(response); } diff --git a/packages/api/src/agents/openai/toolCalls.graph.spec.ts b/packages/api/src/agents/openai/toolCalls.graph.spec.ts index 96c7775efc..cb701dd8f8 100644 --- a/packages/api/src/agents/openai/toolCalls.graph.spec.ts +++ b/packages/api/src/agents/openai/toolCalls.graph.spec.ts @@ -10,6 +10,7 @@ import { Run, } from '@librechat/agents'; import type { BaseMessage } from '@langchain/core/messages'; +import type { EventHandler } from '@librechat/agents'; import type { Response } from 'express'; import type { ChatCompletionChunk, ToolCall } from './types'; import { @@ -264,75 +265,358 @@ describe('tool-call projection with real SDK graph dispatch', () => { '{"i":1}', ]); }); - it('keeps tools executed by the SDK out of the final OpenAI response', async () => { - const frames: string[] = []; - const tracker = createOpenAIStreamTracker(); - const context = { requestId: 'accepted', created: 1, model: 'fixture' }; - const res: Response = Object.create(response); - jest.spyOn(res, 'write').mockImplementation((frame) => { - frames.push(String(frame)); - return true; - }); - const config = { tracker, context, res }; - const executed: string[] = []; - const handlers = createOpenAIHandlers(config); - const run = await Run.create({ - runId: 'accepted-host-integration', - skipCleanup: true, - tokenCounter: () => 1, - graphConfig: { - type: 'standard', - maxContextTokens: 100_000, - llmConfig: { - provider: Providers.OPENAI, - streaming: false, - streamUsage: false, + it.each([false, true])( + 'observes SDK tool results only when opted in (include=%s)', + async (observe) => { + const frames: string[] = []; + const tracker = createOpenAIStreamTracker(); + const context = { requestId: 'accepted', created: 1, model: 'fixture' }; + const res: Response = Object.create(response); + jest.spyOn(res, 'write').mockImplementation((frame) => { + frames.push(String(frame)); + return true; + }); + const config = { + tracker, + context, + res, + include: observe ? ['server_tool_results' as const] : undefined, + }; + const executed: string[] = []; + const handlers = createOpenAIHandlers(config); + const owners: unknown[] = []; + if (observe) { + const complete = handlers.on_run_step_completed; + handlers.on_run_step_completed = { + handle(event, data, metadata, graph) { + owners.push(metadata?.agentId); + return complete.handle(event, data, metadata, graph); + }, + }; + } + const run = await Run.create({ + runId: 'accepted-host-integration', + skipCleanup: true, + tokenCounter: () => 1, + graphConfig: { + type: 'standard', + maxContextTokens: 100_000, + llmConfig: { + provider: Providers.OPENAI, + streaming: false, + streamUsage: false, + }, + tools: [ + tool( + async ({ city }) => { + executed.push(city); + return 'sunny'; + }, + { + name: 'lookup', + description: 'Weather', + schema: z.object({ city: z.string() }), + }, + ), + ], + }, + customHandlers: handlers, + }); + if (run.Graph == null) throw new Error('Missing graph'); + let calls = 0; + run.Graph.overrideModel = { + invoke: async (messages: BaseMessage[]): Promise => { + calls++; + return messages.some((message) => message.getType() === 'tool') + ? new AIMessageChunk('done') + : new AIMessageChunk({ + content: '', + tool_calls: [{ id: 'internal', name: 'lookup', args: { city: 'Paris' } }], + }); + }, + }; + + await run.processStream( + { messages: [new HumanMessage('weather')] }, + { configurable: { thread_id: 'accepted-host-integration' }, version: 'v2' }, + ); + sendFinalChunk(config); + + expect(calls).toBe(2); + expect(executed).toEqual(['Paris']); + if (observe) expect(owners).toEqual([expect.any(String)]); + expect(tracker.toolCalls.size).toBe(0); + const chunks: ChatCompletionChunk[] = frames + .filter((frame) => frame !== 'data: [DONE]\n\n') + .map((frame) => JSON.parse(frame.slice(6))); + expect(chunks.flatMap((chunk) => chunk.choices[0].delta.tool_calls ?? [])).toEqual([]); + expect(chunks.some((chunk) => chunk.choices[0].delta.content === 'done')).toBe(true); + expect(chunks.flatMap((chunk) => chunk.choices[0].delta.tool_results ?? [])).toEqual( + observe + ? [ + { + index: 0, + call_id: 'internal', + name: 'lookup', + arguments: '{"city":"Paris"}', + output: 'sunny', + }, + ] + : [], + ); + expect(chunks[chunks.length - 1].choices[0].finish_reason).toBe('stop'); + }, + ); + + it.each([ + { observe: true, fallback: undefined }, + { observe: true, fallback: 'Paris' }, + { observe: true, fallback: 'Berlin' }, + { observe: false, fallback: undefined }, + ])('observes accepted results without eager speculation (%j)', async ({ observe, fallback }) => { + const frames: string[] = []; + const events: string[] = []; + const owners: unknown[] = []; + const tracker = createOpenAIStreamTracker(); + const config = { + tracker, + context: { requestId: 'eager', created: 1, model: 'fixture' }, + writer: { + write: (frame: string) => { + frames.push(frame); }, - tools: [ - tool( - async ({ city }) => { - executed.push(city); - return 'sunny'; - }, - { - name: 'lookup', - description: 'Weather', - schema: z.object({ city: z.string() }), - }, - ), - ], }, - customHandlers: handlers, + include: observe ? ['server_tool_results' as const] : undefined, + }; + const handlers: Record = createOpenAIHandlers(config); + let completed!: () => void; + const completion = new Promise((resolve) => { + completed = resolve; }); - if (run.Graph == null) throw new Error('Missing graph'); - let calls = 0; - run.Graph.overrideModel = { - invoke: async (messages: BaseMessage[]): Promise => { - calls++; - return messages.some((message) => message.getType() === 'tool') - ? new AIMessageChunk('done') - : new AIMessageChunk({ - content: '', - tool_calls: [{ id: 'internal', name: 'lookup', args: { city: 'Paris' } }], - }); + let timeout: ReturnType | undefined; + const deadline = new Promise((_resolve, reject) => { + timeout = setTimeout(() => reject(new Error('Eager completion was not dispatched')), 3000); + }); + for (const name of ['on_model_response', 'on_run_step_completed'] as const) { + const delegate = handlers[name]; + handlers[name] = { + async handle(event, data, metadata, graph) { + events.push(name); + if (name === 'on_run_step_completed') { + owners.push(metadata?.agentId); + if ( + data == null || + !('result' in data) || + data.result == null || + typeof data.result !== 'object' || + !('id' in data.result) || + typeof data.result.id !== 'string' + ) + throw new Error('Expected SDK completion'); + if (!observe) expect(data.result).toEqual(expect.objectContaining({ eager: true })); + const step = graph?.getRunStep(data.result.id); + expect(step?.stepDetails).toEqual( + expect.objectContaining({ + tool_calls: expect.arrayContaining([expect.objectContaining({ id: 'eager' })]), + }), + ); + } + await delegate?.handle(event, data, metadata, graph); + if (name === 'on_run_step_completed') completed(); + }, + }; + } + const fallbackCalls: unknown[] = []; + const transport = jest.fn(async () => { + const chunks = [ + { + choices: [ + { + index: 0, + delta: { + role: 'assistant', + content: '', + tool_calls: [ + { + index: 0, + id: 'eager', + type: 'function', + function: { + name: 'lookup', + arguments: JSON.stringify({ city: fallback }), + }, + }, + ], + }, + finish_reason: null, + }, + ], + }, + { choices: [{ index: 0, delta: {}, finish_reason: 'tool_calls' }] }, + ]; + const payload = + chunks + .map( + (chunk) => + 'data: ' + + JSON.stringify({ + id: 'fallback-response', + object: 'chat.completion.chunk', + created: 1, + model: 'fallback', + ...chunk, + }) + + '\n\n', + ) + .join('') + 'data: [DONE]\n\n'; + return new globalThis.Response(payload, { + status: 200, + headers: { 'content-type': 'text/event-stream' }, + }); + }); + const acceptance = handlers.on_model_response; + handlers.on_model_response = { + async handle(event, data, metadata, graph) { + if (data != null && 'toolCallDispositions' in data && data.toolCalls.length > 0) { + fallbackCalls.push(data.toolCalls[0].args); + // The SDK registry retains the failed primary step even when accepted fallback args differ. + const step = graph?.getRunStep(graph.toolCallStepIds.get('eager') ?? ''); + expect(step?.stepDetails).toEqual( + expect.objectContaining({ + tool_calls: expect.arrayContaining([ + expect.objectContaining({ args: { city: 'Paris' } }), + ]), + }), + ); + } + await acceptance.handle(event, data, metadata, graph); }, }; - - await run.processStream( - { messages: [new HumanMessage('weather')] }, - { configurable: { thread_id: 'accepted-host-integration' }, version: 'v2' }, - ); - sendFinalChunk(config); - - expect(calls).toBe(2); - expect(executed).toEqual(['Paris']); - expect(tracker.toolCalls.size).toBe(0); - const chunks: ChatCompletionChunk[] = frames - .filter((frame) => frame !== 'data: [DONE]\n\n') - .map((frame) => JSON.parse(frame.slice(6))); - expect(chunks.flatMap((chunk) => chunk.choices[0].delta.tool_calls ?? [])).toEqual([]); - expect(chunks.some((chunk) => chunk.choices[0].delta.content === 'done')).toBe(true); - expect(chunks[chunks.length - 1].choices[0].finish_reason).toBe('stop'); + const executed: string[] = []; + handlers.on_tool_execute = { + handle(_event, data) { + if (data == null || !('resolve' in data)) + throw new Error('Expected SDK tool execution request'); + events.push('execute'); + data.resolve( + data.toolCalls.map((call) => { + executed.push(call.id); + return { toolCallId: call.id, status: 'success', content: 'protected result' }; + }), + ); + }, + }; + try { + const run = await Run.create({ + runId: 'eager-host-integration', + skipCleanup: true, + tokenCounter: () => 1, + eagerEventToolExecution: { enabled: tracker.toolExecution?.eagerToolExecution ?? true }, + graphConfig: { + type: 'standard', + maxContextTokens: 100_000, + llmConfig: { + provider: Providers.OPENAI, + streaming: true, + streamUsage: false, + fallbacks: fallback + ? [ + { + provider: Providers.OPENAI, + clientOptions: { + apiKey: 'fixture', + model: 'fallback', + streaming: true, + maxRetries: 0, + configuration: { fetch: transport }, + }, + }, + ] + : undefined, + }, + tools: [], + toolDefinitions: [ + { + name: 'lookup', + description: 'Fixture', + parameters: { + type: 'object', + properties: { city: { type: 'string' } }, + required: ['city'], + }, + }, + ], + }, + customHandlers: handlers, + }); + if (run.Graph == null) throw new Error('Missing graph'); + run.Graph.overrideModel = { + invoke: async () => { + throw new Error('Expected streaming provider invocation'); + }, + stream: async (messages: BaseMessage[]) => { + return (async function* () { + if (messages.some((message) => message.getType() === 'tool')) { + yield new AIMessageChunk('done'); + return; + } + yield new AIMessageChunk({ + content: '', + tool_call_chunks: [ + { + id: 'eager', + name: 'lookup', + args: '{"city":"Paris"}', + index: 0, + type: 'tool_call_chunk', + }, + ], + response_metadata: { finish_reason: 'tool_calls' }, + }); + if (!observe) await Promise.race([completion, deadline]); + if (fallback) throw new Error('Fixture primary failed after eager completion'); + })(); + }, + }; + await run.processStream( + { messages: [new HumanMessage('weather')] }, + { configurable: { thread_id: 'eager-host-integration' }, version: 'v2' }, + ); + sendFinalChunk(config); + if (observe) { + expect(events.indexOf('on_model_response')).toBeLessThan(events.indexOf('execute')); + expect(owners).toEqual(['default']); + } else { + expect(events.indexOf('on_run_step_completed')).toBeLessThan( + events.indexOf('on_model_response'), + ); + expect(owners).toEqual([undefined]); + } + expect(fallbackCalls).toEqual([{ city: fallback ?? 'Paris' }]); + expect(transport).toHaveBeenCalledTimes(fallback ? 1 : 0); + expect(executed).toEqual(['eager']); + expect(tracker.toolCalls.size).toBe(0); + const chunks: ChatCompletionChunk[] = frames + .filter((frame) => frame !== 'data: [DONE]\n\n') + .map((frame) => JSON.parse(frame.slice(6))); + expect(chunks.flatMap((chunk) => chunk.choices[0].delta.tool_calls ?? [])).toEqual([]); + expect(chunks.flatMap((chunk) => chunk.choices[0].delta.tool_results ?? [])).toEqual( + observe + ? [ + { + index: 0, + call_id: 'eager', + name: 'lookup', + arguments: JSON.stringify({ city: fallback ?? 'Paris' }), + output: 'protected result', + }, + ] + : [], + ); + expect(chunks[chunks.length - 1].choices[0].finish_reason).toBe('stop'); + } finally { + clearTimeout(timeout); + } }); it('assembles the name before emitting when real SDK chunks split it', async () => { diff --git a/packages/api/src/agents/openai/types.ts b/packages/api/src/agents/openai/types.ts index a03ac3107b..03b3c6e83e 100644 --- a/packages/api/src/agents/openai/types.ts +++ b/packages/api/src/agents/openai/types.ts @@ -51,6 +51,8 @@ export interface ChatMessage { * OpenAI chat completion request */ export interface ChatCompletionRequest { + /** LibreChat extension: observe completed server tools without requesting client execution. */ + include?: 'server_tool_results'[]; /** Agent ID to invoke (maps to model in OpenAI spec) */ model: string; /** Conversation messages */ @@ -100,6 +102,20 @@ export interface CompletionUsage { subagent?: CompletionUsageTotals; } +/** A completed server execution, distinct from an actionable client tool_call. */ +export interface ServerToolResult { + call_id: string; + name: string; + arguments: string; + output: string; +} + +export interface ServerToolResultFields { + tool_results: ServerToolResult[]; + /** Projection budget exhausted: these records are not a complete audit log. */ + tool_results_incomplete?: true; +} + /** * Non-streaming choice */ @@ -110,6 +126,8 @@ export interface ChatCompletionChoice { content: string | null; /** Reasoning/thinking content (OpenRouter convention) */ reasoning?: string | null; + tool_results?: ServerToolResult[]; + tool_results_incomplete?: true; tool_calls?: ToolCall[]; }; finish_reason: 'stop' | 'length' | 'tool_calls' | 'content_filter' | null; @@ -138,6 +156,8 @@ export interface ChatCompletionChunkChoice { content?: string | null; /** Reasoning/thinking content (OpenRouter convention) */ reasoning?: string | null; + tool_results?: Array; + tool_results_incomplete?: true; tool_calls?: Array<{ index: number; id?: string; diff --git a/packages/api/src/agents/run.ts b/packages/api/src/agents/run.ts index 68d7f78cd5..dd62716ac2 100644 --- a/packages/api/src/agents/run.ts +++ b/packages/api/src/agents/run.ts @@ -2357,6 +2357,7 @@ export async function createRun({ sdkResumeState, streaming = true, streamUsage = true, + eagerToolExecution = true, }: { agents: RunAgent[]; signal: AbortSignal; @@ -2366,6 +2367,8 @@ export async function createRun({ conversationId?: string; streaming?: boolean; streamUsage?: boolean; + /** Host-owned execution policy; observed Chat results require post-acceptance execution. */ + eagerToolExecution?: boolean; requestBody?: t.RequestBody; codeApprovalMode?: CodeApprovalMode; user?: IUser; @@ -3292,7 +3295,7 @@ export async function createRun({ // `interrupt()` from its tool body, which must run inside the Pregel task // frame — a speculative eager execution could never pause the run. eagerEventToolExecution: { - enabled: true, + enabled: eagerToolExecution, excludeToolNames: [ CREATE_FILE_TOOL_NAME, EDIT_FILE_TOOL_NAME,