diff --git a/api/server/controllers/agents/__tests__/openai.spec.js b/api/server/controllers/agents/__tests__/openai.spec.js index b02664babc..1adfef2ba2 100644 --- a/api/server/controllers/agents/__tests__/openai.spec.js +++ b/api/server/controllers/agents/__tests__/openai.spec.js @@ -217,6 +217,13 @@ jest.mock('@librechat/api', () => ({ buildAgentContextAttachmentsByAgentId: (...args) => mockBuildAgentContextAttachmentsByAgentId(...args), createChunk: jest.fn().mockReturnValue({}), + /** Not stubbed: the outward tool-call index this allocates is the behavior the + * controller is responsible for wiring, so the spec runs the real projection. */ + createOpenAIToolCallStream: (...args) => + jest.requireActual('@librechat/api').createOpenAIToolCallStream(...args), + completeOpenAIToolCalls: jest.requireActual('@librechat/api').completeOpenAIToolCalls, + OpenAIRunStepHandler: jest.requireActual('@librechat/api').OpenAIRunStepHandler, + OpenAIRunStepDeltaHandler: jest.requireActual('@librechat/api').OpenAIRunStepDeltaHandler, buildRunToolSet: jest.fn().mockReturnValue(new Set()), buildInitialToolSessions: jest.fn().mockReturnValue(mockInitialSessions), AgentRunEnvelopeError: MockAgentRunEnvelopeError, @@ -518,6 +525,227 @@ describe('OpenAIChatCompletionController', () => { }; }); + it.each([true, false])( + 'projects interleaved calls and preserves the final stop (stream=%s)', + async (streaming) => { + const api = require('@librechat/api'); + const actual = jest.requireActual('@librechat/api'); + const names = [ + 'createChunk', + 'writeSSE', + 'sendFinalChunk', + 'buildNonStreamingResponse', + 'createOpenAIStreamTracker', + 'createOpenAIContentAggregator', + ]; + const original = names.map((name) => [name, api[name].getMockImplementation()]); + for (const name of names) api[name].mockImplementation(actual[name]); + req.body.stream = streaming; + api.validateRequest.mockReturnValueOnce({ request: req.body }); + mockProcessStream.mockImplementationOnce(async () => { + const { customHandlers: h } = api.createRun.mock.calls.at(-1)[0]; + const meta = { langgraph_node: 'agent=test', langgraph_step: 1 }; + for (const [step, id, index] of [ + ['s1', 'a', 0], + ['s2', 'b', 1], + ]) { + await h.on_run_step.handle( + 'on_run_step', + { + id: step, + stepDetails: { + type: 'tool_calls', + tool_calls: [{ id, name: 'get_time' }], + }, + }, + meta, + ); + await h.on_run_step_delta.handle( + 'on_run_step_delta', + { + id: step, + delta: { + type: 'tool_calls', + tool_calls: [{ id, name: 'get_time', index }], + }, + }, + meta, + ); + } + for (const [index, city] of [ + [0, 'Madrid'], + [1, 'Paris'], + ]) { + await h.on_run_step_delta.handle( + 'on_run_step_delta', + { + id: 's2', + delta: { + type: 'tool_calls', + tool_calls: [{ index, args: JSON.stringify({ city }) }], + }, + }, + meta, + ); + } + await h.on_message_delta.handle( + 'on_message_delta', + { + id: 'answer', + delta: { + content: [{ type: 'text', text: 'Both tools completed.' }], + }, + }, + { ...meta, langgraph_step: 3 }, + ); + }); + try { + await OpenAIChatCompletionController(req, res); + if (streaming) { + const frames = res.write.mock.calls + .map(([frame]) => frame) + .filter((frame) => frame !== 'data: [DONE]\n\n') + .map((frame) => JSON.parse(frame.slice(6))); + expect(frames.at(-1).choices[0].finish_reason).toBe('stop'); + expect( + frames + .flatMap((frame) => frame.choices[0].delta.tool_calls ?? []) + .filter((call) => call.function?.arguments), + ).toEqual([ + { index: 0, function: { arguments: '{"city":"Madrid"}' } }, + { index: 1, function: { arguments: '{"city":"Paris"}' } }, + ]); + } else { + expect(res.json).toHaveBeenCalledWith( + expect.objectContaining({ + choices: [ + expect.objectContaining({ + finish_reason: 'stop', + message: expect.objectContaining({ + content: 'Both tools completed.', + tool_calls: [ + { + id: 'a', + type: 'function', + function: { name: 'get_time', arguments: '{"city":"Madrid"}' }, + }, + { + id: 'b', + type: 'function', + function: { name: 'get_time', arguments: '{"city":"Paris"}' }, + }, + ], + }), + }), + ], + }), + ); + } + } finally { + for (const [name, implementation] of original) api[name].mockImplementation(implementation); + } + }, + ); + + it.each( + [true, false].flatMap((stream) => + ['native-string', 'wire-object', 'split', 'idless'].map((shape) => [stream, shape]), + ), + )( + 'publishes complete identity and arguments before terminal output (stream=%s, shape=%s)', + async (streaming, shape) => { + const api = require('@librechat/api'); + const actual = jest.requireActual('@librechat/api'); + const names = [ + 'createChunk', + 'writeSSE', + 'sendFinalChunk', + 'buildNonStreamingResponse', + 'createOpenAIStreamTracker', + 'createOpenAIContentAggregator', + ]; + const originals = names.map((name) => [name, api[name].getMockImplementation()]); + for (const name of names) api[name].mockImplementation(actual[name]); + req.body.stream = streaming; + api.validateRequest.mockReturnValueOnce({ request: req.body }); + mockProcessStream.mockImplementationOnce(async () => { + const { customHandlers: h } = api.createRun.mock.calls.at(-1)[0]; + await h.on_run_step.handle('on_run_step', { + id: 'complete', + stepDetails: { + type: 'tool_calls', + tool_calls: [ + (() => { + if (shape === 'native-string') + return { id: 'a', name: 'get_time', args: '{"city":"Madrid"}' }; + if (shape === 'wire-object') + return { id: 'a', function: { name: 'get_time', arguments: { city: 'Madrid' } } }; + if (shape === 'idless') return { name: 'get_time', args: { city: 'Madrid' } }; + return { id: 'a', name: 'get_', args: {} }; + })(), + ], + }, + }); + if (shape === 'split') { + for (const fragment of [ + { id: 'a', name: 'get_', index: 0, args: '{"city":"Madrid"}' }, + { index: 0, name: 'time' }, + ]) { + await h.on_run_step_delta.handle('on_run_step_delta', { + id: 'complete', + delta: { type: 'tool_calls', tool_calls: [fragment] }, + }); + } + } + }); + try { + await OpenAIChatCompletionController(req, res); + if (streaming) { + const frames = res.write.mock.calls + .map(([frame]) => frame) + .filter((frame) => frame !== 'data: [DONE]\n\n') + .map((frame) => JSON.parse(frame.slice(6))); + expect( + frames + .flatMap((frame) => frame.choices[0].delta.tool_calls ?? []) + .map((call) => call.function?.arguments ?? '') + .join(''), + ).toBe('{"city":"Madrid"}'); + expect( + frames + .flatMap((frame) => frame.choices[0].delta.tool_calls ?? []) + .find((call) => call.id).function.name, + ).toBe('get_time'); + expect(frames.at(-1).choices[0].finish_reason).toBe('tool_calls'); + } else { + expect( + res.json.mock.calls[0][0].choices[0].message.tool_calls[0].function.arguments, + ).toBe('{"city":"Madrid"}'); + } + } finally { + for (const [name, implementation] of originals) + api[name].mockImplementation(implementation); + } + }, + ); + + it('records completed model usage even when terminal snapshot validation fails', async () => { + mockProcessStream.mockImplementationOnce(async () => { + const h = require('@librechat/api').createRun.mock.calls.at(-1)[0].customHandlers; + await h.on_run_step.handle('on_run_step', { + id: 'bad', + stepDetails: { + type: 'tool_calls', + tool_calls: [{ id: 'a', function: { name: 'get_time', arguments: 'NOT-JSON' } }], + }, + }); + }); + await OpenAIChatCompletionController(req, res); + expect(mockRecordCollectedUsage).toHaveBeenCalledTimes(1); + expect(require('@librechat/api').buildNonStreamingResponse).not.toHaveBeenCalled(); + expect(JSON.stringify(res.json.mock.calls)).not.toContain('NOT-JSON'); + }); + it('enrolls, starts, and settles the remote execution lifecycle', async () => { await OpenAIChatCompletionController(req, res); diff --git a/api/server/controllers/agents/openai.js b/api/server/controllers/agents/openai.js index 65e2230594..2a40bd0567 100644 --- a/api/server/controllers/agents/openai.js +++ b/api/server/controllers/agents/openai.js @@ -59,6 +59,10 @@ const { createToolExecuteHandler, createOwnedToolEndHandler, buildNonStreamingResponse, + OpenAIRunStepHandler, + OpenAIRunStepDeltaHandler, + createOpenAIToolCallStream, + completeOpenAIToolCalls, createOpenAIStreamTracker, resolveAgentScopedSkillIds, createOpenAIContentAggregator, @@ -974,6 +978,19 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { } }; + /** + * Shared by both run-step events, because the outward index a client keys + * tool-call fragments by is allocated per call and belongs to neither + * event alone. + */ + const toolCallStream = createOpenAIToolCallStream({ + signal: execution.signal, + toolCalls: isStreaming ? tracker.toolCalls : aggregator.toolCalls, + ...(isStreaming && { + emit: (delta) => writeSSE(res, createChunk(context, delta)), + }), + }); + // Event handlers for OpenAI-compatible streaming const handlers = { // Text content streaming @@ -1001,77 +1018,11 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { } }), - // Tool call initiation - streams id and name (from on_run_step) - on_run_step: createHandler((data) => { - const stepDetails = data?.stepDetails; - if (stepDetails?.type === 'tool_calls' && stepDetails.tool_calls) { - for (const tc of stepDetails.tool_calls) { - const toolIndex = data.index ?? 0; - const toolId = tc.id ?? ''; - const toolName = tc.name ?? ''; - const toolCall = { - id: toolId, - type: 'function', - function: { name: toolName, arguments: '' }, - }; - - // Track tool call in tracker or aggregator - if (isStreaming) { - if (!tracker.toolCalls.has(toolIndex)) { - tracker.toolCalls.set(toolIndex, toolCall); - } - // Stream initial tool call chunk (like OpenAI does) - writeSSE( - res, - createChunk(context, { - tool_calls: [{ index: toolIndex, ...toolCall }], - }), - ); - } else { - if (!aggregator.toolCalls.has(toolIndex)) { - aggregator.toolCalls.set(toolIndex, toolCall); - } - } - } - } - }), + // Tool call initiation - declares id and name (from on_run_step) + on_run_step: new OpenAIRunStepHandler(toolCallStream), // Tool call argument streaming (from on_run_step_delta) - on_run_step_delta: createHandler((data) => { - const delta = data?.delta; - if (delta?.type === 'tool_calls' && delta.tool_calls) { - for (const tc of delta.tool_calls) { - const args = tc.args ?? ''; - if (!args) { - continue; - } - - const toolIndex = tc.index ?? 0; - - // Update tool call arguments - const targetMap = isStreaming ? tracker.toolCalls : aggregator.toolCalls; - const tracked = targetMap.get(toolIndex); - if (tracked) { - tracked.function.arguments += args; - } - - // Stream argument delta (only for streaming) - if (isStreaming) { - writeSSE( - res, - createChunk(context, { - tool_calls: [ - { - index: toolIndex, - function: { arguments: args }, - }, - ], - }), - ); - } - } - } - }), + on_run_step_delta: new OpenAIRunStepDeltaHandler(toolCallStream), // Usage tracking on_chat_model_end: { @@ -1084,7 +1035,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { } }, }, - on_run_step_completed: createHandler(), + on_run_step_completed: new OpenAIRunStepHandler(toolCallStream), // Use proper ToolEndHandler for processing artifacts (images, file citations, code output) on_tool_end: createOwnedToolEndHandler(toolEndCallback, logger), on_chain_stream: createHandler(), @@ -1188,47 +1139,49 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { version: 'v2', }; - await run.processStream({ messages: formattedMessages }, config, { - callbacks: { - [Callback.TOOL_ERROR]: (graph, error, toolId) => { - logger.error(`[OpenAI API] Tool Error "${toolId}"`, getSafeErrorMetadata(error)); + await completeOpenAIToolCalls(toolCallStream, async () => { + await run.processStream({ messages: formattedMessages }, config, { + callbacks: { + [Callback.TOOL_ERROR]: (graph, error, toolId) => { + logger.error(`[OpenAI API] Tool Error "${toolId}"`, getSafeErrorMetadata(error)); + }, }, - }, - }); + }); - // Record token usage against balance - const balanceConfig = getBalanceConfig(appConfig); - const transactionsConfig = getTransactionsConfig(appConfig); - execution.track( - recordCollectedUsage( - { - spendTokens: db.spendTokens, - spendStructuredTokens: db.spendStructuredTokens, - pricing: { - getMultiplier: db.getMultiplier, - getCacheMultiplier: db.getCacheMultiplier, + // Record token usage against balance + const balanceConfig = getBalanceConfig(appConfig); + const transactionsConfig = getTransactionsConfig(appConfig); + execution.track( + recordCollectedUsage( + { + spendTokens: db.spendTokens, + spendStructuredTokens: db.spendStructuredTokens, + pricing: { + getMultiplier: db.getMultiplier, + getCacheMultiplier: db.getCacheMultiplier, + }, + bulkWriteOps: { + insertMany: db.bulkInsertTransactions, + updateBalance: db.updateBalance, + }, }, - bulkWriteOps: { - insertMany: db.bulkInsertTransactions, - updateBalance: db.updateBalance, + { + user: userId, + conversationId, + collectedUsage, + context: 'message', + messageId: responseId, + balance: balanceConfig, + transactions: transactionsConfig, + model: primaryConfig.model || agent.model_parameters?.model, + endpointTokenConfig: primaryConfig.endpointTokenConfig, + resolveEndpointTokenConfig, }, - }, - { - user: userId, - conversationId, - collectedUsage, - context: 'message', - messageId: responseId, - balance: balanceConfig, - transactions: transactionsConfig, - model: primaryConfig.model || agent.model_parameters?.model, - endpointTokenConfig: primaryConfig.endpointTokenConfig, - resolveEndpointTokenConfig, - }, - ).catch((err) => { - logger.error('[OpenAI API] Error recording usage:', getSafeErrorMetadata(err)); - }), - ); + ).catch((err) => { + logger.error('[OpenAI API] Error recording usage:', getSafeErrorMetadata(err)); + }), + ); + }); const usage = buildCompletionUsage(collectedUsage); diff --git a/packages/api/src/agents/openai/README.md b/packages/api/src/agents/openai/README.md new file mode 100644 index 0000000000..f259fe42d6 --- /dev/null +++ b/packages/api/src/agents/openai/README.md @@ -0,0 +1,65 @@ +# Agents API response projection + +This module projects an **agent run**, which can include several model invocations and server-executed tools, onto an OpenAI-compatible response. It is not a transparent proxy for a single model completion. Changing tool execution ownership or terminal-turn selection is a separate protocol decision. + +## Owners and boundaries + +```text +provider adapter -> agents graph -> run-step declarations / argument deltas + | + tool-call projection + / \ + SSE chunks JSON accumulator + \ / + successful response completion +``` + +The Express controller and packaged `createAgentChatCompletion` service both use the same tool-call projection in streaming and non-streaming mode. The service's content handlers also share text, reasoning and usage handling between modes. The controller retains its existing run lifecycle, usage recording, artifact writes and authorization boundaries. + +## Identity + +- A declaring run step owns call identity. Optional provider IDs are aliases, not a prerequisite. Declaration slots keep id-less calls distinct and make snapshot replay idempotent. Provider IDs can repeat in later steps and are not completion-wide identities. +- Outward indexes are allocated once per call across the response. Raw ID substrings are assembled by provider index; shared prefixes such as `call_` are never cross-call aliases. Calls without IDs receive `call_`; collisions receive a suffix. Internal tool execution IDs are not rewritten. +- Argument fragments use provider indexes scoped to a model invocation. A graph delta may name the latest step while referring to an earlier parallel call. When available, the SDK's `getStepBaseKey` supplies invocation scope, including stream segments. A step's scope is retained so late fragments cannot acquire a newer segment's identity. +- Callers without the graph object use metadata scope; callers without that metadata are isolated by step. This fallback cannot reconstruct a missing retry/segment identity. +- Explicit and identified provider indexes take precedence over positional fallback. Positions are used only for multi-call declarations. An unbound singleton may adopt its first provider index; it does not absorb arbitrary later indexes. + +## Argument representations + +A run-step declaration can contain a **snapshot**: native `args` or OpenAI-shaped `function.arguments`. A run-step delta carries an **append-only fragment**. They must never be concatenated as though they were the same representation. + +Both snapshot argument fields accept JSON **strings or objects**, using the shared `Agents.ToolCall` and `Agents.AgentFunctionToolCall` field types. Strings are retained verbatim; objects are serialized once. Nonempty argument fragments take precedence over all snapshots for that call, regardless of arrival order. + +### Publication timing + +Tool-call IDs and names can also arrive as substrings. There is no per-field completion seal in these events, and the OpenAI-compatible consumer freezes the name on its first chunk. Emitting `get_` and later correcting it to `get_weather` does not work. + +Consequently, **tool-call chunks are published only at successful response completion**, after assembling identity and validating arguments. The first chunk contains the complete ID/name; the next carries the complete arguments at the same index. Text and reasoning continue streaming while the run executes. This deliberately sacrifices incremental tool-input display, rather than guessing which names are complete or breaking existing clients. Native JSON responses receive the same completed calls. + +Name/ID/argument fragments are append-only and take precedence over the corresponding declaration snapshot. Snapshot replay is idempotent; raw fragments are not deduplicated because identical adjacent fragments can be legitimate and the events carry no sequence token. Split IDs require a provider index for unambiguous correlation. Exactly-once raw dispatch remains an upstream requirement. + +## Terminal behavior + +`completeOpenAIToolCalls` wraps execution and owns the success/failure boundary. The controller schedules completed-run usage recording before final validation. + +| State/event | Behavior | +| ---------------------------------------- | ---------------------------------------------------------------- | +| Open + declaration | Allocate identity once; retain snapshot separately | +| Open + identified fragment | Bind provider index and append/buffer fragment | +| Successful execution | Assemble and validate all calls before publishing tool chunks | +| Failure, abort or invalid terminal input | Discard pending calls; publish no tool-call chunks | +| Finished | Repeated finish is harmless; later tool events are ignored | +| Aborted | Later tool events are ignored; finish cannot reopen the response | + +Missing names, missing arguments, malformed JSON or unattributable argument data fail terminal projection with bounded errors that omit provider content. A snapshot never repairs a truncated raw stream. Projection bookkeeping is response-local and released on finish/abort; only successfully validated calls are materialized for the response builder. Publication seals before calling the transport, so a transport error or reentrant finish cannot duplicate calls. Already transmitted frames cannot be retracted. + +The existing finish-reason policy is preserved: a response with final text after server-executed tools remains `stop`. Historical tool calls are still exposed by this API. This patch does not make it safe to interpret every advertised call as a new instruction for the caller to execute. + +## Verification and limits + +- `toolCalls.contract.spec.ts`: shared field types, strings/objects, partial names and IDs, optional IDs, declaration order and transport reentrancy. +- `toolCalls.spec.ts`: identity, representations, ordering, replay, malformed input, failure and abort. +- `toolCalls.graph.spec.ts`: real SDK interleaving, complete-only chunks and stream segments. +- `service.spec.ts` and the Express controller spec: both response modes, actual serializers, fallback flush and failure/usage ordering. + +These tests do not run a live provider, real tool side effects, a distributed multi-agent deployment or a persisted resume. There is no new database, cache, authorization policy or migration. Code and built packages must deploy together. No cross-version event identity is invented when the upstream source omits it. diff --git a/packages/api/src/agents/openai/handlers.ts b/packages/api/src/agents/openai/handlers.ts index 0ecd7c4715..3c5de3fb07 100644 --- a/packages/api/src/agents/openai/handlers.ts +++ b/packages/api/src/agents/openai/handlers.ts @@ -5,6 +5,8 @@ * streaming format (SSE with chat.completion.chunk objects). */ import type { Response as ServerResponse } from 'express'; +import type { Agents } from 'librechat-data-provider'; +import type { Graph } from '@librechat/agents'; import type { ChatCompletionChunkChoice, OpenAIResponseContext, @@ -58,6 +60,9 @@ export function writeSSE(res: ServerResponse, data: ChatCompletionChunk | string * Only tracks what's needed for finish_reason and usage - doesn't store content. */ export interface OpenAIStreamTracker { + /** Successful response completion, installed by the response handlers. */ + finishToolCalls?: () => void; + abortToolCalls?: () => void; /** Whether any text content was emitted */ hasText: boolean; /** Whether any reasoning content was emitted */ @@ -105,6 +110,9 @@ export function createOpenAIStreamTracker(): OpenAIStreamTracker { * Uses arrays for O(n) text accumulation instead of O(n²) string concatenation. */ export interface OpenAIContentAggregator { + /** Successful response completion, installed by the response handlers. */ + finishToolCalls?: () => void; + abortToolCalls?: () => void; /** Accumulated text chunks */ textChunks: string[]; /** Accumulated reasoning/thinking chunks */ @@ -154,6 +162,7 @@ export function createOpenAIContentAggregator(): OpenAIContentAggregator { * Handler configuration for OpenAI streaming */ export interface OpenAIStreamHandlerConfig { + signal?: AbortSignal; res: ServerResponse; context: OpenAIResponseContext; tracker: OpenAIStreamTracker; @@ -187,22 +196,44 @@ export const StepTypes = { */ export interface MessageDeltaData { id?: string; - content?: Array<{ type: string; text?: string }>; + content?: Array<{ type: string; text?: string; think?: string }>; + delta?: { content?: MessageDeltaData['content'] }; } +/** + * One tool-call fragment of a run step delta, as `@librechat/agents` emits it: + * `id`, `name`, and `args` can all be substrings. `index` identifies their + * provider stream within the current model invocation. Reuse the shared chunk + * contract and accept equivalent function-shaped name/argument fragments. + */ +export type RunStepToolCallChunk = Agents.ToolCallChunk & { + function?: { name?: string; arguments?: string }; +}; + export interface RunStepDeltaData { + /** The run step these fragments belong to. */ id?: string; delta?: { type?: string; - tool_calls?: Array<{ - index?: number; - id?: string; - type?: string; - function?: { - name?: string; - arguments?: string; - }; - }>; + tool_calls?: RunStepToolCallChunk[]; + }; +} + +/** A tool call as the run step that opened it declares it. */ +export type RunStepToolCall = Partial> & { + index?: number; + type?: string; + function?: Partial; +}; + +export interface RunStepData { + /** The run step's own id, shared with every delta dispatched for it. */ + id?: string; + /** The step's position in the response content, not a tool-call index. */ + index?: number; + stepDetails?: { + type?: string; + tool_calls?: RunStepToolCall[]; }; } @@ -240,20 +271,372 @@ export interface EventHandler { ): void | Promise; } +/** + * Projects the graph's tool-call events onto the OpenAI wire format. + * + * A client accumulates `delta.tool_calls` by `index`: the first chunk at an + * index declares the call and must carry `id` and `function.name`, and every + * later chunk at that index belongs to that same call. Neither index the graph + * reports can serve as that key. A run step's `index` is its position in the + * response's content, so a tool call that follows text does not start at zero, + * and consecutive steps each carry a single-element `tool_calls` array rather + * than one array holding every call. A fragment's `index` is the provider's + * content-block index, which restarts at zero on each model invocation of the + * run, so two different calls can share it. + * + * Outward indexes identify calls within declaring steps, using declaration slots + * when IDs are absent. Provider indexes correlate raw fragments within an + * invocation. IDs and names are assembled before publication: a client can freeze + * both on its first chunk, and the graph provides no per-field completion seal. + */ +export interface OpenAIToolCallStreamConfig { + /** Completed tool calls, populated at successful finish and keyed by outward index. */ + toolCalls: Map; + /** Emits complete identity/argument chunks at finish. Omitted for non-streaming. */ + emit?: (delta: ChatCompletionChunkChoice['delta']) => void; + signal?: AbortSignal; +} + +export interface OpenAIToolCallStream { + /** Declares the tool calls a run step opened. */ + onRunStep: (data: RunStepData, metadata?: Record, graph?: ToolCallGraph) => void; + /** Accumulates the name and argument fragments streamed for a run step. */ + onRunStepDelta: ( + data: RunStepDeltaData, + metadata?: Record, + graph?: ToolCallGraph, + ) => void; + /** Publish assembled, validated calls once after success, before DONE/JSON. */ + finish: () => void; + /** Discard pending calls on failure and reject further writes. */ + abort: () => void; +} + +type ToolCallGraph = Pick; + +interface ProjectedToolCall { + index: number; + snapshotId?: string; + snapshotName?: string; + snapshotArgs?: string; + idFragments: string[]; + nameFragments: string[]; + argFragments: string[]; +} + +interface ToolCallStep { + byId: Map; + byDeclaration: Map; + byProviderIndex: Map; + /** Declaration position is a fallback, never stronger than a provider binding. */ + byPosition: Map; + calls: Set; +} + +export function createOpenAIToolCallStream( + config: OpenAIToolCallStreamConfig, +): OpenAIToolCallStream { + const { toolCalls, emit, signal } = config; + let phase: 'open' | 'finished' | 'aborted' = 'open'; + let unattributableArguments = false; + const calls: ProjectedToolCall[] = []; + const steps = new Map(); + const invocations = new Map>(); + const stepScopes = new Map(); + + /** The SDK can send earlier parallel calls under the latest step ID. Its + * invocation key owns segment/checkpoint transitions; cache it at declaration + * so late events never borrow a new segment. Step isolation is the fallback. */ + const getBindings = ( + stepId: string, + metadata?: Record, + graph?: ToolCallGraph, + ): Map => { + let scope = stepScopes.get(stepId); + if (scope === undefined) { + if (graph && metadata) { + scope = graph.getStepBaseKey(metadata); + } else if ( + typeof metadata?.langgraph_node === 'string' && + typeof metadata.langgraph_step === 'number' + ) { + scope = JSON.stringify([ + metadata.run_id ?? '', + metadata.thread_id ?? '', + metadata.langgraph_node, + metadata.langgraph_step, + metadata.langgraph_checkpoint_ns ?? metadata.checkpoint_ns ?? '', + ]); + } else { + scope = JSON.stringify(['step', stepId]); + } + stepScopes.set(stepId, scope); + } + let bindings = invocations.get(scope); + if (bindings === undefined) { + bindings = new Map(); + invocations.set(scope, bindings); + } + return bindings; + }; + + const getStep = (stepId: string): ToolCallStep => { + let step = steps.get(stepId); + if (step === undefined) { + step = { + byId: new Map(), + byDeclaration: new Map(), + byProviderIndex: new Map(), + byPosition: new Map(), + calls: new Set(), + }; + steps.set(stepId, step); + } + return step; + }; + + const allocate = (step: ToolCallStep): ProjectedToolCall => { + const call: ProjectedToolCall = { + index: calls.length, + idFragments: [], + nameFragments: [], + argFragments: [], + }; + calls.push(call); + step.calls.add(call); + return call; + }; + + const abort = (): void => { + if (phase !== 'finished') { + phase = 'aborted'; + } + calls.length = 0; + steps.clear(); + stepScopes.clear(); + invocations.clear(); + }; + + const writable = (): boolean => { + if (signal?.aborted) { + abort(); + } + return phase === 'open'; + }; + + return { + abort, + finish: () => { + if (phase === 'finished') { + return; + } + if (!writable()) { + const error = new Error('Agent response aborted'); + error.name = 'AbortError'; + throw error; + } + try { + if (unattributableArguments) { + throw new Error('Unattributable tool call arguments in agent response'); + } + const ready: ToolCall[] = []; + const ids = new Set(); + /** There is no per-field name/ID seal in the graph event contract. An + * OpenAI-compatible client freezes the name on the first outward chunk. + * Validate and assemble every call before publishing any: text still + * streams, but tool chunks wait for successful response completion. */ + for (const call of calls) { + const name = call.nameFragments.join('') || call.snapshotName; + const args = call.argFragments.length ? call.argFragments.join('') : call.snapshotArgs; + if (!name || args === undefined || args === '') { + throw new Error('Incomplete tool call in agent response'); + } + try { + JSON.parse(args); + } catch { + throw new Error('Invalid tool call arguments in agent response'); + } + let id = call.idFragments.join('') || call.snapshotId || `call_${call.index}`; + while (ids.has(id)) { + id = `${id}_${call.index}`; + } + ids.add(id); + ready.push({ id, type: 'function', function: { name, arguments: args } }); + } + for (const [index, call] of ready.entries()) { + toolCalls.set(index, call); + } + /** Seal before invoking transport callbacks, including reentrant callers. + * An emission failure cannot be retried into duplicate tool requests. */ + phase = 'finished'; + for (const [index, call] of ready.entries()) { + emit?.({ + tool_calls: [ + { + index, + id: call.id, + type: 'function', + function: { name: call.function.name, arguments: '' }, + }, + ], + }); + emit?.({ tool_calls: [{ index, function: { arguments: call.function.arguments } }] }); + } + } finally { + abort(); + } + }, + onRunStep: (data, metadata, graph) => { + if (!writable()) { + return; + } + const details = data?.stepDetails; + if (details?.type !== StepTypes.TOOL_CALLS || !Array.isArray(details.tool_calls)) { + return; + } + const step = getStep(data.id ?? ''); + const bindings = getBindings(data.id ?? '', metadata, graph); + for (const [position, toolCall] of details.tool_calls.entries()) { + const key = + toolCall.index === undefined ? `position:${position}` : `index:${toolCall.index}`; + let call = + (toolCall.id ? step.byId.get(toolCall.id) : undefined) ?? step.byDeclaration.get(key); + if (call === undefined && toolCall.index !== undefined) { + call = step.byProviderIndex.get(toolCall.index); + } + /** A declaration can follow its raw chunks. Positions are meaningful + * only inside this declaring step, never across the whole response. */ + if (call === undefined) { + call = step.byProviderIndex.get(position); + if ( + call === undefined && + step.byDeclaration.size === 0 && + details.tool_calls.length === 1 && + step.calls.size === 1 + ) { + call = step.calls.values().next().value; + } + } + call ??= allocate(step); + step.byDeclaration.set(key, call); + if (toolCall.id) { + call.snapshotId = toolCall.id; + step.byId.set(toolCall.id, call); + } + call.snapshotName = toolCall.name ?? toolCall.function?.name ?? call.snapshotName; + const args = toolCall.function?.arguments ?? toolCall.args; + if (args !== undefined) { + /** Both public snapshot fields accept raw JSON strings or objects. */ + try { + call.snapshotArgs = typeof args === 'string' ? args : JSON.stringify(args); + } catch { + throw new Error('Invalid tool call arguments in agent response'); + } + } + if (toolCall.index !== undefined) { + step.byProviderIndex.set(toolCall.index, call); + bindings.set(toolCall.index, call); + } else if (details.tool_calls.length > 1) { + step.byPosition.set(position, call); + } + } + }, + onRunStepDelta: (data, metadata, graph) => { + if (!writable()) { + return; + } + const delta = data?.delta; + if (delta?.type !== StepTypes.TOOL_CALLS || !Array.isArray(delta.tool_calls)) { + return; + } + const step = getStep(data.id ?? ''); + const bindings = getBindings(data.id ?? '', metadata, graph); + for (const fragment of delta.tool_calls) { + const name = fragment.name ?? fragment.function?.name ?? ''; + const args = fragment.args ?? fragment.function?.arguments ?? ''; + /** An index identifies an already-bound raw stream even if this event + * carries only a substring of its ID. Never key identity by that suffix. */ + let call = fragment.index === undefined ? undefined : bindings.get(fragment.index); + call ??= fragment.id ? step.byId.get(fragment.id) : undefined; + call ??= fragment.index === undefined ? undefined : step.byPosition.get(fragment.index); + if ( + call === undefined && + step.calls.size === 1 && + (fragment.index === undefined || step.byProviderIndex.size === 0) + ) { + const singleton = step.calls.values().next().value; + /** Without an index, a different full ID starts another raw call, not + * another substring of the old call. Split IDs need their index. */ + if ( + !fragment.id || + fragment.index !== undefined || + (singleton?.snapshotId === undefined && singleton?.idFragments.length === 0) + ) { + call = singleton; + } + } + if (call === undefined && (fragment.id || name)) { + call = allocate(step); + } + if (call === undefined) { + unattributableArguments ||= !!args; + continue; + } + if (fragment.index !== undefined) { + step.byProviderIndex.set(fragment.index, call); + bindings.set(fragment.index, call); + } + if (fragment.id) { + call.idFragments.push(fragment.id); + /** Partial IDs are not aliases across indexed calls: two parallel + * calls can both begin with `call_`. Their index is authoritative. */ + if (fragment.index === undefined) { + step.byId.set(call.idFragments.join(''), call); + } + } + if (name) { + call.nameFragments.push(name); + } + if (args) { + call.argFragments.push(args); + } + } + }, + }; +} + +/** Success publishes validated calls; failure/abort never does. Both hosts + * use this boundary so late provider events cannot mutate a settled response. */ +export async function completeOpenAIToolCalls( + lifecycle: Pick, + execute: () => Promise, +): Promise { + try { + await execute(); + lifecycle.finish(); + } finally { + lifecycle.abort(); + } +} + /** * Handler for message delta events - streams text content */ export class OpenAIMessageDeltaHandler implements EventHandler { - constructor(private config: OpenAIStreamHandlerConfig) {} + constructor(private config: OpenAIContentHandlerConfig) {} handle(_event: string, data: MessageDeltaData): void { - const content = data?.content; + const content = data?.delta?.content ?? data?.content; if (!content || !Array.isArray(content)) { return; } for (const part of content) { if (part.type === 'text' && part.text) { + if ('aggregator' in this.config) { + this.config.aggregator.addText(part.text); + continue; + } this.config.tracker.addText(); const chunk = createChunk(this.config.context, { content: part.text }); writeSSE(this.config.res, chunk); @@ -263,86 +646,35 @@ export class OpenAIMessageDeltaHandler implements EventHandler { } /** - * Handler for run step delta events - streams tool calls + * Handler for run step delta events - accumulates streamed tool call fragments */ export class OpenAIRunStepDeltaHandler implements EventHandler { - constructor(private config: OpenAIStreamHandlerConfig) {} + constructor(private toolCallStream: OpenAIToolCallStream) {} - handle(_event: string, data: RunStepDeltaData): void { - const delta = data?.delta; - if (!delta || delta.type !== StepTypes.TOOL_CALLS) { - return; - } - - const toolCalls = delta.tool_calls; - if (!toolCalls || !Array.isArray(toolCalls)) { - return; - } - - for (const tc of toolCalls) { - if (tc.index === undefined) { - continue; - } - - // Initialize tool call in tracker if needed - let trackedTc = this.config.tracker.toolCalls.get(tc.index); - if (!trackedTc && tc.id) { - trackedTc = { - id: tc.id, - type: 'function', - function: { - name: '', - arguments: '', - }, - }; - this.config.tracker.toolCalls.set(tc.index, trackedTc); - } - - // Build the streaming delta - const streamDelta: ChatCompletionChunkChoice['delta'] = { - tool_calls: [ - { - index: tc.index, - ...(tc.id && { id: tc.id }), - ...(tc.type && { type: tc.type as 'function' }), - ...(tc.function && { - function: { - ...(tc.function.name && { name: tc.function.name }), - ...(tc.function.arguments && { arguments: tc.function.arguments }), - }, - }), - }, - ], - }; - - // Update tracked tool call - if (trackedTc) { - if (tc.function?.name) { - trackedTc.function.name += tc.function.name; - } - if (tc.function?.arguments) { - trackedTc.function.arguments += tc.function.arguments; - } - } - - const chunk = createChunk(this.config.context, streamDelta); - writeSSE(this.config.res, chunk); - } + handle( + _event: string, + data: RunStepDeltaData, + metadata?: Record, + graph?: ToolCallGraph, + ): void { + this.toolCallStream.onRunStepDelta(data, metadata, graph); } } /** - * Handler for run step events - sends initial tool call info + * Handler for run step events - declares a tool call's id and name at its + * outward index, before any argument fragment references that index */ export class OpenAIRunStepHandler implements EventHandler { - constructor(private config: OpenAIStreamHandlerConfig) {} + constructor(private toolCallStream: OpenAIToolCallStream) {} - handle(_event: string, data: { stepDetails?: { type?: string } }): void { - // Run step events are primarily for LibreChat UI, we use deltas for streaming - // This handler is a no-op for OpenAI format - if (data?.stepDetails?.type === StepTypes.TOOL_CALLS) { - // Tool calls will be streamed via delta events - } + handle( + _event: string, + data: RunStepData, + metadata?: Record, + graph?: ToolCallGraph, + ): void { + this.toolCallStream.onRunStep(data, metadata, graph); } } @@ -350,7 +682,7 @@ export class OpenAIRunStepHandler implements EventHandler { * Handler for model end events - captures usage */ export class OpenAIModelEndHandler implements EventHandler { - constructor(private config: OpenAIStreamHandlerConfig) {} + constructor(private config: OpenAIContentHandlerConfig) {} handle(_event: string, data: ModelEndData): void { const usage = data?.output?.usage_metadata; @@ -358,9 +690,10 @@ export class OpenAIModelEndHandler implements EventHandler { return; } - this.config.tracker.usage.promptTokens += usage.input_tokens ?? 0; - this.config.tracker.usage.completionTokens += usage.output_tokens ?? 0; - this.config.tracker.usage.reasoningTokens += + const target = 'aggregator' in this.config ? this.config.aggregator : this.config.tracker; + target.usage.promptTokens += usage.input_tokens ?? 0; + target.usage.completionTokens += usage.output_tokens ?? 0; + target.usage.reasoningTokens += usage.output_token_details?.reasoning ?? usage.output_token_details?.reasoning_tokens ?? 0; } } @@ -389,21 +722,26 @@ export class OpenAIToolEndHandler implements EventHandler { * Streams reasoning/thinking content using the `delta.reasoning` field (OpenRouter convention). */ export class OpenAIReasoningDeltaHandler implements EventHandler { - constructor(private config: OpenAIStreamHandlerConfig) {} + constructor(private config: OpenAIContentHandlerConfig) {} handle(_event: string, data: MessageDeltaData): void { - const content = data?.content; + const content = data?.delta?.content ?? data?.content; if (!content || !Array.isArray(content)) { return; } for (const part of content) { - if (part.type === 'text' && part.text) { + if ((part.type === 'text' || part.type === 'think') && (part.think || part.text)) { + const text = part.think || part.text!; + if ('aggregator' in this.config) { + this.config.aggregator.addReasoning(text); + continue; + } // Mark that reasoning was emitted this.config.tracker.addReasoning(); // Stream as delta.reasoning (OpenRouter convention) - const chunk = createChunk(this.config.context, { reasoning: part.text }); + const chunk = createChunk(this.config.context, { reasoning: text }); writeSSE(this.config.res, chunk); } } @@ -413,15 +751,34 @@ export class OpenAIReasoningDeltaHandler implements EventHandler { /** * Create all handlers for OpenAI streaming format */ +export interface OpenAIAggregationHandlerConfig { + aggregator: OpenAIContentAggregator; + signal?: AbortSignal; +} + +type OpenAIContentHandlerConfig = OpenAIStreamHandlerConfig | OpenAIAggregationHandlerConfig; + export function createOpenAIHandlers( - config: OpenAIStreamHandlerConfig, + config: OpenAIContentHandlerConfig, toolExecuteOptions?: ToolExecuteOptions, ): Record { + /** One projection across both events, so a call keeps a single outward index. */ + const toolCallStream = createOpenAIToolCallStream({ + signal: config.signal, + toolCalls: 'aggregator' in config ? config.aggregator.toolCalls : config.tracker.toolCalls, + emit: + 'aggregator' in config + ? undefined + : (delta) => writeSSE(config.res, createChunk(config.context, delta)), + }); + const target = 'aggregator' in config ? config.aggregator : config.tracker; + target.finishToolCalls = toolCallStream.finish; + target.abortToolCalls = toolCallStream.abort; const handlers: Record = { [GraphEvents.ON_MESSAGE_DELTA]: new OpenAIMessageDeltaHandler(config), - [GraphEvents.ON_RUN_STEP_DELTA]: new OpenAIRunStepDeltaHandler(config), - [GraphEvents.ON_RUN_STEP]: new OpenAIRunStepHandler(config), - [GraphEvents.ON_RUN_STEP_COMPLETED]: new OpenAIRunStepHandler(config), + [GraphEvents.ON_RUN_STEP_DELTA]: new OpenAIRunStepDeltaHandler(toolCallStream), + [GraphEvents.ON_RUN_STEP]: new OpenAIRunStepHandler(toolCallStream), + [GraphEvents.ON_RUN_STEP_COMPLETED]: new OpenAIRunStepHandler(toolCallStream), [GraphEvents.CHAT_MODEL_END]: new OpenAIModelEndHandler(config), [GraphEvents.CHAT_MODEL_STREAM]: new OpenAIChatModelStreamHandler(), [GraphEvents.TOOL_END]: new OpenAIToolEndHandler(), @@ -444,6 +801,7 @@ export function sendFinalChunk( usageOverride?: CompletionUsage, ): void { const { res, context, tracker } = config; + tracker.finishToolCalls?.(); // Determine finish reason based on content let reason = finishReason; diff --git a/packages/api/src/agents/openai/service.spec.ts b/packages/api/src/agents/openai/service.spec.ts index 782f081074..8e1381f8e9 100644 --- a/packages/api/src/agents/openai/service.spec.ts +++ b/packages/api/src/agents/openai/service.spec.ts @@ -2,6 +2,7 @@ import { GraphEvents } from '@librechat/agents'; import { ErrorTypes, Permissions, PermissionTypes } from 'librechat-data-provider'; import type { FiltersConfig } from 'librechat-data-provider'; import type { ChatCompletionDependencies } from './service'; +import type { ChatCompletionChunk } from './types'; import { createAgentChatCompletion } from './service'; jest.mock('@librechat/data-schemas', () => ({ @@ -104,6 +105,143 @@ describe('createAgentChatCompletion - MCP permission user propagation', () => { }; }); + it.each( + [true, false].flatMap((stream) => + ['wire-string', 'wire-object', 'native-string', 'idless'].map( + (shape) => [stream, shape] as const, + ), + ), + )( + 'retains complete and mixed snapshots through the service (stream=%s, shape=%s)', + async (stream, shape) => { + const req = createMockReq( + { id: 'user' }, + { model: 'agent_test', messages: [{ role: 'user', content: 'hi' }], stream }, + ); + const res = createMockRes(); + processStream.mockImplementationOnce(async () => { + const { customHandlers: h } = createRun.mock.calls[0][0] as Parameters< + NonNullable + >[0]; + const meta = { langgraph_node: 'agent=test', langgraph_step: 1 }; + await h.on_run_step.handle( + 'on_run_step', + { + id: 'step', + stepDetails: { + type: 'tool_calls', + tool_calls: [ + (() => { + if (shape === 'idless') + return { name: 'get_time', args: { city: 'Madrid' }, index: 0 }; + if (shape === 'native-string') + return { id: 'a', name: 'get_time', args: '{"city":"Madrid"}', index: 0 }; + return { + id: 'a', + function: { + name: 'get_time', + arguments: shape === 'wire-object' ? { city: 'Madrid' } : '{"city":"Madrid"}', + }, + index: 0, + }; + })(), + { id: 'b', name: 'get_time', args: {}, index: 1 }, + ], + }, + }, + meta, + ); + await h.on_run_step_delta.handle( + 'on_run_step_delta', + { + id: 'step', + delta: { type: 'tool_calls', tool_calls: [{ index: 1, args: '{"city":"Paris"}' }] }, + }, + meta, + ); + await h.on_message_delta.handle('on_message_delta', { + delta: { content: [{ type: 'text', text: 'Finished.' }] }, + }); + }); + await createAgentChatCompletion(req, res, deps); + if (stream) { + const frames = (res.write as jest.Mock).mock.calls + .map(([frame]: [string]) => frame) + .filter((frame) => frame !== 'data: [DONE]\n\n'); + const chunks: ChatCompletionChunk[] = frames.map((frame) => JSON.parse(frame.slice(6))); + const args = new Map(); + for (const chunk of chunks) + for (const call of chunk.choices[0].delta.tool_calls ?? []) + args.set(call.index, (args.get(call.index) ?? '') + (call.function?.arguments ?? '')); + expect(args.get(0)).toBe('{"city":"Madrid"}'); + expect(args.get(1)).toBe('{"city":"Paris"}'); + expect(chunks[chunks.length - 1].choices[0].finish_reason).toBe('stop'); + } else { + expect(getResponseMock(res, 'json')).toHaveBeenCalledWith( + expect.objectContaining({ + choices: [ + expect.objectContaining({ + finish_reason: 'stop', + message: expect.objectContaining({ + content: 'Finished.', + tool_calls: [ + { + id: shape === 'idless' ? 'call_0' : 'a', + type: 'function', + function: { name: 'get_time', arguments: '{"city":"Madrid"}' }, + }, + { + id: 'b', + type: 'function', + function: { name: 'get_time', arguments: '{"city":"Paris"}' }, + }, + ], + }), + }), + ], + }), + ); + } + }, + ); + + it.each([true, false])( + 'does not publish fallback arguments after provider failure (stream=%s)', + async (stream) => { + const req = createMockReq( + { id: 'user' }, + { model: 'agent_test', messages: [{ role: 'user', content: 'hi' }], stream }, + ); + const res = createMockRes(); + processStream.mockImplementationOnce(async () => { + const { customHandlers: h } = createRun.mock.calls[0][0] as Parameters< + NonNullable + >[0]; + await h.on_run_step.handle('on_run_step', { + id: 'step', + stepDetails: { + type: 'tool_calls', + tool_calls: [ + { id: 'a', function: { name: 'get_time', arguments: '{"city":"DO_NOT_FLUSH"}' } }, + ], + }, + }); + throw new Error('provider failed'); + }); + await createAgentChatCompletion(req, res, deps); + expect(JSON.stringify((res.write as jest.Mock).mock.calls)).not.toContain('DO_NOT_FLUSH'); + expect(JSON.stringify(getResponseMock(res, 'json').mock.calls)).not.toContain('DO_NOT_FLUSH'); + const { customHandlers: h } = createRun.mock.calls[0][0] as Parameters< + NonNullable + >[0]; + await h.on_run_step_delta.handle('on_run_step_delta', { + id: 'step', + delta: { type: 'tool_calls', tool_calls: [{ id: 'a', args: '{"late":true}' }] }, + }); + expect(JSON.stringify((res.write as jest.Mock).mock.calls)).not.toContain('late'); + }, + ); + it('forwards the role-bearing safe user to createRun and configurable.user', async () => { const req = createMockReq({ id: 'user-123', diff --git a/packages/api/src/agents/openai/service.ts b/packages/api/src/agents/openai/service.ts index e3c2f7c97e..1cfb002865 100644 --- a/packages/api/src/agents/openai/service.ts +++ b/packages/api/src/agents/openai/service.ts @@ -63,6 +63,7 @@ import { createOpenAIContentAggregator, createOpenAIStreamTracker, createOpenAIHandlers, + completeOpenAIToolCalls, sendFinalChunk, createChunk, writeSSE, @@ -833,19 +834,21 @@ export async function createAgentChatCompletion( : null; // Create event handlers - const eventHandlers = - isStreaming && handlerConfig - ? createOpenAIHandlers( - handlerConfig, - deps.toolExecuteOptions == null - ? undefined - : { - ...deps.toolExecuteOptions, - runSignal: abortController.signal, - foregroundRunId: requestId, - }, - ) - : {}; + const eventHandlers = createOpenAIHandlers( + handlerConfig + ? { ...handlerConfig, signal: abortController.signal } + : { + aggregator: aggregator!, + signal: abortController.signal, + }, + deps.toolExecuteOptions == null + ? undefined + : { + ...deps.toolExecuteOptions, + runSignal: abortController.signal, + foregroundRunId: requestId, + }, + ); // Convert messages to internal format const messages = convertMessages(request.messages); @@ -877,32 +880,40 @@ export async function createAgentChatCompletion( }); if (run) { - await run.processStream( - { messages }, + const target = tracker ?? aggregator!; + await completeOpenAIToolCalls( { - runName: 'AgentRun', - configurable: { - thread_id: conversationId, - user_id: userId, - user: safeUser, - requestBody: mcpRequestBody, - /** Same per-agent channel the in-repo controllers thread via - * `loadTools`: without it, the executor's PTC path cannot - * strip host-injected `intent` params from the schemas the - * sandbox bridge advertises on this route. */ - ...(initializedAgent.intentToolNames?.length - ? { intentToolNames: initializedAgent.intentToolNames } - : {}), - }, - recursionLimit: resolveRecursionLimit( - agentsConfig as Partial | undefined, - initializedAgent, - ), - signal: abortController.signal, - streamMode: 'values', - version: 'v2', + finish: () => target.finishToolCalls?.(), + abort: () => target.abortToolCalls?.(), }, - {}, + () => + run.processStream( + { messages }, + { + runName: 'AgentRun', + configurable: { + thread_id: conversationId, + user_id: userId, + user: safeUser, + requestBody: mcpRequestBody, + /** Same per-agent channel the in-repo controllers thread via + * `loadTools`: without it, the executor's PTC path cannot + * strip host-injected `intent` params from the schemas the + * sandbox bridge advertises on this route. */ + ...(initializedAgent.intentToolNames?.length + ? { intentToolNames: initializedAgent.intentToolNames } + : {}), + }, + recursionLimit: resolveRecursionLimit( + agentsConfig as Partial | undefined, + initializedAgent, + ), + signal: abortController.signal, + streamMode: 'values', + version: 'v2', + }, + {}, + ), ); } } @@ -912,6 +923,7 @@ export async function createAgentChatCompletion( sendFinalChunk(handlerConfig); res.end(); } else if (aggregator) { + aggregator.finishToolCalls?.(); // Build and send non-streaming response const usage: CompletionUsage = { prompt_tokens: aggregator.usage.promptTokens, diff --git a/packages/api/src/agents/openai/toolCalls.contract.spec.ts b/packages/api/src/agents/openai/toolCalls.contract.spec.ts new file mode 100644 index 0000000000..02bded4a6c --- /dev/null +++ b/packages/api/src/agents/openai/toolCalls.contract.spec.ts @@ -0,0 +1,308 @@ +import type { Agents } from 'librechat-data-provider'; +import type { ChatCompletionChunkChoice, ToolCall } from './types'; +import { createOpenAIToolCallStream } from './handlers'; + +/** Public contract fixtures, not a narrower reconstruction of the SDK fields. */ +describe('tool-call field normalization', () => { + it.each(['native-string', 'native-object', 'wire-string', 'wire-object'])( + 'normalizes %s exactly once', + (shape) => { + const toolCalls = new Map(); + const deltas: ChatCompletionChunkChoice['delta'][] = []; + const stream = createOpenAIToolCallStream({ toolCalls, emit: (delta) => deltas.push(delta) }); + const args = shape.endsWith('string') ? '{"city":"Madrid"}' : { city: 'Madrid' }; + const call: Agents.AgentToolCall = shape.startsWith('native') + ? { id: 'a', name: 'lookup', args } + : { id: 'a', type: 'function', function: { name: 'lookup', arguments: args } }; + stream.onRunStep({ id: 'step', stepDetails: { type: 'tool_calls', tool_calls: [call] } }); + stream.finish(); + expect(toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}'); + expect( + deltas + .flatMap((d) => d.tool_calls ?? []) + .map((c) => c.function?.arguments ?? '') + .join(''), + ).toBe('{"city":"Madrid"}'); + }, + ); + + it.each([false, true])( + 'assembles name substrings before the consumer freezes them (snapshot=%s)', + (snapshot) => { + const toolCalls = new Map(); + const deltas: ChatCompletionChunkChoice['delta'][] = []; + const stream = createOpenAIToolCallStream({ toolCalls, emit: (delta) => deltas.push(delta) }); + if (snapshot) + stream.onRunStep({ + id: 'step', + stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'a', name: 'get_', args: {} }] }, + }); + for (const chunk of [ + { id: 'a', name: 'get_', index: 0, args: '{}' }, + { name: 'weather', index: 0 }, + ]) { + stream.onRunStepDelta({ id: 'step', delta: { type: 'tool_calls', tool_calls: [chunk] } }); + } + stream.finish(); + expect(toolCalls.get(0)?.function.name).toBe('get_weather'); + expect( + deltas + .flatMap((d) => d.tool_calls ?? []) + .filter((c) => c.id) + .map((c) => c.function?.name), + ).toEqual(['get_weather']); + }, + ); + + it('synthesizes stable, distinct IDs for id-less complete calls and replayed declarations', () => { + const toolCalls = new Map(); + const stream = createOpenAIToolCallStream({ toolCalls }); + const calls: Agents.ToolCall[] = [ + { name: 'lookup', args: {} }, + { name: 'lookup', args: { city: 'Paris' } }, + ]; + const event = { id: 'step', stepDetails: { type: 'tool_calls', tool_calls: calls } }; + stream.onRunStep(event); + stream.onRunStep(event); + stream.onRunStep({ ...event, id: 'later' }); + stream.finish(); + expect(toolCalls.size).toBe(4); + expect(new Set([...toolCalls.values()].map((c) => c.id)).size).toBe(4); + expect([...toolCalls.values()].every((c) => !!c.id)).toBe(true); + }); + it.each(['native', 'wire'])('assembles split names and IDs with %s delta fields', (shape) => { + const toolCalls = new Map(); + const stream = createOpenAIToolCallStream({ toolCalls }); + for (const [id, name, args] of [ + ['call_', 'get_', '{"city":'], + ['123', 'weather', '"Paris"}'], + ]) { + stream.onRunStepDelta({ + id: 'step', + delta: { + type: 'tool_calls', + tool_calls: [ + shape === 'native' + ? { index: 4, id, name, args } + : { index: 4, id, function: { name, arguments: args } }, + ], + }, + }); + } + stream.finish(); + expect([...toolCalls.values()]).toEqual([ + { + id: 'call_123', + type: 'function', + function: { name: 'get_weather', arguments: '{"city":"Paris"}' }, + }, + ]); + }); + + it.each([true, false])( + 'keeps id-less calls distinct with mixed raw data (explicit index=%s)', + (explicit) => { + const toolCalls = new Map(); + const stream = createOpenAIToolCallStream({ toolCalls }); + stream.onRunStep({ + id: 'step', + stepDetails: { + type: 'tool_calls', + tool_calls: [ + { name: 'get_', args: {}, ...(explicit && { index: 5 }) }, + { name: 'lookup', args: { city: 'Paris' }, ...(explicit && { index: 8 }) }, + ], + }, + }); + const index = explicit ? 5 : 0; + stream.onRunStepDelta({ + id: 'step', + delta: { + type: 'tool_calls', + tool_calls: [{ index, name: 'get_', args: '{"city":"Madrid"}' }], + }, + }); + stream.onRunStepDelta({ + id: 'step', + delta: { type: 'tool_calls', tool_calls: [{ index, name: 'weather' }] }, + }); + stream.finish(); + expect([...toolCalls.values()].map((call) => call.function)).toEqual([ + { name: 'get_weather', arguments: '{"city":"Madrid"}' }, + { name: 'lookup', arguments: '{"city":"Paris"}' }, + ]); + expect(new Set([...toolCalls.values()].map((call) => call.id)).size).toBe(2); + }, + ); + + it('correlates an id-less declaration with a later identified chunk rather than adding a call', () => { + const toolCalls = new Map(); + const stream = createOpenAIToolCallStream({ toolCalls }); + stream.onRunStep({ + id: 'step', + stepDetails: { type: 'tool_calls', tool_calls: [{ name: 'lookup', args: {} }] }, + }); + stream.onRunStepDelta({ + id: 'step', + delta: { + type: 'tool_calls', + tool_calls: [{ id: 'provider', name: 'lookup', index: 9, args: '{}' }], + }, + }); + stream.finish(); + expect(toolCalls.size).toBe(1); + expect(toolCalls.get(0)?.id).toBe('provider'); + }); + + it('assembles name fragments before and after a full snapshot without appending the snapshot', () => { + const toolCalls = new Map(); + const stream = createOpenAIToolCallStream({ toolCalls }); + stream.onRunStepDelta({ + id: 'step', + delta: { type: 'tool_calls', tool_calls: [{ id: 'a', index: 0, name: 'get_', args: '{}' }] }, + }); + stream.onRunStep({ + id: 'step', + stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'a', name: 'get_weather', args: {} }] }, + }); + stream.onRunStepDelta({ + id: 'step', + delta: { type: 'tool_calls', tool_calls: [{ index: 0, name: 'weather' }] }, + }); + stream.finish(); + expect(toolCalls.get(0)?.function.name).toBe('get_weather'); + }); + + it('seals before invoking an emitter that reenters finish or throws', () => { + const toolCalls = new Map(); + const emit = jest.fn(() => { + stream.finish(); + throw new Error('transport failed'); + }); + const stream = createOpenAIToolCallStream({ toolCalls, emit }); + stream.onRunStep({ + id: 'step', + stepDetails: { type: 'tool_calls', tool_calls: [{ name: 'lookup', args: {} }] }, + }); + expect(() => stream.finish()).toThrow('transport failed'); + expect(() => stream.finish()).not.toThrow(); + expect(emit).toHaveBeenCalledTimes(1); + }); + + it('does not emit any tool identity if another call is malformed', () => { + const toolCalls = new Map(); + const emit = jest.fn(); + const stream = createOpenAIToolCallStream({ toolCalls, emit }); + stream.onRunStep({ + id: 'step', + stepDetails: { + type: 'tool_calls', + tool_calls: [ + { name: 'lookup', args: {} }, + { name: 'lookup', args: 'NOT_JSON' }, + ], + }, + }); + expect(() => stream.finish()).toThrow('Invalid tool call arguments'); + expect(emit).not.toHaveBeenCalled(); + expect(toolCalls.size).toBe(0); + }); + it('matches late id-less snapshot arrays to already indexed raw calls', () => { + const toolCalls = new Map(); + const stream = createOpenAIToolCallStream({ toolCalls }); + for (const index of [0, 1]) + stream.onRunStepDelta({ + id: 'step', + delta: { + type: 'tool_calls', + tool_calls: [{ index, name: 'lookup', args: JSON.stringify({ index }) }], + }, + }); + stream.onRunStep({ + id: 'step', + stepDetails: { + type: 'tool_calls', + tool_calls: [ + { name: 'lookup', args: { index: 0 } }, + { name: 'lookup', args: { index: 1 } }, + ], + }, + }); + stream.finish(); + expect(toolCalls.size).toBe(2); + expect([...toolCalls.values()].map((call) => call.function.arguments)).toEqual([ + '{"index":0}', + '{"index":1}', + ]); + }); + + it('separates fully identified raw calls when no provider index is supplied', () => { + const toolCalls = new Map(); + const stream = createOpenAIToolCallStream({ toolCalls }); + for (const id of ['a', 'b']) + stream.onRunStepDelta({ + id: 'step', + delta: { type: 'tool_calls', tool_calls: [{ id, name: 'lookup', args: '{}' }] }, + }); + stream.finish(); + expect([...toolCalls.values()].map((call) => call.id)).toEqual(['a', 'b']); + }); + + it('does not guess that identical name fragments are replayed data', () => { + const toolCalls = new Map(); + const stream = createOpenAIToolCallStream({ toolCalls }); + for (const name of ['a', 'a']) + stream.onRunStepDelta({ + id: 'step', + delta: { type: 'tool_calls', tool_calls: [{ index: 0, name }] }, + }); + stream.onRunStepDelta({ + id: 'step', + delta: { type: 'tool_calls', tool_calls: [{ index: 0, args: '{}' }] }, + }); + stream.finish(); + expect(toolCalls.get(0)?.function.name).toBe('aa'); + }); + it.each([true, false])( + 'does not alias parallel shared ID prefixes (snapshots=%s)', + (snapshots) => { + const toolCalls = new Map(); + const stream = createOpenAIToolCallStream({ toolCalls }); + if (snapshots) + stream.onRunStep({ + id: 'step', + stepDetails: { + type: 'tool_calls', + tool_calls: [ + { id: 'call_0', name: 'get_weather', args: {} }, + { id: 'call_1', name: 'get_weather', args: {} }, + ], + }, + }); + for (const index of [0, 1]) + stream.onRunStepDelta({ + id: 'step', + delta: { + type: 'tool_calls', + tool_calls: [{ index, id: 'call_', name: 'get_', args: '{"i":' }], + }, + }); + for (const index of [1, 0]) + stream.onRunStepDelta({ + id: 'step', + delta: { + type: 'tool_calls', + tool_calls: [{ index, id: String(index), name: 'weather', args: `${index}}` }], + }, + }); + stream.finish(); + expect([...toolCalls.values()]).toEqual( + [0, 1].map((i) => ({ + id: `call_${i}`, + type: 'function', + function: { name: 'get_weather', arguments: JSON.stringify({ i }) }, + })), + ); + }, + ); +}); diff --git a/packages/api/src/agents/openai/toolCalls.graph.spec.ts b/packages/api/src/agents/openai/toolCalls.graph.spec.ts new file mode 100644 index 0000000000..302cc65dff --- /dev/null +++ b/packages/api/src/agents/openai/toolCalls.graph.spec.ts @@ -0,0 +1,302 @@ +import { response } from 'express'; +import { AIMessageChunk } from '@librechat/agents/langchain/messages'; +import { + ChatModelStreamHandler, + HandlerRegistry, + Providers, + StandardGraph, +} from '@librechat/agents'; +import type { Response } from 'express'; +import type { ChatCompletionChunk, ToolCall } from './types'; +import { + createOpenAIHandlers, + createOpenAIStreamTracker, + createOpenAIContentAggregator, + createOpenAIToolCallStream, + OpenAIRunStepHandler, + OpenAIRunStepDeltaHandler, + sendFinalChunk, +} from './handlers'; + +/** No graph event fixtures: the SDK generates run steps and deltas from real + * AIMessageChunk instances. Only the provider transport is omitted. */ +describe('tool-call projection with real SDK graph dispatch', () => { + it.each([true, false])( + 'preserves interleaved calls across SDK steps (stream=%s)', + async (streaming) => { + const frames: string[] = []; + const tracker = createOpenAIStreamTracker(); + const context = { requestId: 'test', created: 1, model: 'fixture' }; + const res: Response = Object.create(response); + jest.spyOn(res, 'write').mockImplementation((frame) => { + frames.push(String(frame)); + return true; + }); + const config = { tracker, context, res }; + const stream = createOpenAIToolCallStream({ toolCalls: tracker.toolCalls }); + const handlers = streaming + ? createOpenAIHandlers(config) + : { + on_run_step: new OpenAIRunStepHandler(stream), + on_run_step_delta: new OpenAIRunStepDeltaHandler(stream), + }; + const graph = new StandardGraph({ + runId: 'test', + agents: [{ agentId: 'agent', provider: Providers.OPENAI, tools: [] }], + }); + graph.config = { configurable: { run_id: 'test', thread_id: 'thread' } }; + graph.handlerRegistry = new HandlerRegistry(); + for (const [event, handler] of Object.entries(handlers)) { + graph.handlerRegistry.register(event, handler); + } + const producer = new ChatModelStreamHandler(); + for (const chunk of [ + { index: 0, id: 'a', name: 'get_time', args: '' }, + { index: 1, id: 'b', name: 'get_time', args: '' }, + { index: 0, args: '{"city":' }, + { index: 1, args: '{"city":"Paris"}' }, + { index: 0, args: '"Madrid"}' }, + ]) { + await producer.handle( + 'on_chat_model_stream', + { + chunk: new AIMessageChunk({ + content: '', + tool_call_chunks: [{ ...chunk, type: 'tool_call_chunk' }], + }), + }, + { langgraph_node: 'agent=agent', langgraph_step: 1 }, + graph, + ); + } + expect(frames).toEqual([]); + if (streaming) tracker.finishToolCalls?.(); + else stream.finish(); + expect([...tracker.toolCalls.values()].map((call) => call.function.arguments)).toEqual([ + '{"city":"Madrid"}', + '{"city":"Paris"}', + ]); + if (streaming) { + const received = new Map(); + for (const frame of frames) { + const chunk: ChatCompletionChunk = JSON.parse(frame.slice(6)); + for (const part of chunk.choices[0].delta.tool_calls ?? []) { + const existing = received.get(part.index); + if (existing) { + existing.function.arguments += part.function?.arguments ?? ''; + } else { + expect(part.id).toBeDefined(); + expect(part.function?.name).toBe('get_time'); + received.set(part.index, { + id: part.id!, + type: 'function', + function: { + name: part.function!.name!, + arguments: part.function?.arguments ?? '', + }, + }); + } + } + } + expect([...received.values()]).toEqual([...tracker.toolCalls.values()]); + await producer.handle( + 'on_chat_model_stream', + { + chunk: new AIMessageChunk({ content: 'Both tools completed. Here is the answer.' }), + }, + { langgraph_node: 'agent=agent', langgraph_step: 3 }, + graph, + ); + expect(tracker.hasText).toBe(true); + sendFinalChunk(config, 'stop'); + const final: ChatCompletionChunk = JSON.parse(frames[frames.length - 2].slice(6)); + expect(final.choices[0].finish_reason).toBe('stop'); + } + }, + ); + + it('isolates provider index zero across agents, invocations and checkpoint namespaces', () => { + const tracker = createOpenAIStreamTracker(); + const stream = createOpenAIToolCallStream({ toolCalls: tracker.toolCalls }); + const metadata = [ + { langgraph_node: 'agent=a', langgraph_step: 1 }, + { langgraph_node: 'agent=b', langgraph_step: 1 }, + { langgraph_node: 'agent=a', langgraph_step: 3 }, + { langgraph_node: 'agent=a', langgraph_step: 3, checkpoint_ns: 'child' }, + ]; + for (const [i, meta] of metadata.entries()) { + stream.onRunStep( + { + id: `step_${i}`, + stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'call_0', name: 'get_time' }] }, + }, + meta, + ); + stream.onRunStepDelta( + { + id: `step_${i}`, + delta: { type: 'tool_calls', tool_calls: [{ index: 0, id: 'call_0', name: 'get_time' }] }, + }, + meta, + ); + } + for (const [i, meta] of metadata.entries()) { + stream.onRunStepDelta( + { + id: `step_${i}`, + delta: { type: 'tool_calls', tool_calls: [{ index: 0, args: JSON.stringify({ i }) }] }, + }, + meta, + ); + } + stream.finish(); + expect([...tracker.toolCalls.values()].map((call) => call.function.arguments)).toEqual( + metadata.map((_, i) => JSON.stringify({ i })), + ); + expect(new Set([...tracker.toolCalls.values()].map((call) => call.id)).size).toBe(4); + }); + it.each([true, false])( + 'projects complete-only SDK messages at response completion (stream=%s)', + async (streaming) => { + const frames: string[] = []; + const res: Response = Object.create(response); + jest.spyOn(res, 'write').mockImplementation((frame) => { + frames.push(String(frame)); + return true; + }); + const tracker = createOpenAIStreamTracker(); + const aggregator = createOpenAIContentAggregator(); + const config = { + tracker, + res, + context: { requestId: 'complete', created: 1, model: 'fixture' }, + }; + const handlers = createOpenAIHandlers(streaming ? config : { aggregator }); + const graph = new StandardGraph({ + runId: 'complete', + agents: [{ agentId: 'agent', provider: Providers.OPENAI, tools: [] }], + }); + graph.config = { configurable: { run_id: 'complete', thread_id: 'thread' } }; + graph.handlerRegistry = new HandlerRegistry(); + for (const [event, handler] of Object.entries(handlers)) + graph.handlerRegistry.register(event, handler); + await new ChatModelStreamHandler().handle( + 'on_chat_model_stream', + { + chunk: new AIMessageChunk({ + content: '', + tool_calls: [{ id: 'a', name: 'get_time', args: { city: 'Madrid' } }], + }), + }, + { langgraph_node: 'agent=agent', langgraph_step: 1 }, + graph, + ); + const target = streaming ? tracker : aggregator; + expect(target.toolCalls.size).toBe(0); + if (streaming) sendFinalChunk(config); + else target.finishToolCalls?.(); + expect(target.toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}'); + if (streaming) { + const chunks: ChatCompletionChunk[] = frames + .filter((frame) => frame !== 'data: [DONE]\n\n') + .map((frame) => JSON.parse(frame.slice(6))); + expect( + chunks + .flatMap((chunk) => chunk.choices[0].delta.tool_calls ?? []) + .map((call) => call.function?.arguments) + .join(''), + ).toBe('{"city":"Madrid"}'); + } + }, + ); + + it('uses graph-owned stream segments without stealing late fragments from the old segment', () => { + const graph = new StandardGraph({ + runId: 'segments', + agents: [{ agentId: 'a', provider: Providers.OPENAI, tools: [] }], + }); + graph.config = { configurable: { run_id: 'segments', thread_id: 'thread' } }; + const meta = { langgraph_node: 'agent=a', langgraph_step: 1 }; + const tracker = createOpenAIStreamTracker(); + const stream = createOpenAIToolCallStream({ toolCalls: tracker.toolCalls }); + const start = new OpenAIRunStepHandler(stream); + const delta = new OpenAIRunStepDeltaHandler(stream); + for (let i = 0; i < 2; i++) { + start.handle( + 'on_run_step', + { + id: `step_${i}`, + stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'call_0', name: 'get_time' }] }, + }, + meta, + graph, + ); + delta.handle( + 'on_run_step_delta', + { + id: `step_${i}`, + delta: { type: 'tool_calls', tool_calls: [{ id: 'call_0', index: 0, name: 'get_time' }] }, + }, + meta, + graph, + ); + graph.advanceStreamSegment(); + } + for (let i = 0; i < 2; i++) + delta.handle( + 'on_run_step_delta', + { + id: `step_${i}`, + delta: { type: 'tool_calls', tool_calls: [{ index: 0, args: JSON.stringify({ i }) }] }, + }, + meta, + graph, + ); + stream.finish(); + expect([...tracker.toolCalls.values()].map((call) => call.function.arguments)).toEqual([ + '{"i":0}', + '{"i":1}', + ]); + }); + it('assembles the name before emitting when real SDK chunks split it', async () => { + const tracker = createOpenAIStreamTracker(); + const emitted: ChatCompletionChunk['choices'][number]['delta'][] = []; + const stream = createOpenAIToolCallStream({ + toolCalls: tracker.toolCalls, + emit: (delta) => emitted.push(delta), + }); + const graph = new StandardGraph({ + runId: 'split', + agents: [{ agentId: 'a', provider: Providers.OPENAI, tools: [] }], + }); + graph.config = { configurable: { run_id: 'split', thread_id: 'split' } }; + graph.handlerRegistry = new HandlerRegistry(); + const handlers: ReturnType = { + on_run_step: new OpenAIRunStepHandler(stream), + on_run_step_delta: new OpenAIRunStepDeltaHandler(stream), + }; + for (const [event, handler] of Object.entries(handlers)) + graph.handlerRegistry.register(event, handler); + const producer = new ChatModelStreamHandler(); + for (const chunk of [ + { id: 'a', index: 0, name: 'get_', args: '' }, + { index: 0, args: '{}' }, + { index: 0, name: 'weather' }, + ]) + await producer.handle( + 'on_chat_model_stream', + { + chunk: new AIMessageChunk({ + content: '', + tool_call_chunks: [{ ...chunk, type: 'tool_call_chunk' }], + }), + }, + { langgraph_node: 'agent=a', langgraph_step: 1 }, + graph, + ); + expect(emitted).toEqual([]); + stream.finish(); + expect(tracker.toolCalls.get(0)?.function.name).toBe('get_weather'); + expect(emitted[0].tool_calls?.[0].function?.name).toBe('get_weather'); + }); +}); diff --git a/packages/api/src/agents/openai/toolCalls.spec.ts b/packages/api/src/agents/openai/toolCalls.spec.ts new file mode 100644 index 0000000000..cc57491400 --- /dev/null +++ b/packages/api/src/agents/openai/toolCalls.spec.ts @@ -0,0 +1,616 @@ +import type { ChatCompletionChunkChoice, OpenAIResponseContext, ToolCall } from './types'; +import type { RunStepData, RunStepDeltaData } from './handlers'; +import { + sendFinalChunk, + completeOpenAIToolCalls, + createOpenAIStreamTracker, + createOpenAIToolCallStream, + createOpenAIContentAggregator, +} from './handlers'; +import { buildNonStreamingResponse } from './service'; + +type Delta = ChatCompletionChunkChoice['delta']; + +/** + * Event payloads below are the shapes `@librechat/agents` actually dispatches + * for a Bedrock Converse stream: the run step opens a call with its id and name + * and carries no tool-call index, the first delta fragment repeats id and name + * with the provider's content-block index, and later fragments carry only that + * index and an argument slice. A step's own `index` is its position in the + * response content, which is why it appears here as an unrelated number. + */ +function runStep(stepId: string, contentIndex: number, id: string, name: string): RunStepData { + return { + id: stepId, + index: contentIndex, + stepDetails: { + type: 'tool_calls', + tool_calls: [{ name, args: {}, id, type: 'tool_call' }], + }, + }; +} + +function opensCall( + stepId: string, + providerIndex: number, + id: string, + name: string, +): RunStepDeltaData { + return { + id: stepId, + delta: { + type: 'tool_calls', + tool_calls: [{ name, id, index: providerIndex, type: 'tool_call_chunk' }], + }, + }; +} + +function streamsArgs(stepId: string, providerIndex: number, args: string): RunStepDeltaData { + return { + id: stepId, + delta: { + type: 'tool_calls', + tool_calls: [{ args, index: providerIndex, type: 'tool_call_chunk' }], + }, + }; +} + +/** + * The accumulation an OpenAI-compatible client performs, including the rule that + * produced the reported `AI_InvalidResponseDataError: Expected 'id' to be a + * string`: the first chunk seen at an index must declare the call. + */ +function accumulateLikeClient(deltas: Delta[]) { + const calls = new Map(); + for (const delta of deltas) { + for (const fragment of delta.tool_calls ?? []) { + const index = fragment.index as number; + const open = calls.get(index); + if (open === undefined) { + if (!fragment.id || !fragment.function?.name) { + throw new Error(`first chunk at index ${index} declares no id and name`); + } + calls.set(index, { + id: fragment.id, + name: fragment.function.name, + arguments: fragment.function.arguments ?? '', + }); + continue; + } + open.arguments += fragment.function?.arguments ?? ''; + } + } + return [...calls.entries()].map(([index, call]) => ({ index, ...call })); +} + +function streamingBridge() { + const deltas: Delta[] = []; + const toolCalls = new Map(); + const stream = createOpenAIToolCallStream({ + toolCalls, + emit: (delta) => deltas.push(delta), + }); + return { stream, deltas, toolCalls }; +} + +describe('outward tool call indexes', () => { + it('keeps one index from the declaration through the argument fragments', () => { + const { stream, deltas, toolCalls } = streamingBridge(); + + stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time')); + stream.onRunStepDelta(opensCall('step_1', 1, 'call_a', 'get_time')); + stream.onRunStepDelta(streamsArgs('step_1', 1, '{"city":')); + stream.onRunStepDelta(streamsArgs('step_1', 1, '"Madrid"}')); + + expect(deltas).toEqual([]); + stream.finish(); + expect(deltas.flatMap((delta) => delta.tool_calls ?? []).map((call) => call.index)).toEqual([ + 0, 0, + ]); + expect(accumulateLikeClient(deltas)).toEqual([ + { index: 0, id: 'call_a', name: 'get_time', arguments: '{"city":"Madrid"}' }, + ]); + expect([...toolCalls.values()]).toEqual([ + { + id: 'call_a', + type: 'function', + function: { name: 'get_time', arguments: '{"city":"Madrid"}' }, + }, + ]); + }); + + it('declares a call once when the run step and the first fragment both name it', () => { + const { stream, deltas } = streamingBridge(); + + stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time')); + stream.onRunStepDelta(opensCall('step_1', 1, 'call_a', 'get_time')); + + stream.finish(); + const declarations = deltas + .flatMap((delta) => delta.tool_calls ?? []) + .filter((call) => call.id !== undefined); + expect(declarations).toEqual([ + { index: 0, id: 'call_a', type: 'function', function: { name: 'get_time', arguments: '' } }, + ]); + }); + + it('separates parallel calls that arrive as consecutive single-call steps', () => { + const { stream, deltas, toolCalls } = streamingBridge(); + + stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time')); + stream.onRunStepDelta(opensCall('step_1', 0, 'call_a', 'get_time')); + stream.onRunStepDelta(streamsArgs('step_1', 0, '{"city":"Madrid"}')); + stream.onRunStep(runStep('step_2', 2, 'call_b', 'get_time')); + stream.onRunStepDelta(opensCall('step_2', 1, 'call_b', 'get_time')); + stream.onRunStepDelta(streamsArgs('step_2', 1, '{"city":"Paris"}')); + + stream.finish(); + expect(accumulateLikeClient(deltas)).toEqual([ + { index: 0, id: 'call_a', name: 'get_time', arguments: '{"city":"Madrid"}' }, + { index: 1, id: 'call_b', name: 'get_time', arguments: '{"city":"Paris"}' }, + ]); + expect([...toolCalls.keys()]).toEqual([0, 1]); + }); + + it('allocates a new index when a later model invocation reuses provider index zero', () => { + const { stream, deltas, toolCalls } = streamingBridge(); + + stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time')); + stream.onRunStepDelta(opensCall('step_1', 0, 'call_a', 'get_time')); + stream.onRunStepDelta(streamsArgs('step_1', 0, '{"city":"Madrid"}')); + stream.onRunStep(runStep('step_3', 3, 'call_b', 'get_time')); + stream.onRunStepDelta(opensCall('step_3', 0, 'call_b', 'get_time')); + stream.onRunStepDelta(streamsArgs('step_3', 0, '{"city":"Paris"}')); + + stream.finish(); + expect(accumulateLikeClient(deltas)).toEqual([ + { index: 0, id: 'call_a', name: 'get_time', arguments: '{"city":"Madrid"}' }, + { index: 1, id: 'call_b', name: 'get_time', arguments: '{"city":"Paris"}' }, + ]); + expect(toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}'); + expect(toolCalls.get(1)?.function.arguments).toBe('{"city":"Paris"}'); + }); + + it('attributes unidentified argument fragments to the call its step opened', () => { + const { stream, deltas } = streamingBridge(); + + stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time')); + stream.onRunStepDelta(streamsArgs('step_1', 7, '{"city":"Madrid"}')); + + stream.finish(); + expect(accumulateLikeClient(deltas)).toEqual([ + { index: 0, id: 'call_a', name: 'get_time', arguments: '{"city":"Madrid"}' }, + ]); + }); + + it('drops a fragment it cannot attribute rather than charging another call', () => { + const { stream, deltas, toolCalls } = streamingBridge(); + + stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time')); + stream.onRunStepDelta(opensCall('step_1', 0, 'call_a', 'get_time')); + stream.onRunStep(runStep('step_2', 2, 'call_b', 'get_time')); + stream.onRunStepDelta(opensCall('step_2', 1, 'call_b', 'get_time')); + stream.onRunStepDelta({ + id: 'step_unknown', + delta: { type: 'tool_calls', tool_calls: [{ index: 4, args: '{"city":"Lisbon"}' }] }, + }); + + expect(() => stream.finish()).toThrow('Unattributable tool call arguments'); + expect(toolCalls.size).toBe(0); + expect(deltas).toEqual([]); + }); + + it.each([true, false])( + 'isolates repeated provider IDs across interleaved steps (stream=%s)', + (streaming) => { + const deltas: Delta[] = []; + const toolCalls = new Map(); + const stream = createOpenAIToolCallStream({ + toolCalls, + emit: streaming ? (delta) => deltas.push(delta) : undefined, + }); + stream.onRunStep(runStep('agent_a_turn_1', 1, 'call_0', 'get_time')); + stream.onRunStepDelta(opensCall('agent_a_turn_1', 0, 'call_0', 'get_time')); + stream.onRunStepDelta(streamsArgs('agent_a_turn_1', 0, '{"city":')); + stream.onRunStep(runStep('agent_b_turn_1', 2, 'call_0', 'get_time')); + stream.onRunStepDelta(opensCall('agent_b_turn_1', 0, 'call_0', 'get_time')); + stream.onRunStepDelta(streamsArgs('agent_b_turn_1', 0, '{"city":"Paris"}')); + stream.onRunStepDelta(streamsArgs('agent_a_turn_1', 0, '"Madrid"}')); + stream.onRunStep(runStep('agent_a_turn_2', 3, 'call_0', 'get_time')); + stream.onRunStepDelta(opensCall('agent_a_turn_2', 0, 'call_0', 'get_time')); + stream.onRunStepDelta(streamsArgs('agent_a_turn_2', 0, '{"city":"Lisbon"}')); + stream.finish(); + expect([...toolCalls.keys()]).toEqual([0, 1, 2]); + expect([...toolCalls.values()].map((call) => call.function.arguments)).toEqual([ + '{"city":"Madrid"}', + '{"city":"Paris"}', + '{"city":"Lisbon"}', + ]); + if (streaming) { + expect(accumulateLikeClient(deltas).map((call) => call.arguments)).toEqual([ + '{"city":"Madrid"}', + '{"city":"Paris"}', + '{"city":"Lisbon"}', + ]); + } + }, + ); + + it('declares OpenAI-shaped names without waiting for an identifying delta', () => { + const { stream, deltas, toolCalls } = streamingBridge(); + stream.onRunStep({ + id: 'step_function', + stepDetails: { + type: 'tool_calls', + tool_calls: [ + { + id: 'call_a', + type: 'function', + function: { name: 'get_time', arguments: '{"city":"Madrid"}' }, + }, + ], + }, + }); + stream.finish(); + expect(toolCalls.get(0)?.function.name).toBe('get_time'); + expect(accumulateLikeClient(deltas)).toMatchObject([ + { index: 0, id: 'call_a', name: 'get_time' }, + ]); + }); + + it.each([true, false])( + 'binds multiple declared calls before ID-less fragments (explicit indexes=%s)', + (explicit) => { + const { stream, deltas, toolCalls } = streamingBridge(); + const indexes = explicit ? [4, 8] : [0, 1]; + stream.onRunStep({ + id: 'parallel', + index: 12, + stepDetails: { + type: 'tool_calls', + tool_calls: [ + { id: 'call_a', name: 'get_time', ...(explicit && { index: indexes[0] }) }, + { id: 'call_b', name: 'get_time', ...(explicit && { index: indexes[1] }) }, + ], + }, + }); + stream.onRunStepDelta(streamsArgs('parallel', indexes[1], '{"city":"Paris"}')); + stream.onRunStepDelta(streamsArgs('parallel', indexes[0], '{"city":"Madrid"}')); + stream.finish(); + stream.finish(); + expect(accumulateLikeClient(deltas).map((call) => call.arguments)).toEqual([ + '{"city":"Madrid"}', + '{"city":"Paris"}', + ]); + expect([...toolCalls.values()].map((call) => call.function.arguments)).toEqual([ + '{"city":"Madrid"}', + '{"city":"Paris"}', + ]); + }, + ); + + it('does not re-declare a replayed step or bind unrelated indexes to a known call', () => { + const { stream, deltas, toolCalls } = streamingBridge(); + stream.onRunStep(runStep('step', 1, 'call_a', 'get_time')); + stream.onRunStep(runStep('step', 1, 'call_a', 'get_time')); + stream.onRunStepDelta(opensCall('step', 3, 'call_a', 'get_time')); + stream.onRunStepDelta(streamsArgs('step', 9, '{"wrong":true}')); + stream.onRunStepDelta(streamsArgs('step', 3, '{}')); + expect(() => stream.finish()).toThrow('Unattributable tool call arguments'); + expect(deltas).toEqual([]); + expect(toolCalls.size).toBe(0); + }); + + it('buffers identified arguments until a later name-only fragment can declare the call', () => { + const { stream, deltas, toolCalls } = streamingBridge(); + stream.onRunStepDelta({ + id: 'step', + delta: { type: 'tool_calls', tool_calls: [{ id: 'call_a', index: 3, args: '{"city":' }] }, + }); + expect(deltas).toEqual([]); + stream.onRunStepDelta({ + id: 'step', + delta: { + type: 'tool_calls', + tool_calls: [{ index: 3, function: { name: 'get_time', arguments: '"Madrid"}' } }], + }, + }); + stream.finish(); + expect(accumulateLikeClient(deltas)).toEqual([ + { index: 0, id: 'call_a', name: 'get_time', arguments: '{"city":"Madrid"}' }, + ]); + expect(toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}'); + }); + + it('prefers identified provider indexes to declaration position', () => { + const { stream, deltas } = streamingBridge(); + stream.onRunStep({ + id: 'step', + stepDetails: { + type: 'tool_calls', + tool_calls: [ + { id: 'call_a', name: 'get_time' }, + { id: 'call_b', name: 'get_time' }, + ], + }, + }); + stream.onRunStepDelta(opensCall('step', 1, 'call_a', 'get_time')); + stream.onRunStepDelta(opensCall('step', 0, 'call_b', 'get_time')); + stream.onRunStepDelta(streamsArgs('step', 0, '{"city":"Paris"}')); + stream.onRunStepDelta(streamsArgs('step', 1, '{"city":"Madrid"}')); + stream.finish(); + expect(accumulateLikeClient(deltas).map((call) => call.arguments)).toEqual([ + '{"city":"Madrid"}', + '{"city":"Paris"}', + ]); + }); + + it('never merges unnamed-index fragments into a multi-call step', () => { + const { stream, toolCalls } = streamingBridge(); + stream.onRunStep({ + id: 'step', + stepDetails: { + type: 'tool_calls', + tool_calls: [ + { id: 'a', name: 'get_time' }, + { id: 'b', name: 'get_time' }, + ], + }, + }); + stream.onRunStepDelta({ + id: 'step', + delta: { type: 'tool_calls', tool_calls: [{ args: 'unattributable' }] }, + }); + expect(() => stream.finish()).toThrow('Unattributable tool call arguments'); + expect(toolCalls.size).toBe(0); + }); + + it('gives id-based consumers unique IDs and starts each response with fresh state', () => { + const { stream, toolCalls } = streamingBridge(); + stream.onRunStep(runStep('s1', 1, 'call_0', 'get_time')); + stream.onRunStep(runStep('s2', 2, 'call_0', 'get_time')); + stream.onRunStep(runStep('s3', 3, 'call_0_1', 'get_time')); + stream.finish(); + expect(new Set([...toolCalls.values()].map((call) => call.id)).size).toBe(3); + const second = streamingBridge(); + second.stream.onRunStep(runStep('s1', 1, 'call_0', 'get_time')); + second.stream.finish(); + expect([...second.toolCalls.keys()]).toEqual([0]); + expect(second.toolCalls.get(0)?.id).toBe('call_0'); + }); + + it('accumulates the same calls for a non-streaming response with no emitter', () => { + const aggregator = createOpenAIContentAggregator(); + const stream = createOpenAIToolCallStream({ toolCalls: aggregator.toolCalls }); + + stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time')); + stream.onRunStepDelta(opensCall('step_1', 0, 'call_a', 'get_time')); + stream.onRunStepDelta(streamsArgs('step_1', 0, '{"city":"Madrid"}')); + stream.onRunStep(runStep('step_3', 3, 'call_b', 'get_time')); + stream.onRunStepDelta(opensCall('step_3', 0, 'call_b', 'get_time')); + stream.onRunStepDelta(streamsArgs('step_3', 0, '{"city":"Paris"}')); + + stream.finish(); + const response = buildNonStreamingResponse( + { requestId: 'chatcmpl-test', created: 1778317637, model: 'agent_test' }, + 'Checking both.', + '', + aggregator.toolCalls, + { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, + ); + + expect(response.choices[0].message.tool_calls).toEqual([ + { + id: 'call_a', + type: 'function', + function: { name: 'get_time', arguments: '{"city":"Madrid"}' }, + }, + { + id: 'call_b', + type: 'function', + function: { name: 'get_time', arguments: '{"city":"Paris"}' }, + }, + ]); + }); +}); + +describe('complete snapshots and streamed fragments', () => { + function snapshot(step = 'complete', args = '{"city":"Madrid"}') { + return { + id: step, + stepDetails: { + type: 'tool_calls', + tool_calls: [{ id: 'a', function: { name: 'get_time', arguments: args } }], + }, + }; + } + + it.each(['before', 'after', 'between'])( + 'never concatenates a snapshot %s raw fragments', + (order) => { + const { stream, deltas, toolCalls } = streamingBridge(); + if (order === 'before') stream.onRunStep(snapshot()); + stream.onRunStepDelta(opensCall('complete', 0, 'a', 'get_time')); + stream.onRunStepDelta(streamsArgs('complete', 0, '{"city":')); + if (order === 'between') stream.onRunStep(snapshot()); + stream.onRunStepDelta(streamsArgs('complete', 0, '"Madrid"}')); + if (order === 'after') stream.onRunStep(snapshot()); + stream.finish(); + expect(accumulateLikeClient(deltas)[0].arguments).toBe('{"city":"Madrid"}'); + expect(toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}'); + }, + ); + + it('captures a native snapshot without retaining the producer object', () => { + const { stream, toolCalls } = streamingBridge(); + const args = { city: 'Madrid' }; + stream.onRunStep({ + id: 'step', + stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'a', name: 'get_time', args }] }, + }); + args.city = 'Paris'; + stream.finish(); + expect(toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}'); + }); + + it('preserves empty native objects and does not flush a snapshot on identifying-only chunks', () => { + const { stream, deltas } = streamingBridge(); + stream.onRunStep({ + id: 'complete', + stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'a', name: 'noop', args: {} }] }, + }); + stream.onRunStepDelta(opensCall('complete', 0, 'a', 'noop')); + expect(deltas).toEqual([]); + stream.finish(); + expect(accumulateLikeClient(deltas)[0].arguments).toBe('{}'); + }); + + it('flushes replayed declarations only once and ignores writes after completion', () => { + const { stream, deltas } = streamingBridge(); + stream.onRunStep(snapshot()); + stream.onRunStep(snapshot()); + stream.finish(); + stream.finish(); + stream.onRunStep(snapshot('late')); + stream.onRunStepDelta(streamsArgs('complete', 0, 'late')); + expect(accumulateLikeClient(deltas)).toEqual([ + { index: 0, id: 'a', name: 'get_time', arguments: '{"city":"Madrid"}' }, + ]); + }); + + it.each(['abort', 'reject', 'sync throw'])( + 'does not flush fallback snapshots on %s', + async (failure) => { + const signal = new AbortController(); + const deltas: Delta[] = []; + const toolCalls = new Map(); + const stream = createOpenAIToolCallStream({ + toolCalls, + signal: signal.signal, + emit: (delta) => deltas.push(delta), + }); + stream.onRunStep(snapshot()); + const error = new Error('provider failed'); + const run = completeOpenAIToolCalls(stream, () => { + if (failure === 'sync throw') throw error; + if (failure === 'reject') return Promise.reject(error); + signal.abort(); + return Promise.resolve(); + }); + if (failure === 'abort') await expect(run).rejects.toMatchObject({ name: 'AbortError' }); + else await expect(run).rejects.toBe(error); + stream.onRunStep(snapshot('late')); + stream.onRunStepDelta(streamsArgs('complete', 0, '{"city":"Paris"}')); + expect(() => stream.finish()).toThrow('Agent response aborted'); + expect(toolCalls.size).toBe(0); + expect(deltas).toEqual([]); + const retry = streamingBridge(); + retry.stream.onRunStep(snapshot()); + retry.stream.finish(); + expect(retry.toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}'); + }, + ); + + it('does not report success when argument fragments could not be attributed', () => { + const { stream } = streamingBridge(); + stream.onRunStepDelta(streamsArgs('unknown', 9, '{}')); + expect(() => stream.finish()).toThrow('Unattributable tool call arguments'); + }); + + it('never repairs truncated streamed arguments with a complete snapshot', () => { + const { stream, toolCalls } = streamingBridge(); + stream.onRunStep(snapshot()); + stream.onRunStepDelta(streamsArgs('complete', 0, '{"city":')); + expect(() => stream.finish()).toThrow('Invalid tool call arguments'); + expect(toolCalls.size).toBe(0); + }); + + it('fails incomplete identity rather than reporting a successful missing call', () => { + const { stream } = streamingBridge(); + stream.onRunStepDelta({ + id: 'complete', + delta: { type: 'tool_calls', tool_calls: [{ id: 'a', index: 0, args: '{}' }] }, + }); + expect(() => stream.finish()).toThrow('Incomplete tool call'); + }); + + it('validates all terminal snapshots before flushing any of them', () => { + const { stream, deltas } = streamingBridge(); + stream.onRunStep(snapshot()); + stream.onRunStep(snapshot('bad', 'PRIVATE-INCOMPLETE-INPUT')); + expect(() => stream.finish()).toThrow('Invalid tool call arguments in agent response'); + expect(deltas).toEqual([]); + }); + + it('keeps a raw stream authoritative over an earlier parsed placeholder', () => { + const { stream, deltas } = streamingBridge(); + stream.onRunStep(snapshot('complete', '{}')); + stream.onRunStepDelta(streamsArgs('complete', 0, '{"city":"Madrid"}')); + stream.finish(); + expect(accumulateLikeClient(deltas)[0].arguments).toBe('{"city":"Madrid"}'); + }); + + it.each(['wire', 'native'])('retains complete %s declarations without deltas', (shape) => { + const { stream, deltas, toolCalls } = streamingBridge(); + stream.onRunStep({ + id: 'complete', + stepDetails: { + type: 'tool_calls', + tool_calls: [ + shape === 'wire' + ? { id: 'a', function: { name: 'get_time', arguments: '{"city":"Madrid"}' } } + : { id: 'a', name: 'get_time', args: { city: 'Madrid' } }, + ], + }, + }); + stream.finish(); + expect(toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}'); + expect(accumulateLikeClient(deltas)[0].arguments).toBe('{"city":"Madrid"}'); + }); +}); + +describe('finish reason for a response that called tools', () => { + const context: OpenAIResponseContext = { + requestId: 'chatcmpl-test', + created: 1778317637, + model: 'agent_test', + }; + + it('preserves stop after a server-executed tool followed by final text', () => { + const tracker = createOpenAIStreamTracker(); + const written: string[] = []; + const stream = createOpenAIToolCallStream({ + toolCalls: tracker.toolCalls, + emit: () => undefined, + }); + + stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time')); + + stream.finish(); + tracker.addText(); + sendFinalChunk({ + context, + tracker, + res: { write: (chunk: string) => written.push(chunk) } as never, + }); + + const final = JSON.parse(written[0].replace(/^data: /, '')); + expect(final.choices[0].finish_reason).toBe('stop'); + }); + + it('preserves stop in the final non-streaming answer after server tools', () => { + const toolCalls = new Map(); + const stream = createOpenAIToolCallStream({ toolCalls }); + + stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time')); + + stream.finish(); + const response = buildNonStreamingResponse(context, 'Checking.', '', toolCalls, { + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + }); + + expect(response.choices[0].finish_reason).toBe('stop'); + }); +});