🎽 fix: Give Each Streamed Tool Call a Stable Outward Index (#16209)
Some checks are pending
Backend Unit Tests / Build packages (push) Waiting to run
Backend Unit Tests / Codegraph select (push) Waiting to run
Backend Unit Tests / TypeScript type checks (push) Blocked by required conditions
Backend Unit Tests / Circular dependency checks (push) Waiting to run
Backend Unit Tests / Tests: api (shard 1/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 2/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 3/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: data-provider (push) Blocked by required conditions
Backend Unit Tests / Tests: data-schemas (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 1/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 2/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 3/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 4/4) (push) Blocked by required conditions
Codegraph E2E Votes / vote (full suite) (push) Waiting to run
Docker Dev Branch Images Build / publish (push) Waiting to run
Frontend Unit Tests / Client build recovery regression (push) Waiting to run
Frontend Unit Tests / Codegraph select (push) Waiting to run
Frontend Unit Tests / Build packages (push) Waiting to run
Frontend Unit Tests / TypeScript type checks (client) (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/client (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 1/2) (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 2/2) (push) Blocked by required conditions
Frontend Unit Tests / Vite build verification (push) Blocked by required conditions
Docker Dev Images Build / publish (push) Waiting to run
Sync Helm Chart Tags / Ignore non-main push (push) Waiting to run
Sync Helm Chart Tags / Sync chart tags (push) Waiting to run

* fix: give each streamed tool call a stable outward index

* fix: scope tool-call fragments and preserve completed answers

* fix: finalize agent tool-call snapshots without duplicating deltas

* fix: assemble complete tool identities before response publication

---------

Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
lia-by-librechat[bot] 2026-09-22 11:00:01 -04:00 • committed by GitHub
parent e7a451cbf8
commit ef89fbd5b3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 2218 additions and 238 deletions

View file

@ -217,6 +217,13 @@ jest.mock('@librechat/api', () => ({
buildAgentContextAttachmentsByAgentId: (...args) =>
mockBuildAgentContextAttachmentsByAgentId(...args),
createChunk: jest.fn().mockReturnValue({}),
/** Not stubbed: the outward tool-call index this allocates is the behavior the
* controller is responsible for wiring, so the spec runs the real projection. */
createOpenAIToolCallStream: (...args) =>
jest.requireActual('@librechat/api').createOpenAIToolCallStream(...args),
completeOpenAIToolCalls: jest.requireActual('@librechat/api').completeOpenAIToolCalls,
OpenAIRunStepHandler: jest.requireActual('@librechat/api').OpenAIRunStepHandler,
OpenAIRunStepDeltaHandler: jest.requireActual('@librechat/api').OpenAIRunStepDeltaHandler,
buildRunToolSet: jest.fn().mockReturnValue(new Set()),
buildInitialToolSessions: jest.fn().mockReturnValue(mockInitialSessions),
AgentRunEnvelopeError: MockAgentRunEnvelopeError,
@ -518,6 +525,227 @@ describe('OpenAIChatCompletionController', () => {
};
});
it.each([true, false])(
'projects interleaved calls and preserves the final stop (stream=%s)',
async (streaming) => {
const api = require('@librechat/api');
const actual = jest.requireActual('@librechat/api');
const names = [
'createChunk',
'writeSSE',
'sendFinalChunk',
'buildNonStreamingResponse',
'createOpenAIStreamTracker',
'createOpenAIContentAggregator',
];
const original = names.map((name) => [name, api[name].getMockImplementation()]);
for (const name of names) api[name].mockImplementation(actual[name]);
req.body.stream = streaming;
api.validateRequest.mockReturnValueOnce({ request: req.body });
mockProcessStream.mockImplementationOnce(async () => {
const { customHandlers: h } = api.createRun.mock.calls.at(-1)[0];
const meta = { langgraph_node: 'agent=test', langgraph_step: 1 };
for (const [step, id, index] of [
['s1', 'a', 0],
['s2', 'b', 1],
]) {
await h.on_run_step.handle(
'on_run_step',
{
id: step,
stepDetails: {
type: 'tool_calls',
tool_calls: [{ id, name: 'get_time' }],
},
},
meta,
);
await h.on_run_step_delta.handle(
'on_run_step_delta',
{
id: step,
delta: {
type: 'tool_calls',
tool_calls: [{ id, name: 'get_time', index }],
},
},
meta,
);
}
for (const [index, city] of [
[0, 'Madrid'],
[1, 'Paris'],
]) {
await h.on_run_step_delta.handle(
'on_run_step_delta',
{
id: 's2',
delta: {
type: 'tool_calls',
tool_calls: [{ index, args: JSON.stringify({ city }) }],
},
},
meta,
);
}
await h.on_message_delta.handle(
'on_message_delta',
{
id: 'answer',
delta: {
content: [{ type: 'text', text: 'Both tools completed.' }],
},
},
{ ...meta, langgraph_step: 3 },
);
});
try {
await OpenAIChatCompletionController(req, res);
if (streaming) {
const frames = res.write.mock.calls
.map(([frame]) => frame)
.filter((frame) => frame !== 'data: [DONE]\n\n')
.map((frame) => JSON.parse(frame.slice(6)));
expect(frames.at(-1).choices[0].finish_reason).toBe('stop');
expect(
frames
.flatMap((frame) => frame.choices[0].delta.tool_calls ?? [])
.filter((call) => call.function?.arguments),
).toEqual([
{ index: 0, function: { arguments: '{"city":"Madrid"}' } },
{ index: 1, function: { arguments: '{"city":"Paris"}' } },
]);
} else {
expect(res.json).toHaveBeenCalledWith(
expect.objectContaining({
choices: [
expect.objectContaining({
finish_reason: 'stop',
message: expect.objectContaining({
content: 'Both tools completed.',
tool_calls: [
{
id: 'a',
type: 'function',
function: { name: 'get_time', arguments: '{"city":"Madrid"}' },
},
{
id: 'b',
type: 'function',
function: { name: 'get_time', arguments: '{"city":"Paris"}' },
},
],
}),
}),
],
}),
);
}
} finally {
for (const [name, implementation] of original) api[name].mockImplementation(implementation);
}
},
);
it.each(
[true, false].flatMap((stream) =>
['native-string', 'wire-object', 'split', 'idless'].map((shape) => [stream, shape]),
),
)(
'publishes complete identity and arguments before terminal output (stream=%s, shape=%s)',
async (streaming, shape) => {
const api = require('@librechat/api');
const actual = jest.requireActual('@librechat/api');
const names = [
'createChunk',
'writeSSE',
'sendFinalChunk',
'buildNonStreamingResponse',
'createOpenAIStreamTracker',
'createOpenAIContentAggregator',
];
const originals = names.map((name) => [name, api[name].getMockImplementation()]);
for (const name of names) api[name].mockImplementation(actual[name]);
req.body.stream = streaming;
api.validateRequest.mockReturnValueOnce({ request: req.body });
mockProcessStream.mockImplementationOnce(async () => {
const { customHandlers: h } = api.createRun.mock.calls.at(-1)[0];
await h.on_run_step.handle('on_run_step', {
id: 'complete',
stepDetails: {
type: 'tool_calls',
tool_calls: [
(() => {
if (shape === 'native-string')
return { id: 'a', name: 'get_time', args: '{"city":"Madrid"}' };
if (shape === 'wire-object')
return { id: 'a', function: { name: 'get_time', arguments: { city: 'Madrid' } } };
if (shape === 'idless') return { name: 'get_time', args: { city: 'Madrid' } };
return { id: 'a', name: 'get_', args: {} };
})(),
],
},
});
if (shape === 'split') {
for (const fragment of [
{ id: 'a', name: 'get_', index: 0, args: '{"city":"Madrid"}' },
{ index: 0, name: 'time' },
]) {
await h.on_run_step_delta.handle('on_run_step_delta', {
id: 'complete',
delta: { type: 'tool_calls', tool_calls: [fragment] },
});
}
}
});
try {
await OpenAIChatCompletionController(req, res);
if (streaming) {
const frames = res.write.mock.calls
.map(([frame]) => frame)
.filter((frame) => frame !== 'data: [DONE]\n\n')
.map((frame) => JSON.parse(frame.slice(6)));
expect(
frames
.flatMap((frame) => frame.choices[0].delta.tool_calls ?? [])
.map((call) => call.function?.arguments ?? '')
.join(''),
).toBe('{"city":"Madrid"}');
expect(
frames
.flatMap((frame) => frame.choices[0].delta.tool_calls ?? [])
.find((call) => call.id).function.name,
).toBe('get_time');
expect(frames.at(-1).choices[0].finish_reason).toBe('tool_calls');
} else {
expect(
res.json.mock.calls[0][0].choices[0].message.tool_calls[0].function.arguments,
).toBe('{"city":"Madrid"}');
}
} finally {
for (const [name, implementation] of originals)
api[name].mockImplementation(implementation);
}
},
);
it('records completed model usage even when terminal snapshot validation fails', async () => {
mockProcessStream.mockImplementationOnce(async () => {
const h = require('@librechat/api').createRun.mock.calls.at(-1)[0].customHandlers;
await h.on_run_step.handle('on_run_step', {
id: 'bad',
stepDetails: {
type: 'tool_calls',
tool_calls: [{ id: 'a', function: { name: 'get_time', arguments: 'NOT-JSON' } }],
},
});
});
await OpenAIChatCompletionController(req, res);
expect(mockRecordCollectedUsage).toHaveBeenCalledTimes(1);
expect(require('@librechat/api').buildNonStreamingResponse).not.toHaveBeenCalled();
expect(JSON.stringify(res.json.mock.calls)).not.toContain('NOT-JSON');
});
it('enrolls, starts, and settles the remote execution lifecycle', async () => {
await OpenAIChatCompletionController(req, res);

View file

@ -59,6 +59,10 @@ const {
createToolExecuteHandler,
createOwnedToolEndHandler,
buildNonStreamingResponse,
OpenAIRunStepHandler,
OpenAIRunStepDeltaHandler,
createOpenAIToolCallStream,
completeOpenAIToolCalls,
createOpenAIStreamTracker,
resolveAgentScopedSkillIds,
createOpenAIContentAggregator,
@ -974,6 +978,19 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
}
};
/**
* Shared by both run-step events, because the outward index a client keys
* tool-call fragments by is allocated per call and belongs to neither
* event alone.
*/
const toolCallStream = createOpenAIToolCallStream({
signal: execution.signal,
toolCalls: isStreaming ? tracker.toolCalls : aggregator.toolCalls,
...(isStreaming && {
emit: (delta) => writeSSE(res, createChunk(context, delta)),
}),
});
// Event handlers for OpenAI-compatible streaming
const handlers = {
// Text content streaming
@ -1001,77 +1018,11 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
}
}),
// Tool call initiation - streams id and name (from on_run_step)
on_run_step: createHandler((data) => {
const stepDetails = data?.stepDetails;
if (stepDetails?.type === 'tool_calls' && stepDetails.tool_calls) {
for (const tc of stepDetails.tool_calls) {
const toolIndex = data.index ?? 0;
const toolId = tc.id ?? '';
const toolName = tc.name ?? '';
const toolCall = {
id: toolId,
type: 'function',
function: { name: toolName, arguments: '' },
};
// Track tool call in tracker or aggregator
if (isStreaming) {
if (!tracker.toolCalls.has(toolIndex)) {
tracker.toolCalls.set(toolIndex, toolCall);
}
// Stream initial tool call chunk (like OpenAI does)
writeSSE(
res,
createChunk(context, {
tool_calls: [{ index: toolIndex, ...toolCall }],
}),
);
} else {
if (!aggregator.toolCalls.has(toolIndex)) {
aggregator.toolCalls.set(toolIndex, toolCall);
}
}
}
}
}),
// Tool call initiation - declares id and name (from on_run_step)
on_run_step: new OpenAIRunStepHandler(toolCallStream),
// Tool call argument streaming (from on_run_step_delta)
on_run_step_delta: createHandler((data) => {
const delta = data?.delta;
if (delta?.type === 'tool_calls' && delta.tool_calls) {
for (const tc of delta.tool_calls) {
const args = tc.args ?? '';
if (!args) {
continue;
}
const toolIndex = tc.index ?? 0;
// Update tool call arguments
const targetMap = isStreaming ? tracker.toolCalls : aggregator.toolCalls;
const tracked = targetMap.get(toolIndex);
if (tracked) {
tracked.function.arguments += args;
}
// Stream argument delta (only for streaming)
if (isStreaming) {
writeSSE(
res,
createChunk(context, {
tool_calls: [
{
index: toolIndex,
function: { arguments: args },
},
],
}),
);
}
}
}
}),
on_run_step_delta: new OpenAIRunStepDeltaHandler(toolCallStream),
// Usage tracking
on_chat_model_end: {
@ -1084,7 +1035,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
}
},
},
on_run_step_completed: createHandler(),
on_run_step_completed: new OpenAIRunStepHandler(toolCallStream),
// Use proper ToolEndHandler for processing artifacts (images, file citations, code output)
on_tool_end: createOwnedToolEndHandler(toolEndCallback, logger),
on_chain_stream: createHandler(),
@ -1188,47 +1139,49 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
version: 'v2',
};
await run.processStream({ messages: formattedMessages }, config, {
callbacks: {
[Callback.TOOL_ERROR]: (graph, error, toolId) => {
logger.error(`[OpenAI API] Tool Error "${toolId}"`, getSafeErrorMetadata(error));
await completeOpenAIToolCalls(toolCallStream, async () => {
await run.processStream({ messages: formattedMessages }, config, {
callbacks: {
[Callback.TOOL_ERROR]: (graph, error, toolId) => {
logger.error(`[OpenAI API] Tool Error "${toolId}"`, getSafeErrorMetadata(error));
},
},
},
});
});
// Record token usage against balance
const balanceConfig = getBalanceConfig(appConfig);
const transactionsConfig = getTransactionsConfig(appConfig);
execution.track(
recordCollectedUsage(
{
spendTokens: db.spendTokens,
spendStructuredTokens: db.spendStructuredTokens,
pricing: {
getMultiplier: db.getMultiplier,
getCacheMultiplier: db.getCacheMultiplier,
// Record token usage against balance
const balanceConfig = getBalanceConfig(appConfig);
const transactionsConfig = getTransactionsConfig(appConfig);
execution.track(
recordCollectedUsage(
{
spendTokens: db.spendTokens,
spendStructuredTokens: db.spendStructuredTokens,
pricing: {
getMultiplier: db.getMultiplier,
getCacheMultiplier: db.getCacheMultiplier,
},
bulkWriteOps: {
insertMany: db.bulkInsertTransactions,
updateBalance: db.updateBalance,
},
},
bulkWriteOps: {
insertMany: db.bulkInsertTransactions,
updateBalance: db.updateBalance,
{
user: userId,
conversationId,
collectedUsage,
context: 'message',
messageId: responseId,
balance: balanceConfig,
transactions: transactionsConfig,
model: primaryConfig.model || agent.model_parameters?.model,
endpointTokenConfig: primaryConfig.endpointTokenConfig,
resolveEndpointTokenConfig,
},
},
{
user: userId,
conversationId,
collectedUsage,
context: 'message',
messageId: responseId,
balance: balanceConfig,
transactions: transactionsConfig,
model: primaryConfig.model || agent.model_parameters?.model,
endpointTokenConfig: primaryConfig.endpointTokenConfig,
resolveEndpointTokenConfig,
},
).catch((err) => {
logger.error('[OpenAI API] Error recording usage:', getSafeErrorMetadata(err));
}),
);
).catch((err) => {
logger.error('[OpenAI API] Error recording usage:', getSafeErrorMetadata(err));
}),
);
});
const usage = buildCompletionUsage(collectedUsage);

View file

@ -0,0 +1,65 @@
# Agents API response projection
This module projects an **agent run**, which can include several model invocations and server-executed tools, onto an OpenAI-compatible response. It is not a transparent proxy for a single model completion. Changing tool execution ownership or terminal-turn selection is a separate protocol decision.
## Owners and boundaries
```text
provider adapter -> agents graph -> run-step declarations / argument deltas
|
tool-call projection
/ \
SSE chunks JSON accumulator
\ /
successful response completion
```
The Express controller and packaged `createAgentChatCompletion` service both use the same tool-call projection in streaming and non-streaming mode. The service's content handlers also share text, reasoning and usage handling between modes. The controller retains its existing run lifecycle, usage recording, artifact writes and authorization boundaries.
## Identity
- A declaring run step owns call identity. Optional provider IDs are aliases, not a prerequisite. Declaration slots keep id-less calls distinct and make snapshot replay idempotent. Provider IDs can repeat in later steps and are not completion-wide identities.
- Outward indexes are allocated once per call across the response. Raw ID substrings are assembled by provider index; shared prefixes such as `call_` are never cross-call aliases. Calls without IDs receive `call_<index>`; collisions receive a suffix. Internal tool execution IDs are not rewritten.
- Argument fragments use provider indexes scoped to a model invocation. A graph delta may name the latest step while referring to an earlier parallel call. When available, the SDK's `getStepBaseKey` supplies invocation scope, including stream segments. A step's scope is retained so late fragments cannot acquire a newer segment's identity.
- Callers without the graph object use metadata scope; callers without that metadata are isolated by step. This fallback cannot reconstruct a missing retry/segment identity.
- Explicit and identified provider indexes take precedence over positional fallback. Positions are used only for multi-call declarations. An unbound singleton may adopt its first provider index; it does not absorb arbitrary later indexes.
## Argument representations
A run-step declaration can contain a **snapshot**: native `args` or OpenAI-shaped `function.arguments`. A run-step delta carries an **append-only fragment**. They must never be concatenated as though they were the same representation.
Both snapshot argument fields accept JSON **strings or objects**, using the shared `Agents.ToolCall` and `Agents.AgentFunctionToolCall` field types. Strings are retained verbatim; objects are serialized once. Nonempty argument fragments take precedence over all snapshots for that call, regardless of arrival order.
### Publication timing
Tool-call IDs and names can also arrive as substrings. There is no per-field completion seal in these events, and the OpenAI-compatible consumer freezes the name on its first chunk. Emitting `get_` and later correcting it to `get_weather` does not work.
Consequently, **tool-call chunks are published only at successful response completion**, after assembling identity and validating arguments. The first chunk contains the complete ID/name; the next carries the complete arguments at the same index. Text and reasoning continue streaming while the run executes. This deliberately sacrifices incremental tool-input display, rather than guessing which names are complete or breaking existing clients. Native JSON responses receive the same completed calls.
Name/ID/argument fragments are append-only and take precedence over the corresponding declaration snapshot. Snapshot replay is idempotent; raw fragments are not deduplicated because identical adjacent fragments can be legitimate and the events carry no sequence token. Split IDs require a provider index for unambiguous correlation. Exactly-once raw dispatch remains an upstream requirement.
## Terminal behavior
`completeOpenAIToolCalls` wraps execution and owns the success/failure boundary. The controller schedules completed-run usage recording before final validation.
| State/event | Behavior |
| ---------------------------------------- | ---------------------------------------------------------------- |
| Open + declaration | Allocate identity once; retain snapshot separately |
| Open + identified fragment | Bind provider index and append/buffer fragment |
| Successful execution | Assemble and validate all calls before publishing tool chunks |
| Failure, abort or invalid terminal input | Discard pending calls; publish no tool-call chunks |
| Finished | Repeated finish is harmless; later tool events are ignored |
| Aborted | Later tool events are ignored; finish cannot reopen the response |
Missing names, missing arguments, malformed JSON or unattributable argument data fail terminal projection with bounded errors that omit provider content. A snapshot never repairs a truncated raw stream. Projection bookkeeping is response-local and released on finish/abort; only successfully validated calls are materialized for the response builder. Publication seals before calling the transport, so a transport error or reentrant finish cannot duplicate calls. Already transmitted frames cannot be retracted.
The existing finish-reason policy is preserved: a response with final text after server-executed tools remains `stop`. Historical tool calls are still exposed by this API. This patch does not make it safe to interpret every advertised call as a new instruction for the caller to execute.
## Verification and limits
- `toolCalls.contract.spec.ts`: shared field types, strings/objects, partial names and IDs, optional IDs, declaration order and transport reentrancy.
- `toolCalls.spec.ts`: identity, representations, ordering, replay, malformed input, failure and abort.
- `toolCalls.graph.spec.ts`: real SDK interleaving, complete-only chunks and stream segments.
- `service.spec.ts` and the Express controller spec: both response modes, actual serializers, fallback flush and failure/usage ordering.
These tests do not run a live provider, real tool side effects, a distributed multi-agent deployment or a persisted resume. There is no new database, cache, authorization policy or migration. Code and built packages must deploy together. No cross-version event identity is invented when the upstream source omits it.

View file

@ -5,6 +5,8 @@
* streaming format (SSE with chat.completion.chunk objects).
*/
import type { Response as ServerResponse } from 'express';
import type { Agents } from 'librechat-data-provider';
import type { Graph } from '@librechat/agents';
import type {
ChatCompletionChunkChoice,
OpenAIResponseContext,
@ -58,6 +60,9 @@ export function writeSSE(res: ServerResponse, data: ChatCompletionChunk | string
* Only tracks what's needed for finish_reason and usage - doesn't store content.
*/
export interface OpenAIStreamTracker {
/** Successful response completion, installed by the response handlers. */
finishToolCalls?: () => void;
abortToolCalls?: () => void;
/** Whether any text content was emitted */
hasText: boolean;
/** Whether any reasoning content was emitted */
@ -105,6 +110,9 @@ export function createOpenAIStreamTracker(): OpenAIStreamTracker {
* Uses arrays for O(n) text accumulation instead of O(n²) string concatenation.
*/
export interface OpenAIContentAggregator {
/** Successful response completion, installed by the response handlers. */
finishToolCalls?: () => void;
abortToolCalls?: () => void;
/** Accumulated text chunks */
textChunks: string[];
/** Accumulated reasoning/thinking chunks */
@ -154,6 +162,7 @@ export function createOpenAIContentAggregator(): OpenAIContentAggregator {
* Handler configuration for OpenAI streaming
*/
export interface OpenAIStreamHandlerConfig {
signal?: AbortSignal;
res: ServerResponse;
context: OpenAIResponseContext;
tracker: OpenAIStreamTracker;
@ -187,22 +196,44 @@ export const StepTypes = {
*/
export interface MessageDeltaData {
id?: string;
content?: Array<{ type: string; text?: string }>;
content?: Array<{ type: string; text?: string; think?: string }>;
delta?: { content?: MessageDeltaData['content'] };
}
/**
* One tool-call fragment of a run step delta, as `@librechat/agents` emits it:
* `id`, `name`, and `args` can all be substrings. `index` identifies their
* provider stream within the current model invocation. Reuse the shared chunk
* contract and accept equivalent function-shaped name/argument fragments.
*/
export type RunStepToolCallChunk = Agents.ToolCallChunk & {
function?: { name?: string; arguments?: string };
};
export interface RunStepDeltaData {
/** The run step these fragments belong to. */
id?: string;
delta?: {
type?: string;
tool_calls?: Array<{
index?: number;
id?: string;
type?: string;
function?: {
name?: string;
arguments?: string;
};
}>;
tool_calls?: RunStepToolCallChunk[];
};
}
/** A tool call as the run step that opened it declares it. */
export type RunStepToolCall = Partial<Pick<Agents.ToolCall, 'id' | 'name' | 'args'>> & {
index?: number;
type?: string;
function?: Partial<Agents.AgentFunctionToolCall['function']>;
};
export interface RunStepData {
/** The run step's own id, shared with every delta dispatched for it. */
id?: string;
/** The step's position in the response content, not a tool-call index. */
index?: number;
stepDetails?: {
type?: string;
tool_calls?: RunStepToolCall[];
};
}
@ -240,20 +271,372 @@ export interface EventHandler {
): void | Promise<void>;
}
/**
* Projects the graph's tool-call events onto the OpenAI wire format.
*
* A client accumulates `delta.tool_calls` by `index`: the first chunk at an
* index declares the call and must carry `id` and `function.name`, and every
* later chunk at that index belongs to that same call. Neither index the graph
* reports can serve as that key. A run step's `index` is its position in the
* response's content, so a tool call that follows text does not start at zero,
* and consecutive steps each carry a single-element `tool_calls` array rather
* than one array holding every call. A fragment's `index` is the provider's
* content-block index, which restarts at zero on each model invocation of the
* run, so two different calls can share it.
*
* Outward indexes identify calls within declaring steps, using declaration slots
* when IDs are absent. Provider indexes correlate raw fragments within an
* invocation. IDs and names are assembled before publication: a client can freeze
* both on its first chunk, and the graph provides no per-field completion seal.
*/
export interface OpenAIToolCallStreamConfig {
/** Completed tool calls, populated at successful finish and keyed by outward index. */
toolCalls: Map<number, ToolCall>;
/** Emits complete identity/argument chunks at finish. Omitted for non-streaming. */
emit?: (delta: ChatCompletionChunkChoice['delta']) => void;
signal?: AbortSignal;
}
export interface OpenAIToolCallStream {
/** Declares the tool calls a run step opened. */
onRunStep: (data: RunStepData, metadata?: Record<string, unknown>, graph?: ToolCallGraph) => void;
/** Accumulates the name and argument fragments streamed for a run step. */
onRunStepDelta: (
data: RunStepDeltaData,
metadata?: Record<string, unknown>,
graph?: ToolCallGraph,
) => void;
/** Publish assembled, validated calls once after success, before DONE/JSON. */
finish: () => void;
/** Discard pending calls on failure and reject further writes. */
abort: () => void;
}
type ToolCallGraph = Pick<Graph, 'getStepBaseKey'>;
interface ProjectedToolCall {
index: number;
snapshotId?: string;
snapshotName?: string;
snapshotArgs?: string;
idFragments: string[];
nameFragments: string[];
argFragments: string[];
}
interface ToolCallStep {
byId: Map<string, ProjectedToolCall>;
byDeclaration: Map<string, ProjectedToolCall>;
byProviderIndex: Map<number, ProjectedToolCall>;
/** Declaration position is a fallback, never stronger than a provider binding. */
byPosition: Map<number, ProjectedToolCall>;
calls: Set<ProjectedToolCall>;
}
export function createOpenAIToolCallStream(
config: OpenAIToolCallStreamConfig,
): OpenAIToolCallStream {
const { toolCalls, emit, signal } = config;
let phase: 'open' | 'finished' | 'aborted' = 'open';
let unattributableArguments = false;
const calls: ProjectedToolCall[] = [];
const steps = new Map<string, ToolCallStep>();
const invocations = new Map<string, Map<number, ProjectedToolCall>>();
const stepScopes = new Map<string, string>();
/** The SDK can send earlier parallel calls under the latest step ID. Its
* invocation key owns segment/checkpoint transitions; cache it at declaration
* so late events never borrow a new segment. Step isolation is the fallback. */
const getBindings = (
stepId: string,
metadata?: Record<string, unknown>,
graph?: ToolCallGraph,
): Map<number, ProjectedToolCall> => {
let scope = stepScopes.get(stepId);
if (scope === undefined) {
if (graph && metadata) {
scope = graph.getStepBaseKey(metadata);
} else if (
typeof metadata?.langgraph_node === 'string' &&
typeof metadata.langgraph_step === 'number'
) {
scope = JSON.stringify([
metadata.run_id ?? '',
metadata.thread_id ?? '',
metadata.langgraph_node,
metadata.langgraph_step,
metadata.langgraph_checkpoint_ns ?? metadata.checkpoint_ns ?? '',
]);
} else {
scope = JSON.stringify(['step', stepId]);
}
stepScopes.set(stepId, scope);
}
let bindings = invocations.get(scope);
if (bindings === undefined) {
bindings = new Map();
invocations.set(scope, bindings);
}
return bindings;
};
const getStep = (stepId: string): ToolCallStep => {
let step = steps.get(stepId);
if (step === undefined) {
step = {
byId: new Map(),
byDeclaration: new Map(),
byProviderIndex: new Map(),
byPosition: new Map(),
calls: new Set(),
};
steps.set(stepId, step);
}
return step;
};
const allocate = (step: ToolCallStep): ProjectedToolCall => {
const call: ProjectedToolCall = {
index: calls.length,
idFragments: [],
nameFragments: [],
argFragments: [],
};
calls.push(call);
step.calls.add(call);
return call;
};
const abort = (): void => {
if (phase !== 'finished') {
phase = 'aborted';
}
calls.length = 0;
steps.clear();
stepScopes.clear();
invocations.clear();
};
const writable = (): boolean => {
if (signal?.aborted) {
abort();
}
return phase === 'open';
};
return {
abort,
finish: () => {
if (phase === 'finished') {
return;
}
if (!writable()) {
const error = new Error('Agent response aborted');
error.name = 'AbortError';
throw error;
}
try {
if (unattributableArguments) {
throw new Error('Unattributable tool call arguments in agent response');
}
const ready: ToolCall[] = [];
const ids = new Set<string>();
/** There is no per-field name/ID seal in the graph event contract. An
* OpenAI-compatible client freezes the name on the first outward chunk.
* Validate and assemble every call before publishing any: text still
* streams, but tool chunks wait for successful response completion. */
for (const call of calls) {
const name = call.nameFragments.join('') || call.snapshotName;
const args = call.argFragments.length ? call.argFragments.join('') : call.snapshotArgs;
if (!name || args === undefined || args === '') {
throw new Error('Incomplete tool call in agent response');
}
try {
JSON.parse(args);
} catch {
throw new Error('Invalid tool call arguments in agent response');
}
let id = call.idFragments.join('') || call.snapshotId || `call_${call.index}`;
while (ids.has(id)) {
id = `${id}_${call.index}`;
}
ids.add(id);
ready.push({ id, type: 'function', function: { name, arguments: args } });
}
for (const [index, call] of ready.entries()) {
toolCalls.set(index, call);
}
/** Seal before invoking transport callbacks, including reentrant callers.
* An emission failure cannot be retried into duplicate tool requests. */
phase = 'finished';
for (const [index, call] of ready.entries()) {
emit?.({
tool_calls: [
{
index,
id: call.id,
type: 'function',
function: { name: call.function.name, arguments: '' },
},
],
});
emit?.({ tool_calls: [{ index, function: { arguments: call.function.arguments } }] });
}
} finally {
abort();
}
},
onRunStep: (data, metadata, graph) => {
if (!writable()) {
return;
}
const details = data?.stepDetails;
if (details?.type !== StepTypes.TOOL_CALLS || !Array.isArray(details.tool_calls)) {
return;
}
const step = getStep(data.id ?? '');
const bindings = getBindings(data.id ?? '', metadata, graph);
for (const [position, toolCall] of details.tool_calls.entries()) {
const key =
toolCall.index === undefined ? `position:${position}` : `index:${toolCall.index}`;
let call =
(toolCall.id ? step.byId.get(toolCall.id) : undefined) ?? step.byDeclaration.get(key);
if (call === undefined && toolCall.index !== undefined) {
call = step.byProviderIndex.get(toolCall.index);
}
/** A declaration can follow its raw chunks. Positions are meaningful
* only inside this declaring step, never across the whole response. */
if (call === undefined) {
call = step.byProviderIndex.get(position);
if (
call === undefined &&
step.byDeclaration.size === 0 &&
details.tool_calls.length === 1 &&
step.calls.size === 1
) {
call = step.calls.values().next().value;
}
}
call ??= allocate(step);
step.byDeclaration.set(key, call);
if (toolCall.id) {
call.snapshotId = toolCall.id;
step.byId.set(toolCall.id, call);
}
call.snapshotName = toolCall.name ?? toolCall.function?.name ?? call.snapshotName;
const args = toolCall.function?.arguments ?? toolCall.args;
if (args !== undefined) {
/** Both public snapshot fields accept raw JSON strings or objects. */
try {
call.snapshotArgs = typeof args === 'string' ? args : JSON.stringify(args);
} catch {
throw new Error('Invalid tool call arguments in agent response');
}
}
if (toolCall.index !== undefined) {
step.byProviderIndex.set(toolCall.index, call);
bindings.set(toolCall.index, call);
} else if (details.tool_calls.length > 1) {
step.byPosition.set(position, call);
}
}
},
onRunStepDelta: (data, metadata, graph) => {
if (!writable()) {
return;
}
const delta = data?.delta;
if (delta?.type !== StepTypes.TOOL_CALLS || !Array.isArray(delta.tool_calls)) {
return;
}
const step = getStep(data.id ?? '');
const bindings = getBindings(data.id ?? '', metadata, graph);
for (const fragment of delta.tool_calls) {
const name = fragment.name ?? fragment.function?.name ?? '';
const args = fragment.args ?? fragment.function?.arguments ?? '';
/** An index identifies an already-bound raw stream even if this event
* carries only a substring of its ID. Never key identity by that suffix. */
let call = fragment.index === undefined ? undefined : bindings.get(fragment.index);
call ??= fragment.id ? step.byId.get(fragment.id) : undefined;
call ??= fragment.index === undefined ? undefined : step.byPosition.get(fragment.index);
if (
call === undefined &&
step.calls.size === 1 &&
(fragment.index === undefined || step.byProviderIndex.size === 0)
) {
const singleton = step.calls.values().next().value;
/** Without an index, a different full ID starts another raw call, not
* another substring of the old call. Split IDs need their index. */
if (
!fragment.id ||
fragment.index !== undefined ||
(singleton?.snapshotId === undefined && singleton?.idFragments.length === 0)
) {
call = singleton;
}
}
if (call === undefined && (fragment.id || name)) {
call = allocate(step);
}
if (call === undefined) {
unattributableArguments ||= !!args;
continue;
}
if (fragment.index !== undefined) {
step.byProviderIndex.set(fragment.index, call);
bindings.set(fragment.index, call);
}
if (fragment.id) {
call.idFragments.push(fragment.id);
/** Partial IDs are not aliases across indexed calls: two parallel
* calls can both begin with `call_`. Their index is authoritative. */
if (fragment.index === undefined) {
step.byId.set(call.idFragments.join(''), call);
}
}
if (name) {
call.nameFragments.push(name);
}
if (args) {
call.argFragments.push(args);
}
}
},
};
}
/** Success publishes validated calls; failure/abort never does. Both hosts
* use this boundary so late provider events cannot mutate a settled response. */
export async function completeOpenAIToolCalls(
lifecycle: Pick<OpenAIToolCallStream, 'finish' | 'abort'>,
execute: () => Promise<void>,
): Promise<void> {
try {
await execute();
lifecycle.finish();
} finally {
lifecycle.abort();
}
}
/**
* Handler for message delta events - streams text content
*/
export class OpenAIMessageDeltaHandler implements EventHandler {
constructor(private config: OpenAIStreamHandlerConfig) {}
constructor(private config: OpenAIContentHandlerConfig) {}
handle(_event: string, data: MessageDeltaData): void {
const content = data?.content;
const content = data?.delta?.content ?? data?.content;
if (!content || !Array.isArray(content)) {
return;
}
for (const part of content) {
if (part.type === 'text' && part.text) {
if ('aggregator' in this.config) {
this.config.aggregator.addText(part.text);
continue;
}
this.config.tracker.addText();
const chunk = createChunk(this.config.context, { content: part.text });
writeSSE(this.config.res, chunk);
@ -263,86 +646,35 @@ export class OpenAIMessageDeltaHandler implements EventHandler {
}
/**
* Handler for run step delta events - streams tool calls
* Handler for run step delta events - accumulates streamed tool call fragments
*/
export class OpenAIRunStepDeltaHandler implements EventHandler {
constructor(private config: OpenAIStreamHandlerConfig) {}
constructor(private toolCallStream: OpenAIToolCallStream) {}
handle(_event: string, data: RunStepDeltaData): void {
const delta = data?.delta;
if (!delta || delta.type !== StepTypes.TOOL_CALLS) {
return;
}
const toolCalls = delta.tool_calls;
if (!toolCalls || !Array.isArray(toolCalls)) {
return;
}
for (const tc of toolCalls) {
if (tc.index === undefined) {
continue;
}
// Initialize tool call in tracker if needed
let trackedTc = this.config.tracker.toolCalls.get(tc.index);
if (!trackedTc && tc.id) {
trackedTc = {
id: tc.id,
type: 'function',
function: {
name: '',
arguments: '',
},
};
this.config.tracker.toolCalls.set(tc.index, trackedTc);
}
// Build the streaming delta
const streamDelta: ChatCompletionChunkChoice['delta'] = {
tool_calls: [
{
index: tc.index,
...(tc.id && { id: tc.id }),
...(tc.type && { type: tc.type as 'function' }),
...(tc.function && {
function: {
...(tc.function.name && { name: tc.function.name }),
...(tc.function.arguments && { arguments: tc.function.arguments }),
},
}),
},
],
};
// Update tracked tool call
if (trackedTc) {
if (tc.function?.name) {
trackedTc.function.name += tc.function.name;
}
if (tc.function?.arguments) {
trackedTc.function.arguments += tc.function.arguments;
}
}
const chunk = createChunk(this.config.context, streamDelta);
writeSSE(this.config.res, chunk);
}
handle(
_event: string,
data: RunStepDeltaData,
metadata?: Record<string, unknown>,
graph?: ToolCallGraph,
): void {
this.toolCallStream.onRunStepDelta(data, metadata, graph);
}
}
/**
* Handler for run step events - sends initial tool call info
* Handler for run step events - declares a tool call's id and name at its
* outward index, before any argument fragment references that index
*/
export class OpenAIRunStepHandler implements EventHandler {
constructor(private config: OpenAIStreamHandlerConfig) {}
constructor(private toolCallStream: OpenAIToolCallStream) {}
handle(_event: string, data: { stepDetails?: { type?: string } }): void {
// Run step events are primarily for LibreChat UI, we use deltas for streaming
// This handler is a no-op for OpenAI format
if (data?.stepDetails?.type === StepTypes.TOOL_CALLS) {
// Tool calls will be streamed via delta events
}
handle(
_event: string,
data: RunStepData,
metadata?: Record<string, unknown>,
graph?: ToolCallGraph,
): void {
this.toolCallStream.onRunStep(data, metadata, graph);
}
}
@ -350,7 +682,7 @@ export class OpenAIRunStepHandler implements EventHandler {
* Handler for model end events - captures usage
*/
export class OpenAIModelEndHandler implements EventHandler {
constructor(private config: OpenAIStreamHandlerConfig) {}
constructor(private config: OpenAIContentHandlerConfig) {}
handle(_event: string, data: ModelEndData): void {
const usage = data?.output?.usage_metadata;
@ -358,9 +690,10 @@ export class OpenAIModelEndHandler implements EventHandler {
return;
}
this.config.tracker.usage.promptTokens += usage.input_tokens ?? 0;
this.config.tracker.usage.completionTokens += usage.output_tokens ?? 0;
this.config.tracker.usage.reasoningTokens +=
const target = 'aggregator' in this.config ? this.config.aggregator : this.config.tracker;
target.usage.promptTokens += usage.input_tokens ?? 0;
target.usage.completionTokens += usage.output_tokens ?? 0;
target.usage.reasoningTokens +=
usage.output_token_details?.reasoning ?? usage.output_token_details?.reasoning_tokens ?? 0;
}
}
@ -389,21 +722,26 @@ export class OpenAIToolEndHandler implements EventHandler {
* Streams reasoning/thinking content using the `delta.reasoning` field (OpenRouter convention).
*/
export class OpenAIReasoningDeltaHandler implements EventHandler {
constructor(private config: OpenAIStreamHandlerConfig) {}
constructor(private config: OpenAIContentHandlerConfig) {}
handle(_event: string, data: MessageDeltaData): void {
const content = data?.content;
const content = data?.delta?.content ?? data?.content;
if (!content || !Array.isArray(content)) {
return;
}
for (const part of content) {
if (part.type === 'text' && part.text) {
if ((part.type === 'text' || part.type === 'think') && (part.think || part.text)) {
const text = part.think || part.text!;
if ('aggregator' in this.config) {
this.config.aggregator.addReasoning(text);
continue;
}
// Mark that reasoning was emitted
this.config.tracker.addReasoning();
// Stream as delta.reasoning (OpenRouter convention)
const chunk = createChunk(this.config.context, { reasoning: part.text });
const chunk = createChunk(this.config.context, { reasoning: text });
writeSSE(this.config.res, chunk);
}
}
@ -413,15 +751,34 @@ export class OpenAIReasoningDeltaHandler implements EventHandler {
/**
* Create all handlers for OpenAI streaming format
*/
export interface OpenAIAggregationHandlerConfig {
aggregator: OpenAIContentAggregator;
signal?: AbortSignal;
}
type OpenAIContentHandlerConfig = OpenAIStreamHandlerConfig | OpenAIAggregationHandlerConfig;
export function createOpenAIHandlers(
config: OpenAIStreamHandlerConfig,
config: OpenAIContentHandlerConfig,
toolExecuteOptions?: ToolExecuteOptions,
): Record<string, EventHandler> {
/** One projection across both events, so a call keeps a single outward index. */
const toolCallStream = createOpenAIToolCallStream({
signal: config.signal,
toolCalls: 'aggregator' in config ? config.aggregator.toolCalls : config.tracker.toolCalls,
emit:
'aggregator' in config
? undefined
: (delta) => writeSSE(config.res, createChunk(config.context, delta)),
});
const target = 'aggregator' in config ? config.aggregator : config.tracker;
target.finishToolCalls = toolCallStream.finish;
target.abortToolCalls = toolCallStream.abort;
const handlers: Record<string, EventHandler> = {
[GraphEvents.ON_MESSAGE_DELTA]: new OpenAIMessageDeltaHandler(config),
[GraphEvents.ON_RUN_STEP_DELTA]: new OpenAIRunStepDeltaHandler(config),
[GraphEvents.ON_RUN_STEP]: new OpenAIRunStepHandler(config),
[GraphEvents.ON_RUN_STEP_COMPLETED]: new OpenAIRunStepHandler(config),
[GraphEvents.ON_RUN_STEP_DELTA]: new OpenAIRunStepDeltaHandler(toolCallStream),
[GraphEvents.ON_RUN_STEP]: new OpenAIRunStepHandler(toolCallStream),
[GraphEvents.ON_RUN_STEP_COMPLETED]: new OpenAIRunStepHandler(toolCallStream),
[GraphEvents.CHAT_MODEL_END]: new OpenAIModelEndHandler(config),
[GraphEvents.CHAT_MODEL_STREAM]: new OpenAIChatModelStreamHandler(),
[GraphEvents.TOOL_END]: new OpenAIToolEndHandler(),
@ -444,6 +801,7 @@ export function sendFinalChunk(
usageOverride?: CompletionUsage,
): void {
const { res, context, tracker } = config;
tracker.finishToolCalls?.();
// Determine finish reason based on content
let reason = finishReason;

View file

@ -2,6 +2,7 @@ import { GraphEvents } from '@librechat/agents';
import { ErrorTypes, Permissions, PermissionTypes } from 'librechat-data-provider';
import type { FiltersConfig } from 'librechat-data-provider';
import type { ChatCompletionDependencies } from './service';
import type { ChatCompletionChunk } from './types';
import { createAgentChatCompletion } from './service';
jest.mock('@librechat/data-schemas', () => ({
@ -104,6 +105,143 @@ describe('createAgentChatCompletion - MCP permission user propagation', () => {
};
});
it.each(
[true, false].flatMap((stream) =>
['wire-string', 'wire-object', 'native-string', 'idless'].map(
(shape) => [stream, shape] as const,
),
),
)(
'retains complete and mixed snapshots through the service (stream=%s, shape=%s)',
async (stream, shape) => {
const req = createMockReq(
{ id: 'user' },
{ model: 'agent_test', messages: [{ role: 'user', content: 'hi' }], stream },
);
const res = createMockRes();
processStream.mockImplementationOnce(async () => {
const { customHandlers: h } = createRun.mock.calls[0][0] as Parameters<
NonNullable<ChatCompletionDependencies['createRun']>
>[0];
const meta = { langgraph_node: 'agent=test', langgraph_step: 1 };
await h.on_run_step.handle(
'on_run_step',
{
id: 'step',
stepDetails: {
type: 'tool_calls',
tool_calls: [
(() => {
if (shape === 'idless')
return { name: 'get_time', args: { city: 'Madrid' }, index: 0 };
if (shape === 'native-string')
return { id: 'a', name: 'get_time', args: '{"city":"Madrid"}', index: 0 };
return {
id: 'a',
function: {
name: 'get_time',
arguments: shape === 'wire-object' ? { city: 'Madrid' } : '{"city":"Madrid"}',
},
index: 0,
};
})(),
{ id: 'b', name: 'get_time', args: {}, index: 1 },
],
},
},
meta,
);
await h.on_run_step_delta.handle(
'on_run_step_delta',
{
id: 'step',
delta: { type: 'tool_calls', tool_calls: [{ index: 1, args: '{"city":"Paris"}' }] },
},
meta,
);
await h.on_message_delta.handle('on_message_delta', {
delta: { content: [{ type: 'text', text: 'Finished.' }] },
});
});
await createAgentChatCompletion(req, res, deps);
if (stream) {
const frames = (res.write as jest.Mock).mock.calls
.map(([frame]: [string]) => frame)
.filter((frame) => frame !== 'data: [DONE]\n\n');
const chunks: ChatCompletionChunk[] = frames.map((frame) => JSON.parse(frame.slice(6)));
const args = new Map<number, string>();
for (const chunk of chunks)
for (const call of chunk.choices[0].delta.tool_calls ?? [])
args.set(call.index, (args.get(call.index) ?? '') + (call.function?.arguments ?? ''));
expect(args.get(0)).toBe('{"city":"Madrid"}');
expect(args.get(1)).toBe('{"city":"Paris"}');
expect(chunks[chunks.length - 1].choices[0].finish_reason).toBe('stop');
} else {
expect(getResponseMock(res, 'json')).toHaveBeenCalledWith(
expect.objectContaining({
choices: [
expect.objectContaining({
finish_reason: 'stop',
message: expect.objectContaining({
content: 'Finished.',
tool_calls: [
{
id: shape === 'idless' ? 'call_0' : 'a',
type: 'function',
function: { name: 'get_time', arguments: '{"city":"Madrid"}' },
},
{
id: 'b',
type: 'function',
function: { name: 'get_time', arguments: '{"city":"Paris"}' },
},
],
}),
}),
],
}),
);
}
},
);
it.each([true, false])(
'does not publish fallback arguments after provider failure (stream=%s)',
async (stream) => {
const req = createMockReq(
{ id: 'user' },
{ model: 'agent_test', messages: [{ role: 'user', content: 'hi' }], stream },
);
const res = createMockRes();
processStream.mockImplementationOnce(async () => {
const { customHandlers: h } = createRun.mock.calls[0][0] as Parameters<
NonNullable<ChatCompletionDependencies['createRun']>
>[0];
await h.on_run_step.handle('on_run_step', {
id: 'step',
stepDetails: {
type: 'tool_calls',
tool_calls: [
{ id: 'a', function: { name: 'get_time', arguments: '{"city":"DO_NOT_FLUSH"}' } },
],
},
});
throw new Error('provider failed');
});
await createAgentChatCompletion(req, res, deps);
expect(JSON.stringify((res.write as jest.Mock).mock.calls)).not.toContain('DO_NOT_FLUSH');
expect(JSON.stringify(getResponseMock(res, 'json').mock.calls)).not.toContain('DO_NOT_FLUSH');
const { customHandlers: h } = createRun.mock.calls[0][0] as Parameters<
NonNullable<ChatCompletionDependencies['createRun']>
>[0];
await h.on_run_step_delta.handle('on_run_step_delta', {
id: 'step',
delta: { type: 'tool_calls', tool_calls: [{ id: 'a', args: '{"late":true}' }] },
});
expect(JSON.stringify((res.write as jest.Mock).mock.calls)).not.toContain('late');
},
);
it('forwards the role-bearing safe user to createRun and configurable.user', async () => {
const req = createMockReq({
id: 'user-123',

View file

@ -63,6 +63,7 @@ import {
createOpenAIContentAggregator,
createOpenAIStreamTracker,
createOpenAIHandlers,
completeOpenAIToolCalls,
sendFinalChunk,
createChunk,
writeSSE,
@ -833,19 +834,21 @@ export async function createAgentChatCompletion(
: null;
// Create event handlers
const eventHandlers =
isStreaming && handlerConfig
? createOpenAIHandlers(
handlerConfig,
deps.toolExecuteOptions == null
? undefined
: {
...deps.toolExecuteOptions,
runSignal: abortController.signal,
foregroundRunId: requestId,
},
)
: {};
const eventHandlers = createOpenAIHandlers(
handlerConfig
? { ...handlerConfig, signal: abortController.signal }
: {
aggregator: aggregator!,
signal: abortController.signal,
},
deps.toolExecuteOptions == null
? undefined
: {
...deps.toolExecuteOptions,
runSignal: abortController.signal,
foregroundRunId: requestId,
},
);
// Convert messages to internal format
const messages = convertMessages(request.messages);
@ -877,32 +880,40 @@ export async function createAgentChatCompletion(
});
if (run) {
await run.processStream(
{ messages },
const target = tracker ?? aggregator!;
await completeOpenAIToolCalls(
{
runName: 'AgentRun',
configurable: {
thread_id: conversationId,
user_id: userId,
user: safeUser,
requestBody: mcpRequestBody,
/** Same per-agent channel the in-repo controllers thread via
* `loadTools`: without it, the executor's PTC path cannot
* strip host-injected `intent` params from the schemas the
* sandbox bridge advertises on this route. */
...(initializedAgent.intentToolNames?.length
? { intentToolNames: initializedAgent.intentToolNames }
: {}),
},
recursionLimit: resolveRecursionLimit(
agentsConfig as Partial<TAgentsEndpoint> | undefined,
initializedAgent,
),
signal: abortController.signal,
streamMode: 'values',
version: 'v2',
finish: () => target.finishToolCalls?.(),
abort: () => target.abortToolCalls?.(),
},
{},
() =>
run.processStream(
{ messages },
{
runName: 'AgentRun',
configurable: {
thread_id: conversationId,
user_id: userId,
user: safeUser,
requestBody: mcpRequestBody,
/** Same per-agent channel the in-repo controllers thread via
* `loadTools`: without it, the executor's PTC path cannot
* strip host-injected `intent` params from the schemas the
* sandbox bridge advertises on this route. */
...(initializedAgent.intentToolNames?.length
? { intentToolNames: initializedAgent.intentToolNames }
: {}),
},
recursionLimit: resolveRecursionLimit(
agentsConfig as Partial<TAgentsEndpoint> | undefined,
initializedAgent,
),
signal: abortController.signal,
streamMode: 'values',
version: 'v2',
},
{},
),
);
}
}
@ -912,6 +923,7 @@ export async function createAgentChatCompletion(
sendFinalChunk(handlerConfig);
res.end();
} else if (aggregator) {
aggregator.finishToolCalls?.();
// Build and send non-streaming response
const usage: CompletionUsage = {
prompt_tokens: aggregator.usage.promptTokens,

View file

@ -0,0 +1,308 @@
import type { Agents } from 'librechat-data-provider';
import type { ChatCompletionChunkChoice, ToolCall } from './types';
import { createOpenAIToolCallStream } from './handlers';
/** Public contract fixtures, not a narrower reconstruction of the SDK fields. */
describe('tool-call field normalization', () => {
it.each(['native-string', 'native-object', 'wire-string', 'wire-object'])(
'normalizes %s exactly once',
(shape) => {
const toolCalls = new Map<number, ToolCall>();
const deltas: ChatCompletionChunkChoice['delta'][] = [];
const stream = createOpenAIToolCallStream({ toolCalls, emit: (delta) => deltas.push(delta) });
const args = shape.endsWith('string') ? '{"city":"Madrid"}' : { city: 'Madrid' };
const call: Agents.AgentToolCall = shape.startsWith('native')
? { id: 'a', name: 'lookup', args }
: { id: 'a', type: 'function', function: { name: 'lookup', arguments: args } };
stream.onRunStep({ id: 'step', stepDetails: { type: 'tool_calls', tool_calls: [call] } });
stream.finish();
expect(toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}');
expect(
deltas
.flatMap((d) => d.tool_calls ?? [])
.map((c) => c.function?.arguments ?? '')
.join(''),
).toBe('{"city":"Madrid"}');
},
);
it.each([false, true])(
'assembles name substrings before the consumer freezes them (snapshot=%s)',
(snapshot) => {
const toolCalls = new Map<number, ToolCall>();
const deltas: ChatCompletionChunkChoice['delta'][] = [];
const stream = createOpenAIToolCallStream({ toolCalls, emit: (delta) => deltas.push(delta) });
if (snapshot)
stream.onRunStep({
id: 'step',
stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'a', name: 'get_', args: {} }] },
});
for (const chunk of [
{ id: 'a', name: 'get_', index: 0, args: '{}' },
{ name: 'weather', index: 0 },
]) {
stream.onRunStepDelta({ id: 'step', delta: { type: 'tool_calls', tool_calls: [chunk] } });
}
stream.finish();
expect(toolCalls.get(0)?.function.name).toBe('get_weather');
expect(
deltas
.flatMap((d) => d.tool_calls ?? [])
.filter((c) => c.id)
.map((c) => c.function?.name),
).toEqual(['get_weather']);
},
);
it('synthesizes stable, distinct IDs for id-less complete calls and replayed declarations', () => {
const toolCalls = new Map<number, ToolCall>();
const stream = createOpenAIToolCallStream({ toolCalls });
const calls: Agents.ToolCall[] = [
{ name: 'lookup', args: {} },
{ name: 'lookup', args: { city: 'Paris' } },
];
const event = { id: 'step', stepDetails: { type: 'tool_calls', tool_calls: calls } };
stream.onRunStep(event);
stream.onRunStep(event);
stream.onRunStep({ ...event, id: 'later' });
stream.finish();
expect(toolCalls.size).toBe(4);
expect(new Set([...toolCalls.values()].map((c) => c.id)).size).toBe(4);
expect([...toolCalls.values()].every((c) => !!c.id)).toBe(true);
});
it.each(['native', 'wire'])('assembles split names and IDs with %s delta fields', (shape) => {
const toolCalls = new Map<number, ToolCall>();
const stream = createOpenAIToolCallStream({ toolCalls });
for (const [id, name, args] of [
['call_', 'get_', '{"city":'],
['123', 'weather', '"Paris"}'],
]) {
stream.onRunStepDelta({
id: 'step',
delta: {
type: 'tool_calls',
tool_calls: [
shape === 'native'
? { index: 4, id, name, args }
: { index: 4, id, function: { name, arguments: args } },
],
},
});
}
stream.finish();
expect([...toolCalls.values()]).toEqual([
{
id: 'call_123',
type: 'function',
function: { name: 'get_weather', arguments: '{"city":"Paris"}' },
},
]);
});
it.each([true, false])(
'keeps id-less calls distinct with mixed raw data (explicit index=%s)',
(explicit) => {
const toolCalls = new Map<number, ToolCall>();
const stream = createOpenAIToolCallStream({ toolCalls });
stream.onRunStep({
id: 'step',
stepDetails: {
type: 'tool_calls',
tool_calls: [
{ name: 'get_', args: {}, ...(explicit && { index: 5 }) },
{ name: 'lookup', args: { city: 'Paris' }, ...(explicit && { index: 8 }) },
],
},
});
const index = explicit ? 5 : 0;
stream.onRunStepDelta({
id: 'step',
delta: {
type: 'tool_calls',
tool_calls: [{ index, name: 'get_', args: '{"city":"Madrid"}' }],
},
});
stream.onRunStepDelta({
id: 'step',
delta: { type: 'tool_calls', tool_calls: [{ index, name: 'weather' }] },
});
stream.finish();
expect([...toolCalls.values()].map((call) => call.function)).toEqual([
{ name: 'get_weather', arguments: '{"city":"Madrid"}' },
{ name: 'lookup', arguments: '{"city":"Paris"}' },
]);
expect(new Set([...toolCalls.values()].map((call) => call.id)).size).toBe(2);
},
);
it('correlates an id-less declaration with a later identified chunk rather than adding a call', () => {
const toolCalls = new Map<number, ToolCall>();
const stream = createOpenAIToolCallStream({ toolCalls });
stream.onRunStep({
id: 'step',
stepDetails: { type: 'tool_calls', tool_calls: [{ name: 'lookup', args: {} }] },
});
stream.onRunStepDelta({
id: 'step',
delta: {
type: 'tool_calls',
tool_calls: [{ id: 'provider', name: 'lookup', index: 9, args: '{}' }],
},
});
stream.finish();
expect(toolCalls.size).toBe(1);
expect(toolCalls.get(0)?.id).toBe('provider');
});
it('assembles name fragments before and after a full snapshot without appending the snapshot', () => {
const toolCalls = new Map<number, ToolCall>();
const stream = createOpenAIToolCallStream({ toolCalls });
stream.onRunStepDelta({
id: 'step',
delta: { type: 'tool_calls', tool_calls: [{ id: 'a', index: 0, name: 'get_', args: '{}' }] },
});
stream.onRunStep({
id: 'step',
stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'a', name: 'get_weather', args: {} }] },
});
stream.onRunStepDelta({
id: 'step',
delta: { type: 'tool_calls', tool_calls: [{ index: 0, name: 'weather' }] },
});
stream.finish();
expect(toolCalls.get(0)?.function.name).toBe('get_weather');
});
it('seals before invoking an emitter that reenters finish or throws', () => {
const toolCalls = new Map<number, ToolCall>();
const emit = jest.fn(() => {
stream.finish();
throw new Error('transport failed');
});
const stream = createOpenAIToolCallStream({ toolCalls, emit });
stream.onRunStep({
id: 'step',
stepDetails: { type: 'tool_calls', tool_calls: [{ name: 'lookup', args: {} }] },
});
expect(() => stream.finish()).toThrow('transport failed');
expect(() => stream.finish()).not.toThrow();
expect(emit).toHaveBeenCalledTimes(1);
});
it('does not emit any tool identity if another call is malformed', () => {
const toolCalls = new Map<number, ToolCall>();
const emit = jest.fn();
const stream = createOpenAIToolCallStream({ toolCalls, emit });
stream.onRunStep({
id: 'step',
stepDetails: {
type: 'tool_calls',
tool_calls: [
{ name: 'lookup', args: {} },
{ name: 'lookup', args: 'NOT_JSON' },
],
},
});
expect(() => stream.finish()).toThrow('Invalid tool call arguments');
expect(emit).not.toHaveBeenCalled();
expect(toolCalls.size).toBe(0);
});
it('matches late id-less snapshot arrays to already indexed raw calls', () => {
const toolCalls = new Map<number, ToolCall>();
const stream = createOpenAIToolCallStream({ toolCalls });
for (const index of [0, 1])
stream.onRunStepDelta({
id: 'step',
delta: {
type: 'tool_calls',
tool_calls: [{ index, name: 'lookup', args: JSON.stringify({ index }) }],
},
});
stream.onRunStep({
id: 'step',
stepDetails: {
type: 'tool_calls',
tool_calls: [
{ name: 'lookup', args: { index: 0 } },
{ name: 'lookup', args: { index: 1 } },
],
},
});
stream.finish();
expect(toolCalls.size).toBe(2);
expect([...toolCalls.values()].map((call) => call.function.arguments)).toEqual([
'{"index":0}',
'{"index":1}',
]);
});
it('separates fully identified raw calls when no provider index is supplied', () => {
const toolCalls = new Map<number, ToolCall>();
const stream = createOpenAIToolCallStream({ toolCalls });
for (const id of ['a', 'b'])
stream.onRunStepDelta({
id: 'step',
delta: { type: 'tool_calls', tool_calls: [{ id, name: 'lookup', args: '{}' }] },
});
stream.finish();
expect([...toolCalls.values()].map((call) => call.id)).toEqual(['a', 'b']);
});
it('does not guess that identical name fragments are replayed data', () => {
const toolCalls = new Map<number, ToolCall>();
const stream = createOpenAIToolCallStream({ toolCalls });
for (const name of ['a', 'a'])
stream.onRunStepDelta({
id: 'step',
delta: { type: 'tool_calls', tool_calls: [{ index: 0, name }] },
});
stream.onRunStepDelta({
id: 'step',
delta: { type: 'tool_calls', tool_calls: [{ index: 0, args: '{}' }] },
});
stream.finish();
expect(toolCalls.get(0)?.function.name).toBe('aa');
});
it.each([true, false])(
'does not alias parallel shared ID prefixes (snapshots=%s)',
(snapshots) => {
const toolCalls = new Map<number, ToolCall>();
const stream = createOpenAIToolCallStream({ toolCalls });
if (snapshots)
stream.onRunStep({
id: 'step',
stepDetails: {
type: 'tool_calls',
tool_calls: [
{ id: 'call_0', name: 'get_weather', args: {} },
{ id: 'call_1', name: 'get_weather', args: {} },
],
},
});
for (const index of [0, 1])
stream.onRunStepDelta({
id: 'step',
delta: {
type: 'tool_calls',
tool_calls: [{ index, id: 'call_', name: 'get_', args: '{"i":' }],
},
});
for (const index of [1, 0])
stream.onRunStepDelta({
id: 'step',
delta: {
type: 'tool_calls',
tool_calls: [{ index, id: String(index), name: 'weather', args: `${index}}` }],
},
});
stream.finish();
expect([...toolCalls.values()]).toEqual(
[0, 1].map((i) => ({
id: `call_${i}`,
type: 'function',
function: { name: 'get_weather', arguments: JSON.stringify({ i }) },
})),
);
},
);
});

View file

@ -0,0 +1,302 @@
import { response } from 'express';
import { AIMessageChunk } from '@librechat/agents/langchain/messages';
import {
ChatModelStreamHandler,
HandlerRegistry,
Providers,
StandardGraph,
} from '@librechat/agents';
import type { Response } from 'express';
import type { ChatCompletionChunk, ToolCall } from './types';
import {
createOpenAIHandlers,
createOpenAIStreamTracker,
createOpenAIContentAggregator,
createOpenAIToolCallStream,
OpenAIRunStepHandler,
OpenAIRunStepDeltaHandler,
sendFinalChunk,
} from './handlers';
/** No graph event fixtures: the SDK generates run steps and deltas from real
* AIMessageChunk instances. Only the provider transport is omitted. */
describe('tool-call projection with real SDK graph dispatch', () => {
it.each([true, false])(
'preserves interleaved calls across SDK steps (stream=%s)',
async (streaming) => {
const frames: string[] = [];
const tracker = createOpenAIStreamTracker();
const context = { requestId: 'test', created: 1, model: 'fixture' };
const res: Response = Object.create(response);
jest.spyOn(res, 'write').mockImplementation((frame) => {
frames.push(String(frame));
return true;
});
const config = { tracker, context, res };
const stream = createOpenAIToolCallStream({ toolCalls: tracker.toolCalls });
const handlers = streaming
? createOpenAIHandlers(config)
: {
on_run_step: new OpenAIRunStepHandler(stream),
on_run_step_delta: new OpenAIRunStepDeltaHandler(stream),
};
const graph = new StandardGraph({
runId: 'test',
agents: [{ agentId: 'agent', provider: Providers.OPENAI, tools: [] }],
});
graph.config = { configurable: { run_id: 'test', thread_id: 'thread' } };
graph.handlerRegistry = new HandlerRegistry();
for (const [event, handler] of Object.entries(handlers)) {
graph.handlerRegistry.register(event, handler);
}
const producer = new ChatModelStreamHandler();
for (const chunk of [
{ index: 0, id: 'a', name: 'get_time', args: '' },
{ index: 1, id: 'b', name: 'get_time', args: '' },
{ index: 0, args: '{"city":' },
{ index: 1, args: '{"city":"Paris"}' },
{ index: 0, args: '"Madrid"}' },
]) {
await producer.handle(
'on_chat_model_stream',
{
chunk: new AIMessageChunk({
content: '',
tool_call_chunks: [{ ...chunk, type: 'tool_call_chunk' }],
}),
},
{ langgraph_node: 'agent=agent', langgraph_step: 1 },
graph,
);
}
expect(frames).toEqual([]);
if (streaming) tracker.finishToolCalls?.();
else stream.finish();
expect([...tracker.toolCalls.values()].map((call) => call.function.arguments)).toEqual([
'{"city":"Madrid"}',
'{"city":"Paris"}',
]);
if (streaming) {
const received = new Map<number, ToolCall>();
for (const frame of frames) {
const chunk: ChatCompletionChunk = JSON.parse(frame.slice(6));
for (const part of chunk.choices[0].delta.tool_calls ?? []) {
const existing = received.get(part.index);
if (existing) {
existing.function.arguments += part.function?.arguments ?? '';
} else {
expect(part.id).toBeDefined();
expect(part.function?.name).toBe('get_time');
received.set(part.index, {
id: part.id!,
type: 'function',
function: {
name: part.function!.name!,
arguments: part.function?.arguments ?? '',
},
});
}
}
}
expect([...received.values()]).toEqual([...tracker.toolCalls.values()]);
await producer.handle(
'on_chat_model_stream',
{
chunk: new AIMessageChunk({ content: 'Both tools completed. Here is the answer.' }),
},
{ langgraph_node: 'agent=agent', langgraph_step: 3 },
graph,
);
expect(tracker.hasText).toBe(true);
sendFinalChunk(config, 'stop');
const final: ChatCompletionChunk = JSON.parse(frames[frames.length - 2].slice(6));
expect(final.choices[0].finish_reason).toBe('stop');
}
},
);
it('isolates provider index zero across agents, invocations and checkpoint namespaces', () => {
const tracker = createOpenAIStreamTracker();
const stream = createOpenAIToolCallStream({ toolCalls: tracker.toolCalls });
const metadata = [
{ langgraph_node: 'agent=a', langgraph_step: 1 },
{ langgraph_node: 'agent=b', langgraph_step: 1 },
{ langgraph_node: 'agent=a', langgraph_step: 3 },
{ langgraph_node: 'agent=a', langgraph_step: 3, checkpoint_ns: 'child' },
];
for (const [i, meta] of metadata.entries()) {
stream.onRunStep(
{
id: `step_${i}`,
stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'call_0', name: 'get_time' }] },
},
meta,
);
stream.onRunStepDelta(
{
id: `step_${i}`,
delta: { type: 'tool_calls', tool_calls: [{ index: 0, id: 'call_0', name: 'get_time' }] },
},
meta,
);
}
for (const [i, meta] of metadata.entries()) {
stream.onRunStepDelta(
{
id: `step_${i}`,
delta: { type: 'tool_calls', tool_calls: [{ index: 0, args: JSON.stringify({ i }) }] },
},
meta,
);
}
stream.finish();
expect([...tracker.toolCalls.values()].map((call) => call.function.arguments)).toEqual(
metadata.map((_, i) => JSON.stringify({ i })),
);
expect(new Set([...tracker.toolCalls.values()].map((call) => call.id)).size).toBe(4);
});
it.each([true, false])(
'projects complete-only SDK messages at response completion (stream=%s)',
async (streaming) => {
const frames: string[] = [];
const res: Response = Object.create(response);
jest.spyOn(res, 'write').mockImplementation((frame) => {
frames.push(String(frame));
return true;
});
const tracker = createOpenAIStreamTracker();
const aggregator = createOpenAIContentAggregator();
const config = {
tracker,
res,
context: { requestId: 'complete', created: 1, model: 'fixture' },
};
const handlers = createOpenAIHandlers(streaming ? config : { aggregator });
const graph = new StandardGraph({
runId: 'complete',
agents: [{ agentId: 'agent', provider: Providers.OPENAI, tools: [] }],
});
graph.config = { configurable: { run_id: 'complete', thread_id: 'thread' } };
graph.handlerRegistry = new HandlerRegistry();
for (const [event, handler] of Object.entries(handlers))
graph.handlerRegistry.register(event, handler);
await new ChatModelStreamHandler().handle(
'on_chat_model_stream',
{
chunk: new AIMessageChunk({
content: '',
tool_calls: [{ id: 'a', name: 'get_time', args: { city: 'Madrid' } }],
}),
},
{ langgraph_node: 'agent=agent', langgraph_step: 1 },
graph,
);
const target = streaming ? tracker : aggregator;
expect(target.toolCalls.size).toBe(0);
if (streaming) sendFinalChunk(config);
else target.finishToolCalls?.();
expect(target.toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}');
if (streaming) {
const chunks: ChatCompletionChunk[] = frames
.filter((frame) => frame !== 'data: [DONE]\n\n')
.map((frame) => JSON.parse(frame.slice(6)));
expect(
chunks
.flatMap((chunk) => chunk.choices[0].delta.tool_calls ?? [])
.map((call) => call.function?.arguments)
.join(''),
).toBe('{"city":"Madrid"}');
}
},
);
it('uses graph-owned stream segments without stealing late fragments from the old segment', () => {
const graph = new StandardGraph({
runId: 'segments',
agents: [{ agentId: 'a', provider: Providers.OPENAI, tools: [] }],
});
graph.config = { configurable: { run_id: 'segments', thread_id: 'thread' } };
const meta = { langgraph_node: 'agent=a', langgraph_step: 1 };
const tracker = createOpenAIStreamTracker();
const stream = createOpenAIToolCallStream({ toolCalls: tracker.toolCalls });
const start = new OpenAIRunStepHandler(stream);
const delta = new OpenAIRunStepDeltaHandler(stream);
for (let i = 0; i < 2; i++) {
start.handle(
'on_run_step',
{
id: `step_${i}`,
stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'call_0', name: 'get_time' }] },
},
meta,
graph,
);
delta.handle(
'on_run_step_delta',
{
id: `step_${i}`,
delta: { type: 'tool_calls', tool_calls: [{ id: 'call_0', index: 0, name: 'get_time' }] },
},
meta,
graph,
);
graph.advanceStreamSegment();
}
for (let i = 0; i < 2; i++)
delta.handle(
'on_run_step_delta',
{
id: `step_${i}`,
delta: { type: 'tool_calls', tool_calls: [{ index: 0, args: JSON.stringify({ i }) }] },
},
meta,
graph,
);
stream.finish();
expect([...tracker.toolCalls.values()].map((call) => call.function.arguments)).toEqual([
'{"i":0}',
'{"i":1}',
]);
});
it('assembles the name before emitting when real SDK chunks split it', async () => {
const tracker = createOpenAIStreamTracker();
const emitted: ChatCompletionChunk['choices'][number]['delta'][] = [];
const stream = createOpenAIToolCallStream({
toolCalls: tracker.toolCalls,
emit: (delta) => emitted.push(delta),
});
const graph = new StandardGraph({
runId: 'split',
agents: [{ agentId: 'a', provider: Providers.OPENAI, tools: [] }],
});
graph.config = { configurable: { run_id: 'split', thread_id: 'split' } };
graph.handlerRegistry = new HandlerRegistry();
const handlers: ReturnType<typeof createOpenAIHandlers> = {
on_run_step: new OpenAIRunStepHandler(stream),
on_run_step_delta: new OpenAIRunStepDeltaHandler(stream),
};
for (const [event, handler] of Object.entries(handlers))
graph.handlerRegistry.register(event, handler);
const producer = new ChatModelStreamHandler();
for (const chunk of [
{ id: 'a', index: 0, name: 'get_', args: '' },
{ index: 0, args: '{}' },
{ index: 0, name: 'weather' },
])
await producer.handle(
'on_chat_model_stream',
{
chunk: new AIMessageChunk({
content: '',
tool_call_chunks: [{ ...chunk, type: 'tool_call_chunk' }],
}),
},
{ langgraph_node: 'agent=a', langgraph_step: 1 },
graph,
);
expect(emitted).toEqual([]);
stream.finish();
expect(tracker.toolCalls.get(0)?.function.name).toBe('get_weather');
expect(emitted[0].tool_calls?.[0].function?.name).toBe('get_weather');
});
});

View file

@ -0,0 +1,616 @@
import type { ChatCompletionChunkChoice, OpenAIResponseContext, ToolCall } from './types';
import type { RunStepData, RunStepDeltaData } from './handlers';
import {
sendFinalChunk,
completeOpenAIToolCalls,
createOpenAIStreamTracker,
createOpenAIToolCallStream,
createOpenAIContentAggregator,
} from './handlers';
import { buildNonStreamingResponse } from './service';
type Delta = ChatCompletionChunkChoice['delta'];
/**
* Event payloads below are the shapes `@librechat/agents` actually dispatches
* for a Bedrock Converse stream: the run step opens a call with its id and name
* and carries no tool-call index, the first delta fragment repeats id and name
* with the provider's content-block index, and later fragments carry only that
* index and an argument slice. A step's own `index` is its position in the
* response content, which is why it appears here as an unrelated number.
*/
function runStep(stepId: string, contentIndex: number, id: string, name: string): RunStepData {
return {
id: stepId,
index: contentIndex,
stepDetails: {
type: 'tool_calls',
tool_calls: [{ name, args: {}, id, type: 'tool_call' }],
},
};
}
function opensCall(
stepId: string,
providerIndex: number,
id: string,
name: string,
): RunStepDeltaData {
return {
id: stepId,
delta: {
type: 'tool_calls',
tool_calls: [{ name, id, index: providerIndex, type: 'tool_call_chunk' }],
},
};
}
function streamsArgs(stepId: string, providerIndex: number, args: string): RunStepDeltaData {
return {
id: stepId,
delta: {
type: 'tool_calls',
tool_calls: [{ args, index: providerIndex, type: 'tool_call_chunk' }],
},
};
}
/**
* The accumulation an OpenAI-compatible client performs, including the rule that
* produced the reported `AI_InvalidResponseDataError: Expected 'id' to be a
* string`: the first chunk seen at an index must declare the call.
*/
function accumulateLikeClient(deltas: Delta[]) {
const calls = new Map<number, { id: string; name: string; arguments: string }>();
for (const delta of deltas) {
for (const fragment of delta.tool_calls ?? []) {
const index = fragment.index as number;
const open = calls.get(index);
if (open === undefined) {
if (!fragment.id || !fragment.function?.name) {
throw new Error(`first chunk at index ${index} declares no id and name`);
}
calls.set(index, {
id: fragment.id,
name: fragment.function.name,
arguments: fragment.function.arguments ?? '',
});
continue;
}
open.arguments += fragment.function?.arguments ?? '';
}
}
return [...calls.entries()].map(([index, call]) => ({ index, ...call }));
}
function streamingBridge() {
const deltas: Delta[] = [];
const toolCalls = new Map<number, ToolCall>();
const stream = createOpenAIToolCallStream({
toolCalls,
emit: (delta) => deltas.push(delta),
});
return { stream, deltas, toolCalls };
}
describe('outward tool call indexes', () => {
it('keeps one index from the declaration through the argument fragments', () => {
const { stream, deltas, toolCalls } = streamingBridge();
stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time'));
stream.onRunStepDelta(opensCall('step_1', 1, 'call_a', 'get_time'));
stream.onRunStepDelta(streamsArgs('step_1', 1, '{"city":'));
stream.onRunStepDelta(streamsArgs('step_1', 1, '"Madrid"}'));
expect(deltas).toEqual([]);
stream.finish();
expect(deltas.flatMap((delta) => delta.tool_calls ?? []).map((call) => call.index)).toEqual([
0, 0,
]);
expect(accumulateLikeClient(deltas)).toEqual([
{ index: 0, id: 'call_a', name: 'get_time', arguments: '{"city":"Madrid"}' },
]);
expect([...toolCalls.values()]).toEqual([
{
id: 'call_a',
type: 'function',
function: { name: 'get_time', arguments: '{"city":"Madrid"}' },
},
]);
});
it('declares a call once when the run step and the first fragment both name it', () => {
const { stream, deltas } = streamingBridge();
stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time'));
stream.onRunStepDelta(opensCall('step_1', 1, 'call_a', 'get_time'));
stream.finish();
const declarations = deltas
.flatMap((delta) => delta.tool_calls ?? [])
.filter((call) => call.id !== undefined);
expect(declarations).toEqual([
{ index: 0, id: 'call_a', type: 'function', function: { name: 'get_time', arguments: '' } },
]);
});
it('separates parallel calls that arrive as consecutive single-call steps', () => {
const { stream, deltas, toolCalls } = streamingBridge();
stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time'));
stream.onRunStepDelta(opensCall('step_1', 0, 'call_a', 'get_time'));
stream.onRunStepDelta(streamsArgs('step_1', 0, '{"city":"Madrid"}'));
stream.onRunStep(runStep('step_2', 2, 'call_b', 'get_time'));
stream.onRunStepDelta(opensCall('step_2', 1, 'call_b', 'get_time'));
stream.onRunStepDelta(streamsArgs('step_2', 1, '{"city":"Paris"}'));
stream.finish();
expect(accumulateLikeClient(deltas)).toEqual([
{ index: 0, id: 'call_a', name: 'get_time', arguments: '{"city":"Madrid"}' },
{ index: 1, id: 'call_b', name: 'get_time', arguments: '{"city":"Paris"}' },
]);
expect([...toolCalls.keys()]).toEqual([0, 1]);
});
it('allocates a new index when a later model invocation reuses provider index zero', () => {
const { stream, deltas, toolCalls } = streamingBridge();
stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time'));
stream.onRunStepDelta(opensCall('step_1', 0, 'call_a', 'get_time'));
stream.onRunStepDelta(streamsArgs('step_1', 0, '{"city":"Madrid"}'));
stream.onRunStep(runStep('step_3', 3, 'call_b', 'get_time'));
stream.onRunStepDelta(opensCall('step_3', 0, 'call_b', 'get_time'));
stream.onRunStepDelta(streamsArgs('step_3', 0, '{"city":"Paris"}'));
stream.finish();
expect(accumulateLikeClient(deltas)).toEqual([
{ index: 0, id: 'call_a', name: 'get_time', arguments: '{"city":"Madrid"}' },
{ index: 1, id: 'call_b', name: 'get_time', arguments: '{"city":"Paris"}' },
]);
expect(toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}');
expect(toolCalls.get(1)?.function.arguments).toBe('{"city":"Paris"}');
});
it('attributes unidentified argument fragments to the call its step opened', () => {
const { stream, deltas } = streamingBridge();
stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time'));
stream.onRunStepDelta(streamsArgs('step_1', 7, '{"city":"Madrid"}'));
stream.finish();
expect(accumulateLikeClient(deltas)).toEqual([
{ index: 0, id: 'call_a', name: 'get_time', arguments: '{"city":"Madrid"}' },
]);
});
it('drops a fragment it cannot attribute rather than charging another call', () => {
const { stream, deltas, toolCalls } = streamingBridge();
stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time'));
stream.onRunStepDelta(opensCall('step_1', 0, 'call_a', 'get_time'));
stream.onRunStep(runStep('step_2', 2, 'call_b', 'get_time'));
stream.onRunStepDelta(opensCall('step_2', 1, 'call_b', 'get_time'));
stream.onRunStepDelta({
id: 'step_unknown',
delta: { type: 'tool_calls', tool_calls: [{ index: 4, args: '{"city":"Lisbon"}' }] },
});
expect(() => stream.finish()).toThrow('Unattributable tool call arguments');
expect(toolCalls.size).toBe(0);
expect(deltas).toEqual([]);
});
it.each([true, false])(
'isolates repeated provider IDs across interleaved steps (stream=%s)',
(streaming) => {
const deltas: Delta[] = [];
const toolCalls = new Map<number, ToolCall>();
const stream = createOpenAIToolCallStream({
toolCalls,
emit: streaming ? (delta) => deltas.push(delta) : undefined,
});
stream.onRunStep(runStep('agent_a_turn_1', 1, 'call_0', 'get_time'));
stream.onRunStepDelta(opensCall('agent_a_turn_1', 0, 'call_0', 'get_time'));
stream.onRunStepDelta(streamsArgs('agent_a_turn_1', 0, '{"city":'));
stream.onRunStep(runStep('agent_b_turn_1', 2, 'call_0', 'get_time'));
stream.onRunStepDelta(opensCall('agent_b_turn_1', 0, 'call_0', 'get_time'));
stream.onRunStepDelta(streamsArgs('agent_b_turn_1', 0, '{"city":"Paris"}'));
stream.onRunStepDelta(streamsArgs('agent_a_turn_1', 0, '"Madrid"}'));
stream.onRunStep(runStep('agent_a_turn_2', 3, 'call_0', 'get_time'));
stream.onRunStepDelta(opensCall('agent_a_turn_2', 0, 'call_0', 'get_time'));
stream.onRunStepDelta(streamsArgs('agent_a_turn_2', 0, '{"city":"Lisbon"}'));
stream.finish();
expect([...toolCalls.keys()]).toEqual([0, 1, 2]);
expect([...toolCalls.values()].map((call) => call.function.arguments)).toEqual([
'{"city":"Madrid"}',
'{"city":"Paris"}',
'{"city":"Lisbon"}',
]);
if (streaming) {
expect(accumulateLikeClient(deltas).map((call) => call.arguments)).toEqual([
'{"city":"Madrid"}',
'{"city":"Paris"}',
'{"city":"Lisbon"}',
]);
}
},
);
it('declares OpenAI-shaped names without waiting for an identifying delta', () => {
const { stream, deltas, toolCalls } = streamingBridge();
stream.onRunStep({
id: 'step_function',
stepDetails: {
type: 'tool_calls',
tool_calls: [
{
id: 'call_a',
type: 'function',
function: { name: 'get_time', arguments: '{"city":"Madrid"}' },
},
],
},
});
stream.finish();
expect(toolCalls.get(0)?.function.name).toBe('get_time');
expect(accumulateLikeClient(deltas)).toMatchObject([
{ index: 0, id: 'call_a', name: 'get_time' },
]);
});
it.each([true, false])(
'binds multiple declared calls before ID-less fragments (explicit indexes=%s)',
(explicit) => {
const { stream, deltas, toolCalls } = streamingBridge();
const indexes = explicit ? [4, 8] : [0, 1];
stream.onRunStep({
id: 'parallel',
index: 12,
stepDetails: {
type: 'tool_calls',
tool_calls: [
{ id: 'call_a', name: 'get_time', ...(explicit && { index: indexes[0] }) },
{ id: 'call_b', name: 'get_time', ...(explicit && { index: indexes[1] }) },
],
},
});
stream.onRunStepDelta(streamsArgs('parallel', indexes[1], '{"city":"Paris"}'));
stream.onRunStepDelta(streamsArgs('parallel', indexes[0], '{"city":"Madrid"}'));
stream.finish();
stream.finish();
expect(accumulateLikeClient(deltas).map((call) => call.arguments)).toEqual([
'{"city":"Madrid"}',
'{"city":"Paris"}',
]);
expect([...toolCalls.values()].map((call) => call.function.arguments)).toEqual([
'{"city":"Madrid"}',
'{"city":"Paris"}',
]);
},
);
it('does not re-declare a replayed step or bind unrelated indexes to a known call', () => {
const { stream, deltas, toolCalls } = streamingBridge();
stream.onRunStep(runStep('step', 1, 'call_a', 'get_time'));
stream.onRunStep(runStep('step', 1, 'call_a', 'get_time'));
stream.onRunStepDelta(opensCall('step', 3, 'call_a', 'get_time'));
stream.onRunStepDelta(streamsArgs('step', 9, '{"wrong":true}'));
stream.onRunStepDelta(streamsArgs('step', 3, '{}'));
expect(() => stream.finish()).toThrow('Unattributable tool call arguments');
expect(deltas).toEqual([]);
expect(toolCalls.size).toBe(0);
});
it('buffers identified arguments until a later name-only fragment can declare the call', () => {
const { stream, deltas, toolCalls } = streamingBridge();
stream.onRunStepDelta({
id: 'step',
delta: { type: 'tool_calls', tool_calls: [{ id: 'call_a', index: 3, args: '{"city":' }] },
});
expect(deltas).toEqual([]);
stream.onRunStepDelta({
id: 'step',
delta: {
type: 'tool_calls',
tool_calls: [{ index: 3, function: { name: 'get_time', arguments: '"Madrid"}' } }],
},
});
stream.finish();
expect(accumulateLikeClient(deltas)).toEqual([
{ index: 0, id: 'call_a', name: 'get_time', arguments: '{"city":"Madrid"}' },
]);
expect(toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}');
});
it('prefers identified provider indexes to declaration position', () => {
const { stream, deltas } = streamingBridge();
stream.onRunStep({
id: 'step',
stepDetails: {
type: 'tool_calls',
tool_calls: [
{ id: 'call_a', name: 'get_time' },
{ id: 'call_b', name: 'get_time' },
],
},
});
stream.onRunStepDelta(opensCall('step', 1, 'call_a', 'get_time'));
stream.onRunStepDelta(opensCall('step', 0, 'call_b', 'get_time'));
stream.onRunStepDelta(streamsArgs('step', 0, '{"city":"Paris"}'));
stream.onRunStepDelta(streamsArgs('step', 1, '{"city":"Madrid"}'));
stream.finish();
expect(accumulateLikeClient(deltas).map((call) => call.arguments)).toEqual([
'{"city":"Madrid"}',
'{"city":"Paris"}',
]);
});
it('never merges unnamed-index fragments into a multi-call step', () => {
const { stream, toolCalls } = streamingBridge();
stream.onRunStep({
id: 'step',
stepDetails: {
type: 'tool_calls',
tool_calls: [
{ id: 'a', name: 'get_time' },
{ id: 'b', name: 'get_time' },
],
},
});
stream.onRunStepDelta({
id: 'step',
delta: { type: 'tool_calls', tool_calls: [{ args: 'unattributable' }] },
});
expect(() => stream.finish()).toThrow('Unattributable tool call arguments');
expect(toolCalls.size).toBe(0);
});
it('gives id-based consumers unique IDs and starts each response with fresh state', () => {
const { stream, toolCalls } = streamingBridge();
stream.onRunStep(runStep('s1', 1, 'call_0', 'get_time'));
stream.onRunStep(runStep('s2', 2, 'call_0', 'get_time'));
stream.onRunStep(runStep('s3', 3, 'call_0_1', 'get_time'));
stream.finish();
expect(new Set([...toolCalls.values()].map((call) => call.id)).size).toBe(3);
const second = streamingBridge();
second.stream.onRunStep(runStep('s1', 1, 'call_0', 'get_time'));
second.stream.finish();
expect([...second.toolCalls.keys()]).toEqual([0]);
expect(second.toolCalls.get(0)?.id).toBe('call_0');
});
it('accumulates the same calls for a non-streaming response with no emitter', () => {
const aggregator = createOpenAIContentAggregator();
const stream = createOpenAIToolCallStream({ toolCalls: aggregator.toolCalls });
stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time'));
stream.onRunStepDelta(opensCall('step_1', 0, 'call_a', 'get_time'));
stream.onRunStepDelta(streamsArgs('step_1', 0, '{"city":"Madrid"}'));
stream.onRunStep(runStep('step_3', 3, 'call_b', 'get_time'));
stream.onRunStepDelta(opensCall('step_3', 0, 'call_b', 'get_time'));
stream.onRunStepDelta(streamsArgs('step_3', 0, '{"city":"Paris"}'));
stream.finish();
const response = buildNonStreamingResponse(
{ requestId: 'chatcmpl-test', created: 1778317637, model: 'agent_test' },
'Checking both.',
'',
aggregator.toolCalls,
{ prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
);
expect(response.choices[0].message.tool_calls).toEqual([
{
id: 'call_a',
type: 'function',
function: { name: 'get_time', arguments: '{"city":"Madrid"}' },
},
{
id: 'call_b',
type: 'function',
function: { name: 'get_time', arguments: '{"city":"Paris"}' },
},
]);
});
});
describe('complete snapshots and streamed fragments', () => {
function snapshot(step = 'complete', args = '{"city":"Madrid"}') {
return {
id: step,
stepDetails: {
type: 'tool_calls',
tool_calls: [{ id: 'a', function: { name: 'get_time', arguments: args } }],
},
};
}
it.each(['before', 'after', 'between'])(
'never concatenates a snapshot %s raw fragments',
(order) => {
const { stream, deltas, toolCalls } = streamingBridge();
if (order === 'before') stream.onRunStep(snapshot());
stream.onRunStepDelta(opensCall('complete', 0, 'a', 'get_time'));
stream.onRunStepDelta(streamsArgs('complete', 0, '{"city":'));
if (order === 'between') stream.onRunStep(snapshot());
stream.onRunStepDelta(streamsArgs('complete', 0, '"Madrid"}'));
if (order === 'after') stream.onRunStep(snapshot());
stream.finish();
expect(accumulateLikeClient(deltas)[0].arguments).toBe('{"city":"Madrid"}');
expect(toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}');
},
);
it('captures a native snapshot without retaining the producer object', () => {
const { stream, toolCalls } = streamingBridge();
const args = { city: 'Madrid' };
stream.onRunStep({
id: 'step',
stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'a', name: 'get_time', args }] },
});
args.city = 'Paris';
stream.finish();
expect(toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}');
});
it('preserves empty native objects and does not flush a snapshot on identifying-only chunks', () => {
const { stream, deltas } = streamingBridge();
stream.onRunStep({
id: 'complete',
stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'a', name: 'noop', args: {} }] },
});
stream.onRunStepDelta(opensCall('complete', 0, 'a', 'noop'));
expect(deltas).toEqual([]);
stream.finish();
expect(accumulateLikeClient(deltas)[0].arguments).toBe('{}');
});
it('flushes replayed declarations only once and ignores writes after completion', () => {
const { stream, deltas } = streamingBridge();
stream.onRunStep(snapshot());
stream.onRunStep(snapshot());
stream.finish();
stream.finish();
stream.onRunStep(snapshot('late'));
stream.onRunStepDelta(streamsArgs('complete', 0, 'late'));
expect(accumulateLikeClient(deltas)).toEqual([
{ index: 0, id: 'a', name: 'get_time', arguments: '{"city":"Madrid"}' },
]);
});
it.each(['abort', 'reject', 'sync throw'])(
'does not flush fallback snapshots on %s',
async (failure) => {
const signal = new AbortController();
const deltas: Delta[] = [];
const toolCalls = new Map<number, ToolCall>();
const stream = createOpenAIToolCallStream({
toolCalls,
signal: signal.signal,
emit: (delta) => deltas.push(delta),
});
stream.onRunStep(snapshot());
const error = new Error('provider failed');
const run = completeOpenAIToolCalls(stream, () => {
if (failure === 'sync throw') throw error;
if (failure === 'reject') return Promise.reject(error);
signal.abort();
return Promise.resolve();
});
if (failure === 'abort') await expect(run).rejects.toMatchObject({ name: 'AbortError' });
else await expect(run).rejects.toBe(error);
stream.onRunStep(snapshot('late'));
stream.onRunStepDelta(streamsArgs('complete', 0, '{"city":"Paris"}'));
expect(() => stream.finish()).toThrow('Agent response aborted');
expect(toolCalls.size).toBe(0);
expect(deltas).toEqual([]);
const retry = streamingBridge();
retry.stream.onRunStep(snapshot());
retry.stream.finish();
expect(retry.toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}');
},
);
it('does not report success when argument fragments could not be attributed', () => {
const { stream } = streamingBridge();
stream.onRunStepDelta(streamsArgs('unknown', 9, '{}'));
expect(() => stream.finish()).toThrow('Unattributable tool call arguments');
});
it('never repairs truncated streamed arguments with a complete snapshot', () => {
const { stream, toolCalls } = streamingBridge();
stream.onRunStep(snapshot());
stream.onRunStepDelta(streamsArgs('complete', 0, '{"city":'));
expect(() => stream.finish()).toThrow('Invalid tool call arguments');
expect(toolCalls.size).toBe(0);
});
it('fails incomplete identity rather than reporting a successful missing call', () => {
const { stream } = streamingBridge();
stream.onRunStepDelta({
id: 'complete',
delta: { type: 'tool_calls', tool_calls: [{ id: 'a', index: 0, args: '{}' }] },
});
expect(() => stream.finish()).toThrow('Incomplete tool call');
});
it('validates all terminal snapshots before flushing any of them', () => {
const { stream, deltas } = streamingBridge();
stream.onRunStep(snapshot());
stream.onRunStep(snapshot('bad', 'PRIVATE-INCOMPLETE-INPUT'));
expect(() => stream.finish()).toThrow('Invalid tool call arguments in agent response');
expect(deltas).toEqual([]);
});
it('keeps a raw stream authoritative over an earlier parsed placeholder', () => {
const { stream, deltas } = streamingBridge();
stream.onRunStep(snapshot('complete', '{}'));
stream.onRunStepDelta(streamsArgs('complete', 0, '{"city":"Madrid"}'));
stream.finish();
expect(accumulateLikeClient(deltas)[0].arguments).toBe('{"city":"Madrid"}');
});
it.each(['wire', 'native'])('retains complete %s declarations without deltas', (shape) => {
const { stream, deltas, toolCalls } = streamingBridge();
stream.onRunStep({
id: 'complete',
stepDetails: {
type: 'tool_calls',
tool_calls: [
shape === 'wire'
? { id: 'a', function: { name: 'get_time', arguments: '{"city":"Madrid"}' } }
: { id: 'a', name: 'get_time', args: { city: 'Madrid' } },
],
},
});
stream.finish();
expect(toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}');
expect(accumulateLikeClient(deltas)[0].arguments).toBe('{"city":"Madrid"}');
});
});
describe('finish reason for a response that called tools', () => {
const context: OpenAIResponseContext = {
requestId: 'chatcmpl-test',
created: 1778317637,
model: 'agent_test',
};
it('preserves stop after a server-executed tool followed by final text', () => {
const tracker = createOpenAIStreamTracker();
const written: string[] = [];
const stream = createOpenAIToolCallStream({
toolCalls: tracker.toolCalls,
emit: () => undefined,
});
stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time'));
stream.finish();
tracker.addText();
sendFinalChunk({
context,
tracker,
res: { write: (chunk: string) => written.push(chunk) } as never,
});
const final = JSON.parse(written[0].replace(/^data: /, ''));
expect(final.choices[0].finish_reason).toBe('stop');
});
it('preserves stop in the final non-streaming answer after server tools', () => {
const toolCalls = new Map<number, ToolCall>();
const stream = createOpenAIToolCallStream({ toolCalls });
stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time'));
stream.finish();
const response = buildNonStreamingResponse(context, 'Checking.', '', toolCalls, {
prompt_tokens: 0,
completion_tokens: 0,
total_tokens: 0,
});
expect(response.choices[0].finish_reason).toBe('stop');
});
});