mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-16 13:00:55 +00:00
🎽 fix: Give Each Streamed Tool Call a Stable Outward Index (#16209)
Some checks are pending
Backend Unit Tests / Build packages (push) Waiting to run
Backend Unit Tests / Codegraph select (push) Waiting to run
Backend Unit Tests / TypeScript type checks (push) Blocked by required conditions
Backend Unit Tests / Circular dependency checks (push) Waiting to run
Backend Unit Tests / Tests: api (shard 1/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 2/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 3/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: data-provider (push) Blocked by required conditions
Backend Unit Tests / Tests: data-schemas (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 1/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 2/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 3/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 4/4) (push) Blocked by required conditions
Codegraph E2E Votes / vote (full suite) (push) Waiting to run
Docker Dev Branch Images Build / publish (push) Waiting to run
Frontend Unit Tests / Client build recovery regression (push) Waiting to run
Frontend Unit Tests / Codegraph select (push) Waiting to run
Frontend Unit Tests / Build packages (push) Waiting to run
Frontend Unit Tests / TypeScript type checks (client) (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/client (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 1/2) (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 2/2) (push) Blocked by required conditions
Frontend Unit Tests / Vite build verification (push) Blocked by required conditions
Docker Dev Images Build / publish (push) Waiting to run
Sync Helm Chart Tags / Ignore non-main push (push) Waiting to run
Sync Helm Chart Tags / Sync chart tags (push) Waiting to run
Some checks are pending
Backend Unit Tests / Build packages (push) Waiting to run
Backend Unit Tests / Codegraph select (push) Waiting to run
Backend Unit Tests / TypeScript type checks (push) Blocked by required conditions
Backend Unit Tests / Circular dependency checks (push) Waiting to run
Backend Unit Tests / Tests: api (shard 1/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 2/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 3/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: data-provider (push) Blocked by required conditions
Backend Unit Tests / Tests: data-schemas (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 1/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 2/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 3/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 4/4) (push) Blocked by required conditions
Codegraph E2E Votes / vote (full suite) (push) Waiting to run
Docker Dev Branch Images Build / publish (push) Waiting to run
Frontend Unit Tests / Client build recovery regression (push) Waiting to run
Frontend Unit Tests / Codegraph select (push) Waiting to run
Frontend Unit Tests / Build packages (push) Waiting to run
Frontend Unit Tests / TypeScript type checks (client) (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/client (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 1/2) (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 2/2) (push) Blocked by required conditions
Frontend Unit Tests / Vite build verification (push) Blocked by required conditions
Docker Dev Images Build / publish (push) Waiting to run
Sync Helm Chart Tags / Ignore non-main push (push) Waiting to run
Sync Helm Chart Tags / Sync chart tags (push) Waiting to run
* fix: give each streamed tool call a stable outward index * fix: scope tool-call fragments and preserve completed answers * fix: finalize agent tool-call snapshots without duplicating deltas * fix: assemble complete tool identities before response publication --------- Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
parent
e7a451cbf8
commit
ef89fbd5b3
9 changed files with 2218 additions and 238 deletions
|
|
@ -217,6 +217,13 @@ jest.mock('@librechat/api', () => ({
|
|||
buildAgentContextAttachmentsByAgentId: (...args) =>
|
||||
mockBuildAgentContextAttachmentsByAgentId(...args),
|
||||
createChunk: jest.fn().mockReturnValue({}),
|
||||
/** Not stubbed: the outward tool-call index this allocates is the behavior the
|
||||
* controller is responsible for wiring, so the spec runs the real projection. */
|
||||
createOpenAIToolCallStream: (...args) =>
|
||||
jest.requireActual('@librechat/api').createOpenAIToolCallStream(...args),
|
||||
completeOpenAIToolCalls: jest.requireActual('@librechat/api').completeOpenAIToolCalls,
|
||||
OpenAIRunStepHandler: jest.requireActual('@librechat/api').OpenAIRunStepHandler,
|
||||
OpenAIRunStepDeltaHandler: jest.requireActual('@librechat/api').OpenAIRunStepDeltaHandler,
|
||||
buildRunToolSet: jest.fn().mockReturnValue(new Set()),
|
||||
buildInitialToolSessions: jest.fn().mockReturnValue(mockInitialSessions),
|
||||
AgentRunEnvelopeError: MockAgentRunEnvelopeError,
|
||||
|
|
@ -518,6 +525,227 @@ describe('OpenAIChatCompletionController', () => {
|
|||
};
|
||||
});
|
||||
|
||||
it.each([true, false])(
|
||||
'projects interleaved calls and preserves the final stop (stream=%s)',
|
||||
async (streaming) => {
|
||||
const api = require('@librechat/api');
|
||||
const actual = jest.requireActual('@librechat/api');
|
||||
const names = [
|
||||
'createChunk',
|
||||
'writeSSE',
|
||||
'sendFinalChunk',
|
||||
'buildNonStreamingResponse',
|
||||
'createOpenAIStreamTracker',
|
||||
'createOpenAIContentAggregator',
|
||||
];
|
||||
const original = names.map((name) => [name, api[name].getMockImplementation()]);
|
||||
for (const name of names) api[name].mockImplementation(actual[name]);
|
||||
req.body.stream = streaming;
|
||||
api.validateRequest.mockReturnValueOnce({ request: req.body });
|
||||
mockProcessStream.mockImplementationOnce(async () => {
|
||||
const { customHandlers: h } = api.createRun.mock.calls.at(-1)[0];
|
||||
const meta = { langgraph_node: 'agent=test', langgraph_step: 1 };
|
||||
for (const [step, id, index] of [
|
||||
['s1', 'a', 0],
|
||||
['s2', 'b', 1],
|
||||
]) {
|
||||
await h.on_run_step.handle(
|
||||
'on_run_step',
|
||||
{
|
||||
id: step,
|
||||
stepDetails: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [{ id, name: 'get_time' }],
|
||||
},
|
||||
},
|
||||
meta,
|
||||
);
|
||||
await h.on_run_step_delta.handle(
|
||||
'on_run_step_delta',
|
||||
{
|
||||
id: step,
|
||||
delta: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [{ id, name: 'get_time', index }],
|
||||
},
|
||||
},
|
||||
meta,
|
||||
);
|
||||
}
|
||||
for (const [index, city] of [
|
||||
[0, 'Madrid'],
|
||||
[1, 'Paris'],
|
||||
]) {
|
||||
await h.on_run_step_delta.handle(
|
||||
'on_run_step_delta',
|
||||
{
|
||||
id: 's2',
|
||||
delta: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [{ index, args: JSON.stringify({ city }) }],
|
||||
},
|
||||
},
|
||||
meta,
|
||||
);
|
||||
}
|
||||
await h.on_message_delta.handle(
|
||||
'on_message_delta',
|
||||
{
|
||||
id: 'answer',
|
||||
delta: {
|
||||
content: [{ type: 'text', text: 'Both tools completed.' }],
|
||||
},
|
||||
},
|
||||
{ ...meta, langgraph_step: 3 },
|
||||
);
|
||||
});
|
||||
try {
|
||||
await OpenAIChatCompletionController(req, res);
|
||||
if (streaming) {
|
||||
const frames = res.write.mock.calls
|
||||
.map(([frame]) => frame)
|
||||
.filter((frame) => frame !== 'data: [DONE]\n\n')
|
||||
.map((frame) => JSON.parse(frame.slice(6)));
|
||||
expect(frames.at(-1).choices[0].finish_reason).toBe('stop');
|
||||
expect(
|
||||
frames
|
||||
.flatMap((frame) => frame.choices[0].delta.tool_calls ?? [])
|
||||
.filter((call) => call.function?.arguments),
|
||||
).toEqual([
|
||||
{ index: 0, function: { arguments: '{"city":"Madrid"}' } },
|
||||
{ index: 1, function: { arguments: '{"city":"Paris"}' } },
|
||||
]);
|
||||
} else {
|
||||
expect(res.json).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
choices: [
|
||||
expect.objectContaining({
|
||||
finish_reason: 'stop',
|
||||
message: expect.objectContaining({
|
||||
content: 'Both tools completed.',
|
||||
tool_calls: [
|
||||
{
|
||||
id: 'a',
|
||||
type: 'function',
|
||||
function: { name: 'get_time', arguments: '{"city":"Madrid"}' },
|
||||
},
|
||||
{
|
||||
id: 'b',
|
||||
type: 'function',
|
||||
function: { name: 'get_time', arguments: '{"city":"Paris"}' },
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
],
|
||||
}),
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
for (const [name, implementation] of original) api[name].mockImplementation(implementation);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each(
|
||||
[true, false].flatMap((stream) =>
|
||||
['native-string', 'wire-object', 'split', 'idless'].map((shape) => [stream, shape]),
|
||||
),
|
||||
)(
|
||||
'publishes complete identity and arguments before terminal output (stream=%s, shape=%s)',
|
||||
async (streaming, shape) => {
|
||||
const api = require('@librechat/api');
|
||||
const actual = jest.requireActual('@librechat/api');
|
||||
const names = [
|
||||
'createChunk',
|
||||
'writeSSE',
|
||||
'sendFinalChunk',
|
||||
'buildNonStreamingResponse',
|
||||
'createOpenAIStreamTracker',
|
||||
'createOpenAIContentAggregator',
|
||||
];
|
||||
const originals = names.map((name) => [name, api[name].getMockImplementation()]);
|
||||
for (const name of names) api[name].mockImplementation(actual[name]);
|
||||
req.body.stream = streaming;
|
||||
api.validateRequest.mockReturnValueOnce({ request: req.body });
|
||||
mockProcessStream.mockImplementationOnce(async () => {
|
||||
const { customHandlers: h } = api.createRun.mock.calls.at(-1)[0];
|
||||
await h.on_run_step.handle('on_run_step', {
|
||||
id: 'complete',
|
||||
stepDetails: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [
|
||||
(() => {
|
||||
if (shape === 'native-string')
|
||||
return { id: 'a', name: 'get_time', args: '{"city":"Madrid"}' };
|
||||
if (shape === 'wire-object')
|
||||
return { id: 'a', function: { name: 'get_time', arguments: { city: 'Madrid' } } };
|
||||
if (shape === 'idless') return { name: 'get_time', args: { city: 'Madrid' } };
|
||||
return { id: 'a', name: 'get_', args: {} };
|
||||
})(),
|
||||
],
|
||||
},
|
||||
});
|
||||
if (shape === 'split') {
|
||||
for (const fragment of [
|
||||
{ id: 'a', name: 'get_', index: 0, args: '{"city":"Madrid"}' },
|
||||
{ index: 0, name: 'time' },
|
||||
]) {
|
||||
await h.on_run_step_delta.handle('on_run_step_delta', {
|
||||
id: 'complete',
|
||||
delta: { type: 'tool_calls', tool_calls: [fragment] },
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
try {
|
||||
await OpenAIChatCompletionController(req, res);
|
||||
if (streaming) {
|
||||
const frames = res.write.mock.calls
|
||||
.map(([frame]) => frame)
|
||||
.filter((frame) => frame !== 'data: [DONE]\n\n')
|
||||
.map((frame) => JSON.parse(frame.slice(6)));
|
||||
expect(
|
||||
frames
|
||||
.flatMap((frame) => frame.choices[0].delta.tool_calls ?? [])
|
||||
.map((call) => call.function?.arguments ?? '')
|
||||
.join(''),
|
||||
).toBe('{"city":"Madrid"}');
|
||||
expect(
|
||||
frames
|
||||
.flatMap((frame) => frame.choices[0].delta.tool_calls ?? [])
|
||||
.find((call) => call.id).function.name,
|
||||
).toBe('get_time');
|
||||
expect(frames.at(-1).choices[0].finish_reason).toBe('tool_calls');
|
||||
} else {
|
||||
expect(
|
||||
res.json.mock.calls[0][0].choices[0].message.tool_calls[0].function.arguments,
|
||||
).toBe('{"city":"Madrid"}');
|
||||
}
|
||||
} finally {
|
||||
for (const [name, implementation] of originals)
|
||||
api[name].mockImplementation(implementation);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it('records completed model usage even when terminal snapshot validation fails', async () => {
|
||||
mockProcessStream.mockImplementationOnce(async () => {
|
||||
const h = require('@librechat/api').createRun.mock.calls.at(-1)[0].customHandlers;
|
||||
await h.on_run_step.handle('on_run_step', {
|
||||
id: 'bad',
|
||||
stepDetails: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [{ id: 'a', function: { name: 'get_time', arguments: 'NOT-JSON' } }],
|
||||
},
|
||||
});
|
||||
});
|
||||
await OpenAIChatCompletionController(req, res);
|
||||
expect(mockRecordCollectedUsage).toHaveBeenCalledTimes(1);
|
||||
expect(require('@librechat/api').buildNonStreamingResponse).not.toHaveBeenCalled();
|
||||
expect(JSON.stringify(res.json.mock.calls)).not.toContain('NOT-JSON');
|
||||
});
|
||||
|
||||
it('enrolls, starts, and settles the remote execution lifecycle', async () => {
|
||||
await OpenAIChatCompletionController(req, res);
|
||||
|
||||
|
|
|
|||
|
|
@ -59,6 +59,10 @@ const {
|
|||
createToolExecuteHandler,
|
||||
createOwnedToolEndHandler,
|
||||
buildNonStreamingResponse,
|
||||
OpenAIRunStepHandler,
|
||||
OpenAIRunStepDeltaHandler,
|
||||
createOpenAIToolCallStream,
|
||||
completeOpenAIToolCalls,
|
||||
createOpenAIStreamTracker,
|
||||
resolveAgentScopedSkillIds,
|
||||
createOpenAIContentAggregator,
|
||||
|
|
@ -974,6 +978,19 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Shared by both run-step events, because the outward index a client keys
|
||||
* tool-call fragments by is allocated per call and belongs to neither
|
||||
* event alone.
|
||||
*/
|
||||
const toolCallStream = createOpenAIToolCallStream({
|
||||
signal: execution.signal,
|
||||
toolCalls: isStreaming ? tracker.toolCalls : aggregator.toolCalls,
|
||||
...(isStreaming && {
|
||||
emit: (delta) => writeSSE(res, createChunk(context, delta)),
|
||||
}),
|
||||
});
|
||||
|
||||
// Event handlers for OpenAI-compatible streaming
|
||||
const handlers = {
|
||||
// Text content streaming
|
||||
|
|
@ -1001,77 +1018,11 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
}
|
||||
}),
|
||||
|
||||
// Tool call initiation - streams id and name (from on_run_step)
|
||||
on_run_step: createHandler((data) => {
|
||||
const stepDetails = data?.stepDetails;
|
||||
if (stepDetails?.type === 'tool_calls' && stepDetails.tool_calls) {
|
||||
for (const tc of stepDetails.tool_calls) {
|
||||
const toolIndex = data.index ?? 0;
|
||||
const toolId = tc.id ?? '';
|
||||
const toolName = tc.name ?? '';
|
||||
const toolCall = {
|
||||
id: toolId,
|
||||
type: 'function',
|
||||
function: { name: toolName, arguments: '' },
|
||||
};
|
||||
|
||||
// Track tool call in tracker or aggregator
|
||||
if (isStreaming) {
|
||||
if (!tracker.toolCalls.has(toolIndex)) {
|
||||
tracker.toolCalls.set(toolIndex, toolCall);
|
||||
}
|
||||
// Stream initial tool call chunk (like OpenAI does)
|
||||
writeSSE(
|
||||
res,
|
||||
createChunk(context, {
|
||||
tool_calls: [{ index: toolIndex, ...toolCall }],
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
if (!aggregator.toolCalls.has(toolIndex)) {
|
||||
aggregator.toolCalls.set(toolIndex, toolCall);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}),
|
||||
// Tool call initiation - declares id and name (from on_run_step)
|
||||
on_run_step: new OpenAIRunStepHandler(toolCallStream),
|
||||
|
||||
// Tool call argument streaming (from on_run_step_delta)
|
||||
on_run_step_delta: createHandler((data) => {
|
||||
const delta = data?.delta;
|
||||
if (delta?.type === 'tool_calls' && delta.tool_calls) {
|
||||
for (const tc of delta.tool_calls) {
|
||||
const args = tc.args ?? '';
|
||||
if (!args) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const toolIndex = tc.index ?? 0;
|
||||
|
||||
// Update tool call arguments
|
||||
const targetMap = isStreaming ? tracker.toolCalls : aggregator.toolCalls;
|
||||
const tracked = targetMap.get(toolIndex);
|
||||
if (tracked) {
|
||||
tracked.function.arguments += args;
|
||||
}
|
||||
|
||||
// Stream argument delta (only for streaming)
|
||||
if (isStreaming) {
|
||||
writeSSE(
|
||||
res,
|
||||
createChunk(context, {
|
||||
tool_calls: [
|
||||
{
|
||||
index: toolIndex,
|
||||
function: { arguments: args },
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}),
|
||||
on_run_step_delta: new OpenAIRunStepDeltaHandler(toolCallStream),
|
||||
|
||||
// Usage tracking
|
||||
on_chat_model_end: {
|
||||
|
|
@ -1084,7 +1035,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
}
|
||||
},
|
||||
},
|
||||
on_run_step_completed: createHandler(),
|
||||
on_run_step_completed: new OpenAIRunStepHandler(toolCallStream),
|
||||
// Use proper ToolEndHandler for processing artifacts (images, file citations, code output)
|
||||
on_tool_end: createOwnedToolEndHandler(toolEndCallback, logger),
|
||||
on_chain_stream: createHandler(),
|
||||
|
|
@ -1188,47 +1139,49 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
version: 'v2',
|
||||
};
|
||||
|
||||
await run.processStream({ messages: formattedMessages }, config, {
|
||||
callbacks: {
|
||||
[Callback.TOOL_ERROR]: (graph, error, toolId) => {
|
||||
logger.error(`[OpenAI API] Tool Error "${toolId}"`, getSafeErrorMetadata(error));
|
||||
await completeOpenAIToolCalls(toolCallStream, async () => {
|
||||
await run.processStream({ messages: formattedMessages }, config, {
|
||||
callbacks: {
|
||||
[Callback.TOOL_ERROR]: (graph, error, toolId) => {
|
||||
logger.error(`[OpenAI API] Tool Error "${toolId}"`, getSafeErrorMetadata(error));
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
// Record token usage against balance
|
||||
const balanceConfig = getBalanceConfig(appConfig);
|
||||
const transactionsConfig = getTransactionsConfig(appConfig);
|
||||
execution.track(
|
||||
recordCollectedUsage(
|
||||
{
|
||||
spendTokens: db.spendTokens,
|
||||
spendStructuredTokens: db.spendStructuredTokens,
|
||||
pricing: {
|
||||
getMultiplier: db.getMultiplier,
|
||||
getCacheMultiplier: db.getCacheMultiplier,
|
||||
// Record token usage against balance
|
||||
const balanceConfig = getBalanceConfig(appConfig);
|
||||
const transactionsConfig = getTransactionsConfig(appConfig);
|
||||
execution.track(
|
||||
recordCollectedUsage(
|
||||
{
|
||||
spendTokens: db.spendTokens,
|
||||
spendStructuredTokens: db.spendStructuredTokens,
|
||||
pricing: {
|
||||
getMultiplier: db.getMultiplier,
|
||||
getCacheMultiplier: db.getCacheMultiplier,
|
||||
},
|
||||
bulkWriteOps: {
|
||||
insertMany: db.bulkInsertTransactions,
|
||||
updateBalance: db.updateBalance,
|
||||
},
|
||||
},
|
||||
bulkWriteOps: {
|
||||
insertMany: db.bulkInsertTransactions,
|
||||
updateBalance: db.updateBalance,
|
||||
{
|
||||
user: userId,
|
||||
conversationId,
|
||||
collectedUsage,
|
||||
context: 'message',
|
||||
messageId: responseId,
|
||||
balance: balanceConfig,
|
||||
transactions: transactionsConfig,
|
||||
model: primaryConfig.model || agent.model_parameters?.model,
|
||||
endpointTokenConfig: primaryConfig.endpointTokenConfig,
|
||||
resolveEndpointTokenConfig,
|
||||
},
|
||||
},
|
||||
{
|
||||
user: userId,
|
||||
conversationId,
|
||||
collectedUsage,
|
||||
context: 'message',
|
||||
messageId: responseId,
|
||||
balance: balanceConfig,
|
||||
transactions: transactionsConfig,
|
||||
model: primaryConfig.model || agent.model_parameters?.model,
|
||||
endpointTokenConfig: primaryConfig.endpointTokenConfig,
|
||||
resolveEndpointTokenConfig,
|
||||
},
|
||||
).catch((err) => {
|
||||
logger.error('[OpenAI API] Error recording usage:', getSafeErrorMetadata(err));
|
||||
}),
|
||||
);
|
||||
).catch((err) => {
|
||||
logger.error('[OpenAI API] Error recording usage:', getSafeErrorMetadata(err));
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
const usage = buildCompletionUsage(collectedUsage);
|
||||
|
||||
|
|
|
|||
65
packages/api/src/agents/openai/README.md
Normal file
65
packages/api/src/agents/openai/README.md
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
# Agents API response projection
|
||||
|
||||
This module projects an **agent run**, which can include several model invocations and server-executed tools, onto an OpenAI-compatible response. It is not a transparent proxy for a single model completion. Changing tool execution ownership or terminal-turn selection is a separate protocol decision.
|
||||
|
||||
## Owners and boundaries
|
||||
|
||||
```text
|
||||
provider adapter -> agents graph -> run-step declarations / argument deltas
|
||||
|
|
||||
tool-call projection
|
||||
/ \
|
||||
SSE chunks JSON accumulator
|
||||
\ /
|
||||
successful response completion
|
||||
```
|
||||
|
||||
The Express controller and packaged `createAgentChatCompletion` service both use the same tool-call projection in streaming and non-streaming mode. The service's content handlers also share text, reasoning and usage handling between modes. The controller retains its existing run lifecycle, usage recording, artifact writes and authorization boundaries.
|
||||
|
||||
## Identity
|
||||
|
||||
- A declaring run step owns call identity. Optional provider IDs are aliases, not a prerequisite. Declaration slots keep id-less calls distinct and make snapshot replay idempotent. Provider IDs can repeat in later steps and are not completion-wide identities.
|
||||
- Outward indexes are allocated once per call across the response. Raw ID substrings are assembled by provider index; shared prefixes such as `call_` are never cross-call aliases. Calls without IDs receive `call_<index>`; collisions receive a suffix. Internal tool execution IDs are not rewritten.
|
||||
- Argument fragments use provider indexes scoped to a model invocation. A graph delta may name the latest step while referring to an earlier parallel call. When available, the SDK's `getStepBaseKey` supplies invocation scope, including stream segments. A step's scope is retained so late fragments cannot acquire a newer segment's identity.
|
||||
- Callers without the graph object use metadata scope; callers without that metadata are isolated by step. This fallback cannot reconstruct a missing retry/segment identity.
|
||||
- Explicit and identified provider indexes take precedence over positional fallback. Positions are used only for multi-call declarations. An unbound singleton may adopt its first provider index; it does not absorb arbitrary later indexes.
|
||||
|
||||
## Argument representations
|
||||
|
||||
A run-step declaration can contain a **snapshot**: native `args` or OpenAI-shaped `function.arguments`. A run-step delta carries an **append-only fragment**. They must never be concatenated as though they were the same representation.
|
||||
|
||||
Both snapshot argument fields accept JSON **strings or objects**, using the shared `Agents.ToolCall` and `Agents.AgentFunctionToolCall` field types. Strings are retained verbatim; objects are serialized once. Nonempty argument fragments take precedence over all snapshots for that call, regardless of arrival order.
|
||||
|
||||
### Publication timing
|
||||
|
||||
Tool-call IDs and names can also arrive as substrings. There is no per-field completion seal in these events, and the OpenAI-compatible consumer freezes the name on its first chunk. Emitting `get_` and later correcting it to `get_weather` does not work.
|
||||
|
||||
Consequently, **tool-call chunks are published only at successful response completion**, after assembling identity and validating arguments. The first chunk contains the complete ID/name; the next carries the complete arguments at the same index. Text and reasoning continue streaming while the run executes. This deliberately sacrifices incremental tool-input display, rather than guessing which names are complete or breaking existing clients. Native JSON responses receive the same completed calls.
|
||||
|
||||
Name/ID/argument fragments are append-only and take precedence over the corresponding declaration snapshot. Snapshot replay is idempotent; raw fragments are not deduplicated because identical adjacent fragments can be legitimate and the events carry no sequence token. Split IDs require a provider index for unambiguous correlation. Exactly-once raw dispatch remains an upstream requirement.
|
||||
|
||||
## Terminal behavior
|
||||
|
||||
`completeOpenAIToolCalls` wraps execution and owns the success/failure boundary. The controller schedules completed-run usage recording before final validation.
|
||||
|
||||
| State/event | Behavior |
|
||||
| ---------------------------------------- | ---------------------------------------------------------------- |
|
||||
| Open + declaration | Allocate identity once; retain snapshot separately |
|
||||
| Open + identified fragment | Bind provider index and append/buffer fragment |
|
||||
| Successful execution | Assemble and validate all calls before publishing tool chunks |
|
||||
| Failure, abort or invalid terminal input | Discard pending calls; publish no tool-call chunks |
|
||||
| Finished | Repeated finish is harmless; later tool events are ignored |
|
||||
| Aborted | Later tool events are ignored; finish cannot reopen the response |
|
||||
|
||||
Missing names, missing arguments, malformed JSON or unattributable argument data fail terminal projection with bounded errors that omit provider content. A snapshot never repairs a truncated raw stream. Projection bookkeeping is response-local and released on finish/abort; only successfully validated calls are materialized for the response builder. Publication seals before calling the transport, so a transport error or reentrant finish cannot duplicate calls. Already transmitted frames cannot be retracted.
|
||||
|
||||
The existing finish-reason policy is preserved: a response with final text after server-executed tools remains `stop`. Historical tool calls are still exposed by this API. This patch does not make it safe to interpret every advertised call as a new instruction for the caller to execute.
|
||||
|
||||
## Verification and limits
|
||||
|
||||
- `toolCalls.contract.spec.ts`: shared field types, strings/objects, partial names and IDs, optional IDs, declaration order and transport reentrancy.
|
||||
- `toolCalls.spec.ts`: identity, representations, ordering, replay, malformed input, failure and abort.
|
||||
- `toolCalls.graph.spec.ts`: real SDK interleaving, complete-only chunks and stream segments.
|
||||
- `service.spec.ts` and the Express controller spec: both response modes, actual serializers, fallback flush and failure/usage ordering.
|
||||
|
||||
These tests do not run a live provider, real tool side effects, a distributed multi-agent deployment or a persisted resume. There is no new database, cache, authorization policy or migration. Code and built packages must deploy together. No cross-version event identity is invented when the upstream source omits it.
|
||||
|
|
@ -5,6 +5,8 @@
|
|||
* streaming format (SSE with chat.completion.chunk objects).
|
||||
*/
|
||||
import type { Response as ServerResponse } from 'express';
|
||||
import type { Agents } from 'librechat-data-provider';
|
||||
import type { Graph } from '@librechat/agents';
|
||||
import type {
|
||||
ChatCompletionChunkChoice,
|
||||
OpenAIResponseContext,
|
||||
|
|
@ -58,6 +60,9 @@ export function writeSSE(res: ServerResponse, data: ChatCompletionChunk | string
|
|||
* Only tracks what's needed for finish_reason and usage - doesn't store content.
|
||||
*/
|
||||
export interface OpenAIStreamTracker {
|
||||
/** Successful response completion, installed by the response handlers. */
|
||||
finishToolCalls?: () => void;
|
||||
abortToolCalls?: () => void;
|
||||
/** Whether any text content was emitted */
|
||||
hasText: boolean;
|
||||
/** Whether any reasoning content was emitted */
|
||||
|
|
@ -105,6 +110,9 @@ export function createOpenAIStreamTracker(): OpenAIStreamTracker {
|
|||
* Uses arrays for O(n) text accumulation instead of O(n²) string concatenation.
|
||||
*/
|
||||
export interface OpenAIContentAggregator {
|
||||
/** Successful response completion, installed by the response handlers. */
|
||||
finishToolCalls?: () => void;
|
||||
abortToolCalls?: () => void;
|
||||
/** Accumulated text chunks */
|
||||
textChunks: string[];
|
||||
/** Accumulated reasoning/thinking chunks */
|
||||
|
|
@ -154,6 +162,7 @@ export function createOpenAIContentAggregator(): OpenAIContentAggregator {
|
|||
* Handler configuration for OpenAI streaming
|
||||
*/
|
||||
export interface OpenAIStreamHandlerConfig {
|
||||
signal?: AbortSignal;
|
||||
res: ServerResponse;
|
||||
context: OpenAIResponseContext;
|
||||
tracker: OpenAIStreamTracker;
|
||||
|
|
@ -187,22 +196,44 @@ export const StepTypes = {
|
|||
*/
|
||||
export interface MessageDeltaData {
|
||||
id?: string;
|
||||
content?: Array<{ type: string; text?: string }>;
|
||||
content?: Array<{ type: string; text?: string; think?: string }>;
|
||||
delta?: { content?: MessageDeltaData['content'] };
|
||||
}
|
||||
|
||||
/**
|
||||
* One tool-call fragment of a run step delta, as `@librechat/agents` emits it:
|
||||
* `id`, `name`, and `args` can all be substrings. `index` identifies their
|
||||
* provider stream within the current model invocation. Reuse the shared chunk
|
||||
* contract and accept equivalent function-shaped name/argument fragments.
|
||||
*/
|
||||
export type RunStepToolCallChunk = Agents.ToolCallChunk & {
|
||||
function?: { name?: string; arguments?: string };
|
||||
};
|
||||
|
||||
export interface RunStepDeltaData {
|
||||
/** The run step these fragments belong to. */
|
||||
id?: string;
|
||||
delta?: {
|
||||
type?: string;
|
||||
tool_calls?: Array<{
|
||||
index?: number;
|
||||
id?: string;
|
||||
type?: string;
|
||||
function?: {
|
||||
name?: string;
|
||||
arguments?: string;
|
||||
};
|
||||
}>;
|
||||
tool_calls?: RunStepToolCallChunk[];
|
||||
};
|
||||
}
|
||||
|
||||
/** A tool call as the run step that opened it declares it. */
|
||||
export type RunStepToolCall = Partial<Pick<Agents.ToolCall, 'id' | 'name' | 'args'>> & {
|
||||
index?: number;
|
||||
type?: string;
|
||||
function?: Partial<Agents.AgentFunctionToolCall['function']>;
|
||||
};
|
||||
|
||||
export interface RunStepData {
|
||||
/** The run step's own id, shared with every delta dispatched for it. */
|
||||
id?: string;
|
||||
/** The step's position in the response content, not a tool-call index. */
|
||||
index?: number;
|
||||
stepDetails?: {
|
||||
type?: string;
|
||||
tool_calls?: RunStepToolCall[];
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -240,20 +271,372 @@ export interface EventHandler {
|
|||
): void | Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Projects the graph's tool-call events onto the OpenAI wire format.
|
||||
*
|
||||
* A client accumulates `delta.tool_calls` by `index`: the first chunk at an
|
||||
* index declares the call and must carry `id` and `function.name`, and every
|
||||
* later chunk at that index belongs to that same call. Neither index the graph
|
||||
* reports can serve as that key. A run step's `index` is its position in the
|
||||
* response's content, so a tool call that follows text does not start at zero,
|
||||
* and consecutive steps each carry a single-element `tool_calls` array rather
|
||||
* than one array holding every call. A fragment's `index` is the provider's
|
||||
* content-block index, which restarts at zero on each model invocation of the
|
||||
* run, so two different calls can share it.
|
||||
*
|
||||
* Outward indexes identify calls within declaring steps, using declaration slots
|
||||
* when IDs are absent. Provider indexes correlate raw fragments within an
|
||||
* invocation. IDs and names are assembled before publication: a client can freeze
|
||||
* both on its first chunk, and the graph provides no per-field completion seal.
|
||||
*/
|
||||
export interface OpenAIToolCallStreamConfig {
|
||||
/** Completed tool calls, populated at successful finish and keyed by outward index. */
|
||||
toolCalls: Map<number, ToolCall>;
|
||||
/** Emits complete identity/argument chunks at finish. Omitted for non-streaming. */
|
||||
emit?: (delta: ChatCompletionChunkChoice['delta']) => void;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface OpenAIToolCallStream {
|
||||
/** Declares the tool calls a run step opened. */
|
||||
onRunStep: (data: RunStepData, metadata?: Record<string, unknown>, graph?: ToolCallGraph) => void;
|
||||
/** Accumulates the name and argument fragments streamed for a run step. */
|
||||
onRunStepDelta: (
|
||||
data: RunStepDeltaData,
|
||||
metadata?: Record<string, unknown>,
|
||||
graph?: ToolCallGraph,
|
||||
) => void;
|
||||
/** Publish assembled, validated calls once after success, before DONE/JSON. */
|
||||
finish: () => void;
|
||||
/** Discard pending calls on failure and reject further writes. */
|
||||
abort: () => void;
|
||||
}
|
||||
|
||||
type ToolCallGraph = Pick<Graph, 'getStepBaseKey'>;
|
||||
|
||||
interface ProjectedToolCall {
|
||||
index: number;
|
||||
snapshotId?: string;
|
||||
snapshotName?: string;
|
||||
snapshotArgs?: string;
|
||||
idFragments: string[];
|
||||
nameFragments: string[];
|
||||
argFragments: string[];
|
||||
}
|
||||
|
||||
interface ToolCallStep {
|
||||
byId: Map<string, ProjectedToolCall>;
|
||||
byDeclaration: Map<string, ProjectedToolCall>;
|
||||
byProviderIndex: Map<number, ProjectedToolCall>;
|
||||
/** Declaration position is a fallback, never stronger than a provider binding. */
|
||||
byPosition: Map<number, ProjectedToolCall>;
|
||||
calls: Set<ProjectedToolCall>;
|
||||
}
|
||||
|
||||
export function createOpenAIToolCallStream(
|
||||
config: OpenAIToolCallStreamConfig,
|
||||
): OpenAIToolCallStream {
|
||||
const { toolCalls, emit, signal } = config;
|
||||
let phase: 'open' | 'finished' | 'aborted' = 'open';
|
||||
let unattributableArguments = false;
|
||||
const calls: ProjectedToolCall[] = [];
|
||||
const steps = new Map<string, ToolCallStep>();
|
||||
const invocations = new Map<string, Map<number, ProjectedToolCall>>();
|
||||
const stepScopes = new Map<string, string>();
|
||||
|
||||
/** The SDK can send earlier parallel calls under the latest step ID. Its
|
||||
* invocation key owns segment/checkpoint transitions; cache it at declaration
|
||||
* so late events never borrow a new segment. Step isolation is the fallback. */
|
||||
const getBindings = (
|
||||
stepId: string,
|
||||
metadata?: Record<string, unknown>,
|
||||
graph?: ToolCallGraph,
|
||||
): Map<number, ProjectedToolCall> => {
|
||||
let scope = stepScopes.get(stepId);
|
||||
if (scope === undefined) {
|
||||
if (graph && metadata) {
|
||||
scope = graph.getStepBaseKey(metadata);
|
||||
} else if (
|
||||
typeof metadata?.langgraph_node === 'string' &&
|
||||
typeof metadata.langgraph_step === 'number'
|
||||
) {
|
||||
scope = JSON.stringify([
|
||||
metadata.run_id ?? '',
|
||||
metadata.thread_id ?? '',
|
||||
metadata.langgraph_node,
|
||||
metadata.langgraph_step,
|
||||
metadata.langgraph_checkpoint_ns ?? metadata.checkpoint_ns ?? '',
|
||||
]);
|
||||
} else {
|
||||
scope = JSON.stringify(['step', stepId]);
|
||||
}
|
||||
stepScopes.set(stepId, scope);
|
||||
}
|
||||
let bindings = invocations.get(scope);
|
||||
if (bindings === undefined) {
|
||||
bindings = new Map();
|
||||
invocations.set(scope, bindings);
|
||||
}
|
||||
return bindings;
|
||||
};
|
||||
|
||||
const getStep = (stepId: string): ToolCallStep => {
|
||||
let step = steps.get(stepId);
|
||||
if (step === undefined) {
|
||||
step = {
|
||||
byId: new Map(),
|
||||
byDeclaration: new Map(),
|
||||
byProviderIndex: new Map(),
|
||||
byPosition: new Map(),
|
||||
calls: new Set(),
|
||||
};
|
||||
steps.set(stepId, step);
|
||||
}
|
||||
return step;
|
||||
};
|
||||
|
||||
const allocate = (step: ToolCallStep): ProjectedToolCall => {
|
||||
const call: ProjectedToolCall = {
|
||||
index: calls.length,
|
||||
idFragments: [],
|
||||
nameFragments: [],
|
||||
argFragments: [],
|
||||
};
|
||||
calls.push(call);
|
||||
step.calls.add(call);
|
||||
return call;
|
||||
};
|
||||
|
||||
const abort = (): void => {
|
||||
if (phase !== 'finished') {
|
||||
phase = 'aborted';
|
||||
}
|
||||
calls.length = 0;
|
||||
steps.clear();
|
||||
stepScopes.clear();
|
||||
invocations.clear();
|
||||
};
|
||||
|
||||
const writable = (): boolean => {
|
||||
if (signal?.aborted) {
|
||||
abort();
|
||||
}
|
||||
return phase === 'open';
|
||||
};
|
||||
|
||||
return {
|
||||
abort,
|
||||
finish: () => {
|
||||
if (phase === 'finished') {
|
||||
return;
|
||||
}
|
||||
if (!writable()) {
|
||||
const error = new Error('Agent response aborted');
|
||||
error.name = 'AbortError';
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
if (unattributableArguments) {
|
||||
throw new Error('Unattributable tool call arguments in agent response');
|
||||
}
|
||||
const ready: ToolCall[] = [];
|
||||
const ids = new Set<string>();
|
||||
/** There is no per-field name/ID seal in the graph event contract. An
|
||||
* OpenAI-compatible client freezes the name on the first outward chunk.
|
||||
* Validate and assemble every call before publishing any: text still
|
||||
* streams, but tool chunks wait for successful response completion. */
|
||||
for (const call of calls) {
|
||||
const name = call.nameFragments.join('') || call.snapshotName;
|
||||
const args = call.argFragments.length ? call.argFragments.join('') : call.snapshotArgs;
|
||||
if (!name || args === undefined || args === '') {
|
||||
throw new Error('Incomplete tool call in agent response');
|
||||
}
|
||||
try {
|
||||
JSON.parse(args);
|
||||
} catch {
|
||||
throw new Error('Invalid tool call arguments in agent response');
|
||||
}
|
||||
let id = call.idFragments.join('') || call.snapshotId || `call_${call.index}`;
|
||||
while (ids.has(id)) {
|
||||
id = `${id}_${call.index}`;
|
||||
}
|
||||
ids.add(id);
|
||||
ready.push({ id, type: 'function', function: { name, arguments: args } });
|
||||
}
|
||||
for (const [index, call] of ready.entries()) {
|
||||
toolCalls.set(index, call);
|
||||
}
|
||||
/** Seal before invoking transport callbacks, including reentrant callers.
|
||||
* An emission failure cannot be retried into duplicate tool requests. */
|
||||
phase = 'finished';
|
||||
for (const [index, call] of ready.entries()) {
|
||||
emit?.({
|
||||
tool_calls: [
|
||||
{
|
||||
index,
|
||||
id: call.id,
|
||||
type: 'function',
|
||||
function: { name: call.function.name, arguments: '' },
|
||||
},
|
||||
],
|
||||
});
|
||||
emit?.({ tool_calls: [{ index, function: { arguments: call.function.arguments } }] });
|
||||
}
|
||||
} finally {
|
||||
abort();
|
||||
}
|
||||
},
|
||||
onRunStep: (data, metadata, graph) => {
|
||||
if (!writable()) {
|
||||
return;
|
||||
}
|
||||
const details = data?.stepDetails;
|
||||
if (details?.type !== StepTypes.TOOL_CALLS || !Array.isArray(details.tool_calls)) {
|
||||
return;
|
||||
}
|
||||
const step = getStep(data.id ?? '');
|
||||
const bindings = getBindings(data.id ?? '', metadata, graph);
|
||||
for (const [position, toolCall] of details.tool_calls.entries()) {
|
||||
const key =
|
||||
toolCall.index === undefined ? `position:${position}` : `index:${toolCall.index}`;
|
||||
let call =
|
||||
(toolCall.id ? step.byId.get(toolCall.id) : undefined) ?? step.byDeclaration.get(key);
|
||||
if (call === undefined && toolCall.index !== undefined) {
|
||||
call = step.byProviderIndex.get(toolCall.index);
|
||||
}
|
||||
/** A declaration can follow its raw chunks. Positions are meaningful
|
||||
* only inside this declaring step, never across the whole response. */
|
||||
if (call === undefined) {
|
||||
call = step.byProviderIndex.get(position);
|
||||
if (
|
||||
call === undefined &&
|
||||
step.byDeclaration.size === 0 &&
|
||||
details.tool_calls.length === 1 &&
|
||||
step.calls.size === 1
|
||||
) {
|
||||
call = step.calls.values().next().value;
|
||||
}
|
||||
}
|
||||
call ??= allocate(step);
|
||||
step.byDeclaration.set(key, call);
|
||||
if (toolCall.id) {
|
||||
call.snapshotId = toolCall.id;
|
||||
step.byId.set(toolCall.id, call);
|
||||
}
|
||||
call.snapshotName = toolCall.name ?? toolCall.function?.name ?? call.snapshotName;
|
||||
const args = toolCall.function?.arguments ?? toolCall.args;
|
||||
if (args !== undefined) {
|
||||
/** Both public snapshot fields accept raw JSON strings or objects. */
|
||||
try {
|
||||
call.snapshotArgs = typeof args === 'string' ? args : JSON.stringify(args);
|
||||
} catch {
|
||||
throw new Error('Invalid tool call arguments in agent response');
|
||||
}
|
||||
}
|
||||
if (toolCall.index !== undefined) {
|
||||
step.byProviderIndex.set(toolCall.index, call);
|
||||
bindings.set(toolCall.index, call);
|
||||
} else if (details.tool_calls.length > 1) {
|
||||
step.byPosition.set(position, call);
|
||||
}
|
||||
}
|
||||
},
|
||||
onRunStepDelta: (data, metadata, graph) => {
|
||||
if (!writable()) {
|
||||
return;
|
||||
}
|
||||
const delta = data?.delta;
|
||||
if (delta?.type !== StepTypes.TOOL_CALLS || !Array.isArray(delta.tool_calls)) {
|
||||
return;
|
||||
}
|
||||
const step = getStep(data.id ?? '');
|
||||
const bindings = getBindings(data.id ?? '', metadata, graph);
|
||||
for (const fragment of delta.tool_calls) {
|
||||
const name = fragment.name ?? fragment.function?.name ?? '';
|
||||
const args = fragment.args ?? fragment.function?.arguments ?? '';
|
||||
/** An index identifies an already-bound raw stream even if this event
|
||||
* carries only a substring of its ID. Never key identity by that suffix. */
|
||||
let call = fragment.index === undefined ? undefined : bindings.get(fragment.index);
|
||||
call ??= fragment.id ? step.byId.get(fragment.id) : undefined;
|
||||
call ??= fragment.index === undefined ? undefined : step.byPosition.get(fragment.index);
|
||||
if (
|
||||
call === undefined &&
|
||||
step.calls.size === 1 &&
|
||||
(fragment.index === undefined || step.byProviderIndex.size === 0)
|
||||
) {
|
||||
const singleton = step.calls.values().next().value;
|
||||
/** Without an index, a different full ID starts another raw call, not
|
||||
* another substring of the old call. Split IDs need their index. */
|
||||
if (
|
||||
!fragment.id ||
|
||||
fragment.index !== undefined ||
|
||||
(singleton?.snapshotId === undefined && singleton?.idFragments.length === 0)
|
||||
) {
|
||||
call = singleton;
|
||||
}
|
||||
}
|
||||
if (call === undefined && (fragment.id || name)) {
|
||||
call = allocate(step);
|
||||
}
|
||||
if (call === undefined) {
|
||||
unattributableArguments ||= !!args;
|
||||
continue;
|
||||
}
|
||||
if (fragment.index !== undefined) {
|
||||
step.byProviderIndex.set(fragment.index, call);
|
||||
bindings.set(fragment.index, call);
|
||||
}
|
||||
if (fragment.id) {
|
||||
call.idFragments.push(fragment.id);
|
||||
/** Partial IDs are not aliases across indexed calls: two parallel
|
||||
* calls can both begin with `call_`. Their index is authoritative. */
|
||||
if (fragment.index === undefined) {
|
||||
step.byId.set(call.idFragments.join(''), call);
|
||||
}
|
||||
}
|
||||
if (name) {
|
||||
call.nameFragments.push(name);
|
||||
}
|
||||
if (args) {
|
||||
call.argFragments.push(args);
|
||||
}
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Success publishes validated calls; failure/abort never does. Both hosts
|
||||
* use this boundary so late provider events cannot mutate a settled response. */
|
||||
export async function completeOpenAIToolCalls(
|
||||
lifecycle: Pick<OpenAIToolCallStream, 'finish' | 'abort'>,
|
||||
execute: () => Promise<void>,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await execute();
|
||||
lifecycle.finish();
|
||||
} finally {
|
||||
lifecycle.abort();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handler for message delta events - streams text content
|
||||
*/
|
||||
export class OpenAIMessageDeltaHandler implements EventHandler {
|
||||
constructor(private config: OpenAIStreamHandlerConfig) {}
|
||||
constructor(private config: OpenAIContentHandlerConfig) {}
|
||||
|
||||
handle(_event: string, data: MessageDeltaData): void {
|
||||
const content = data?.content;
|
||||
const content = data?.delta?.content ?? data?.content;
|
||||
if (!content || !Array.isArray(content)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const part of content) {
|
||||
if (part.type === 'text' && part.text) {
|
||||
if ('aggregator' in this.config) {
|
||||
this.config.aggregator.addText(part.text);
|
||||
continue;
|
||||
}
|
||||
this.config.tracker.addText();
|
||||
const chunk = createChunk(this.config.context, { content: part.text });
|
||||
writeSSE(this.config.res, chunk);
|
||||
|
|
@ -263,86 +646,35 @@ export class OpenAIMessageDeltaHandler implements EventHandler {
|
|||
}
|
||||
|
||||
/**
|
||||
* Handler for run step delta events - streams tool calls
|
||||
* Handler for run step delta events - accumulates streamed tool call fragments
|
||||
*/
|
||||
export class OpenAIRunStepDeltaHandler implements EventHandler {
|
||||
constructor(private config: OpenAIStreamHandlerConfig) {}
|
||||
constructor(private toolCallStream: OpenAIToolCallStream) {}
|
||||
|
||||
handle(_event: string, data: RunStepDeltaData): void {
|
||||
const delta = data?.delta;
|
||||
if (!delta || delta.type !== StepTypes.TOOL_CALLS) {
|
||||
return;
|
||||
}
|
||||
|
||||
const toolCalls = delta.tool_calls;
|
||||
if (!toolCalls || !Array.isArray(toolCalls)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const tc of toolCalls) {
|
||||
if (tc.index === undefined) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Initialize tool call in tracker if needed
|
||||
let trackedTc = this.config.tracker.toolCalls.get(tc.index);
|
||||
if (!trackedTc && tc.id) {
|
||||
trackedTc = {
|
||||
id: tc.id,
|
||||
type: 'function',
|
||||
function: {
|
||||
name: '',
|
||||
arguments: '',
|
||||
},
|
||||
};
|
||||
this.config.tracker.toolCalls.set(tc.index, trackedTc);
|
||||
}
|
||||
|
||||
// Build the streaming delta
|
||||
const streamDelta: ChatCompletionChunkChoice['delta'] = {
|
||||
tool_calls: [
|
||||
{
|
||||
index: tc.index,
|
||||
...(tc.id && { id: tc.id }),
|
||||
...(tc.type && { type: tc.type as 'function' }),
|
||||
...(tc.function && {
|
||||
function: {
|
||||
...(tc.function.name && { name: tc.function.name }),
|
||||
...(tc.function.arguments && { arguments: tc.function.arguments }),
|
||||
},
|
||||
}),
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
// Update tracked tool call
|
||||
if (trackedTc) {
|
||||
if (tc.function?.name) {
|
||||
trackedTc.function.name += tc.function.name;
|
||||
}
|
||||
if (tc.function?.arguments) {
|
||||
trackedTc.function.arguments += tc.function.arguments;
|
||||
}
|
||||
}
|
||||
|
||||
const chunk = createChunk(this.config.context, streamDelta);
|
||||
writeSSE(this.config.res, chunk);
|
||||
}
|
||||
handle(
|
||||
_event: string,
|
||||
data: RunStepDeltaData,
|
||||
metadata?: Record<string, unknown>,
|
||||
graph?: ToolCallGraph,
|
||||
): void {
|
||||
this.toolCallStream.onRunStepDelta(data, metadata, graph);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handler for run step events - sends initial tool call info
|
||||
* Handler for run step events - declares a tool call's id and name at its
|
||||
* outward index, before any argument fragment references that index
|
||||
*/
|
||||
export class OpenAIRunStepHandler implements EventHandler {
|
||||
constructor(private config: OpenAIStreamHandlerConfig) {}
|
||||
constructor(private toolCallStream: OpenAIToolCallStream) {}
|
||||
|
||||
handle(_event: string, data: { stepDetails?: { type?: string } }): void {
|
||||
// Run step events are primarily for LibreChat UI, we use deltas for streaming
|
||||
// This handler is a no-op for OpenAI format
|
||||
if (data?.stepDetails?.type === StepTypes.TOOL_CALLS) {
|
||||
// Tool calls will be streamed via delta events
|
||||
}
|
||||
handle(
|
||||
_event: string,
|
||||
data: RunStepData,
|
||||
metadata?: Record<string, unknown>,
|
||||
graph?: ToolCallGraph,
|
||||
): void {
|
||||
this.toolCallStream.onRunStep(data, metadata, graph);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -350,7 +682,7 @@ export class OpenAIRunStepHandler implements EventHandler {
|
|||
* Handler for model end events - captures usage
|
||||
*/
|
||||
export class OpenAIModelEndHandler implements EventHandler {
|
||||
constructor(private config: OpenAIStreamHandlerConfig) {}
|
||||
constructor(private config: OpenAIContentHandlerConfig) {}
|
||||
|
||||
handle(_event: string, data: ModelEndData): void {
|
||||
const usage = data?.output?.usage_metadata;
|
||||
|
|
@ -358,9 +690,10 @@ export class OpenAIModelEndHandler implements EventHandler {
|
|||
return;
|
||||
}
|
||||
|
||||
this.config.tracker.usage.promptTokens += usage.input_tokens ?? 0;
|
||||
this.config.tracker.usage.completionTokens += usage.output_tokens ?? 0;
|
||||
this.config.tracker.usage.reasoningTokens +=
|
||||
const target = 'aggregator' in this.config ? this.config.aggregator : this.config.tracker;
|
||||
target.usage.promptTokens += usage.input_tokens ?? 0;
|
||||
target.usage.completionTokens += usage.output_tokens ?? 0;
|
||||
target.usage.reasoningTokens +=
|
||||
usage.output_token_details?.reasoning ?? usage.output_token_details?.reasoning_tokens ?? 0;
|
||||
}
|
||||
}
|
||||
|
|
@ -389,21 +722,26 @@ export class OpenAIToolEndHandler implements EventHandler {
|
|||
* Streams reasoning/thinking content using the `delta.reasoning` field (OpenRouter convention).
|
||||
*/
|
||||
export class OpenAIReasoningDeltaHandler implements EventHandler {
|
||||
constructor(private config: OpenAIStreamHandlerConfig) {}
|
||||
constructor(private config: OpenAIContentHandlerConfig) {}
|
||||
|
||||
handle(_event: string, data: MessageDeltaData): void {
|
||||
const content = data?.content;
|
||||
const content = data?.delta?.content ?? data?.content;
|
||||
if (!content || !Array.isArray(content)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const part of content) {
|
||||
if (part.type === 'text' && part.text) {
|
||||
if ((part.type === 'text' || part.type === 'think') && (part.think || part.text)) {
|
||||
const text = part.think || part.text!;
|
||||
if ('aggregator' in this.config) {
|
||||
this.config.aggregator.addReasoning(text);
|
||||
continue;
|
||||
}
|
||||
// Mark that reasoning was emitted
|
||||
this.config.tracker.addReasoning();
|
||||
|
||||
// Stream as delta.reasoning (OpenRouter convention)
|
||||
const chunk = createChunk(this.config.context, { reasoning: part.text });
|
||||
const chunk = createChunk(this.config.context, { reasoning: text });
|
||||
writeSSE(this.config.res, chunk);
|
||||
}
|
||||
}
|
||||
|
|
@ -413,15 +751,34 @@ export class OpenAIReasoningDeltaHandler implements EventHandler {
|
|||
/**
|
||||
* Create all handlers for OpenAI streaming format
|
||||
*/
|
||||
export interface OpenAIAggregationHandlerConfig {
|
||||
aggregator: OpenAIContentAggregator;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
type OpenAIContentHandlerConfig = OpenAIStreamHandlerConfig | OpenAIAggregationHandlerConfig;
|
||||
|
||||
export function createOpenAIHandlers(
|
||||
config: OpenAIStreamHandlerConfig,
|
||||
config: OpenAIContentHandlerConfig,
|
||||
toolExecuteOptions?: ToolExecuteOptions,
|
||||
): Record<string, EventHandler> {
|
||||
/** One projection across both events, so a call keeps a single outward index. */
|
||||
const toolCallStream = createOpenAIToolCallStream({
|
||||
signal: config.signal,
|
||||
toolCalls: 'aggregator' in config ? config.aggregator.toolCalls : config.tracker.toolCalls,
|
||||
emit:
|
||||
'aggregator' in config
|
||||
? undefined
|
||||
: (delta) => writeSSE(config.res, createChunk(config.context, delta)),
|
||||
});
|
||||
const target = 'aggregator' in config ? config.aggregator : config.tracker;
|
||||
target.finishToolCalls = toolCallStream.finish;
|
||||
target.abortToolCalls = toolCallStream.abort;
|
||||
const handlers: Record<string, EventHandler> = {
|
||||
[GraphEvents.ON_MESSAGE_DELTA]: new OpenAIMessageDeltaHandler(config),
|
||||
[GraphEvents.ON_RUN_STEP_DELTA]: new OpenAIRunStepDeltaHandler(config),
|
||||
[GraphEvents.ON_RUN_STEP]: new OpenAIRunStepHandler(config),
|
||||
[GraphEvents.ON_RUN_STEP_COMPLETED]: new OpenAIRunStepHandler(config),
|
||||
[GraphEvents.ON_RUN_STEP_DELTA]: new OpenAIRunStepDeltaHandler(toolCallStream),
|
||||
[GraphEvents.ON_RUN_STEP]: new OpenAIRunStepHandler(toolCallStream),
|
||||
[GraphEvents.ON_RUN_STEP_COMPLETED]: new OpenAIRunStepHandler(toolCallStream),
|
||||
[GraphEvents.CHAT_MODEL_END]: new OpenAIModelEndHandler(config),
|
||||
[GraphEvents.CHAT_MODEL_STREAM]: new OpenAIChatModelStreamHandler(),
|
||||
[GraphEvents.TOOL_END]: new OpenAIToolEndHandler(),
|
||||
|
|
@ -444,6 +801,7 @@ export function sendFinalChunk(
|
|||
usageOverride?: CompletionUsage,
|
||||
): void {
|
||||
const { res, context, tracker } = config;
|
||||
tracker.finishToolCalls?.();
|
||||
|
||||
// Determine finish reason based on content
|
||||
let reason = finishReason;
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { GraphEvents } from '@librechat/agents';
|
|||
import { ErrorTypes, Permissions, PermissionTypes } from 'librechat-data-provider';
|
||||
import type { FiltersConfig } from 'librechat-data-provider';
|
||||
import type { ChatCompletionDependencies } from './service';
|
||||
import type { ChatCompletionChunk } from './types';
|
||||
import { createAgentChatCompletion } from './service';
|
||||
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
|
|
@ -104,6 +105,143 @@ describe('createAgentChatCompletion - MCP permission user propagation', () => {
|
|||
};
|
||||
});
|
||||
|
||||
it.each(
|
||||
[true, false].flatMap((stream) =>
|
||||
['wire-string', 'wire-object', 'native-string', 'idless'].map(
|
||||
(shape) => [stream, shape] as const,
|
||||
),
|
||||
),
|
||||
)(
|
||||
'retains complete and mixed snapshots through the service (stream=%s, shape=%s)',
|
||||
async (stream, shape) => {
|
||||
const req = createMockReq(
|
||||
{ id: 'user' },
|
||||
{ model: 'agent_test', messages: [{ role: 'user', content: 'hi' }], stream },
|
||||
);
|
||||
const res = createMockRes();
|
||||
processStream.mockImplementationOnce(async () => {
|
||||
const { customHandlers: h } = createRun.mock.calls[0][0] as Parameters<
|
||||
NonNullable<ChatCompletionDependencies['createRun']>
|
||||
>[0];
|
||||
const meta = { langgraph_node: 'agent=test', langgraph_step: 1 };
|
||||
await h.on_run_step.handle(
|
||||
'on_run_step',
|
||||
{
|
||||
id: 'step',
|
||||
stepDetails: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [
|
||||
(() => {
|
||||
if (shape === 'idless')
|
||||
return { name: 'get_time', args: { city: 'Madrid' }, index: 0 };
|
||||
if (shape === 'native-string')
|
||||
return { id: 'a', name: 'get_time', args: '{"city":"Madrid"}', index: 0 };
|
||||
return {
|
||||
id: 'a',
|
||||
function: {
|
||||
name: 'get_time',
|
||||
arguments: shape === 'wire-object' ? { city: 'Madrid' } : '{"city":"Madrid"}',
|
||||
},
|
||||
index: 0,
|
||||
};
|
||||
})(),
|
||||
{ id: 'b', name: 'get_time', args: {}, index: 1 },
|
||||
],
|
||||
},
|
||||
},
|
||||
meta,
|
||||
);
|
||||
await h.on_run_step_delta.handle(
|
||||
'on_run_step_delta',
|
||||
{
|
||||
id: 'step',
|
||||
delta: { type: 'tool_calls', tool_calls: [{ index: 1, args: '{"city":"Paris"}' }] },
|
||||
},
|
||||
meta,
|
||||
);
|
||||
await h.on_message_delta.handle('on_message_delta', {
|
||||
delta: { content: [{ type: 'text', text: 'Finished.' }] },
|
||||
});
|
||||
});
|
||||
await createAgentChatCompletion(req, res, deps);
|
||||
if (stream) {
|
||||
const frames = (res.write as jest.Mock).mock.calls
|
||||
.map(([frame]: [string]) => frame)
|
||||
.filter((frame) => frame !== 'data: [DONE]\n\n');
|
||||
const chunks: ChatCompletionChunk[] = frames.map((frame) => JSON.parse(frame.slice(6)));
|
||||
const args = new Map<number, string>();
|
||||
for (const chunk of chunks)
|
||||
for (const call of chunk.choices[0].delta.tool_calls ?? [])
|
||||
args.set(call.index, (args.get(call.index) ?? '') + (call.function?.arguments ?? ''));
|
||||
expect(args.get(0)).toBe('{"city":"Madrid"}');
|
||||
expect(args.get(1)).toBe('{"city":"Paris"}');
|
||||
expect(chunks[chunks.length - 1].choices[0].finish_reason).toBe('stop');
|
||||
} else {
|
||||
expect(getResponseMock(res, 'json')).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
choices: [
|
||||
expect.objectContaining({
|
||||
finish_reason: 'stop',
|
||||
message: expect.objectContaining({
|
||||
content: 'Finished.',
|
||||
tool_calls: [
|
||||
{
|
||||
id: shape === 'idless' ? 'call_0' : 'a',
|
||||
type: 'function',
|
||||
function: { name: 'get_time', arguments: '{"city":"Madrid"}' },
|
||||
},
|
||||
{
|
||||
id: 'b',
|
||||
type: 'function',
|
||||
function: { name: 'get_time', arguments: '{"city":"Paris"}' },
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
],
|
||||
}),
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each([true, false])(
|
||||
'does not publish fallback arguments after provider failure (stream=%s)',
|
||||
async (stream) => {
|
||||
const req = createMockReq(
|
||||
{ id: 'user' },
|
||||
{ model: 'agent_test', messages: [{ role: 'user', content: 'hi' }], stream },
|
||||
);
|
||||
const res = createMockRes();
|
||||
processStream.mockImplementationOnce(async () => {
|
||||
const { customHandlers: h } = createRun.mock.calls[0][0] as Parameters<
|
||||
NonNullable<ChatCompletionDependencies['createRun']>
|
||||
>[0];
|
||||
await h.on_run_step.handle('on_run_step', {
|
||||
id: 'step',
|
||||
stepDetails: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [
|
||||
{ id: 'a', function: { name: 'get_time', arguments: '{"city":"DO_NOT_FLUSH"}' } },
|
||||
],
|
||||
},
|
||||
});
|
||||
throw new Error('provider failed');
|
||||
});
|
||||
await createAgentChatCompletion(req, res, deps);
|
||||
expect(JSON.stringify((res.write as jest.Mock).mock.calls)).not.toContain('DO_NOT_FLUSH');
|
||||
expect(JSON.stringify(getResponseMock(res, 'json').mock.calls)).not.toContain('DO_NOT_FLUSH');
|
||||
const { customHandlers: h } = createRun.mock.calls[0][0] as Parameters<
|
||||
NonNullable<ChatCompletionDependencies['createRun']>
|
||||
>[0];
|
||||
await h.on_run_step_delta.handle('on_run_step_delta', {
|
||||
id: 'step',
|
||||
delta: { type: 'tool_calls', tool_calls: [{ id: 'a', args: '{"late":true}' }] },
|
||||
});
|
||||
expect(JSON.stringify((res.write as jest.Mock).mock.calls)).not.toContain('late');
|
||||
},
|
||||
);
|
||||
|
||||
it('forwards the role-bearing safe user to createRun and configurable.user', async () => {
|
||||
const req = createMockReq({
|
||||
id: 'user-123',
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ import {
|
|||
createOpenAIContentAggregator,
|
||||
createOpenAIStreamTracker,
|
||||
createOpenAIHandlers,
|
||||
completeOpenAIToolCalls,
|
||||
sendFinalChunk,
|
||||
createChunk,
|
||||
writeSSE,
|
||||
|
|
@ -833,19 +834,21 @@ export async function createAgentChatCompletion(
|
|||
: null;
|
||||
|
||||
// Create event handlers
|
||||
const eventHandlers =
|
||||
isStreaming && handlerConfig
|
||||
? createOpenAIHandlers(
|
||||
handlerConfig,
|
||||
deps.toolExecuteOptions == null
|
||||
? undefined
|
||||
: {
|
||||
...deps.toolExecuteOptions,
|
||||
runSignal: abortController.signal,
|
||||
foregroundRunId: requestId,
|
||||
},
|
||||
)
|
||||
: {};
|
||||
const eventHandlers = createOpenAIHandlers(
|
||||
handlerConfig
|
||||
? { ...handlerConfig, signal: abortController.signal }
|
||||
: {
|
||||
aggregator: aggregator!,
|
||||
signal: abortController.signal,
|
||||
},
|
||||
deps.toolExecuteOptions == null
|
||||
? undefined
|
||||
: {
|
||||
...deps.toolExecuteOptions,
|
||||
runSignal: abortController.signal,
|
||||
foregroundRunId: requestId,
|
||||
},
|
||||
);
|
||||
|
||||
// Convert messages to internal format
|
||||
const messages = convertMessages(request.messages);
|
||||
|
|
@ -877,32 +880,40 @@ export async function createAgentChatCompletion(
|
|||
});
|
||||
|
||||
if (run) {
|
||||
await run.processStream(
|
||||
{ messages },
|
||||
const target = tracker ?? aggregator!;
|
||||
await completeOpenAIToolCalls(
|
||||
{
|
||||
runName: 'AgentRun',
|
||||
configurable: {
|
||||
thread_id: conversationId,
|
||||
user_id: userId,
|
||||
user: safeUser,
|
||||
requestBody: mcpRequestBody,
|
||||
/** Same per-agent channel the in-repo controllers thread via
|
||||
* `loadTools`: without it, the executor's PTC path cannot
|
||||
* strip host-injected `intent` params from the schemas the
|
||||
* sandbox bridge advertises on this route. */
|
||||
...(initializedAgent.intentToolNames?.length
|
||||
? { intentToolNames: initializedAgent.intentToolNames }
|
||||
: {}),
|
||||
},
|
||||
recursionLimit: resolveRecursionLimit(
|
||||
agentsConfig as Partial<TAgentsEndpoint> | undefined,
|
||||
initializedAgent,
|
||||
),
|
||||
signal: abortController.signal,
|
||||
streamMode: 'values',
|
||||
version: 'v2',
|
||||
finish: () => target.finishToolCalls?.(),
|
||||
abort: () => target.abortToolCalls?.(),
|
||||
},
|
||||
{},
|
||||
() =>
|
||||
run.processStream(
|
||||
{ messages },
|
||||
{
|
||||
runName: 'AgentRun',
|
||||
configurable: {
|
||||
thread_id: conversationId,
|
||||
user_id: userId,
|
||||
user: safeUser,
|
||||
requestBody: mcpRequestBody,
|
||||
/** Same per-agent channel the in-repo controllers thread via
|
||||
* `loadTools`: without it, the executor's PTC path cannot
|
||||
* strip host-injected `intent` params from the schemas the
|
||||
* sandbox bridge advertises on this route. */
|
||||
...(initializedAgent.intentToolNames?.length
|
||||
? { intentToolNames: initializedAgent.intentToolNames }
|
||||
: {}),
|
||||
},
|
||||
recursionLimit: resolveRecursionLimit(
|
||||
agentsConfig as Partial<TAgentsEndpoint> | undefined,
|
||||
initializedAgent,
|
||||
),
|
||||
signal: abortController.signal,
|
||||
streamMode: 'values',
|
||||
version: 'v2',
|
||||
},
|
||||
{},
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -912,6 +923,7 @@ export async function createAgentChatCompletion(
|
|||
sendFinalChunk(handlerConfig);
|
||||
res.end();
|
||||
} else if (aggregator) {
|
||||
aggregator.finishToolCalls?.();
|
||||
// Build and send non-streaming response
|
||||
const usage: CompletionUsage = {
|
||||
prompt_tokens: aggregator.usage.promptTokens,
|
||||
|
|
|
|||
308
packages/api/src/agents/openai/toolCalls.contract.spec.ts
Normal file
308
packages/api/src/agents/openai/toolCalls.contract.spec.ts
Normal file
|
|
@ -0,0 +1,308 @@
|
|||
import type { Agents } from 'librechat-data-provider';
|
||||
import type { ChatCompletionChunkChoice, ToolCall } from './types';
|
||||
import { createOpenAIToolCallStream } from './handlers';
|
||||
|
||||
/** Public contract fixtures, not a narrower reconstruction of the SDK fields. */
|
||||
describe('tool-call field normalization', () => {
|
||||
it.each(['native-string', 'native-object', 'wire-string', 'wire-object'])(
|
||||
'normalizes %s exactly once',
|
||||
(shape) => {
|
||||
const toolCalls = new Map<number, ToolCall>();
|
||||
const deltas: ChatCompletionChunkChoice['delta'][] = [];
|
||||
const stream = createOpenAIToolCallStream({ toolCalls, emit: (delta) => deltas.push(delta) });
|
||||
const args = shape.endsWith('string') ? '{"city":"Madrid"}' : { city: 'Madrid' };
|
||||
const call: Agents.AgentToolCall = shape.startsWith('native')
|
||||
? { id: 'a', name: 'lookup', args }
|
||||
: { id: 'a', type: 'function', function: { name: 'lookup', arguments: args } };
|
||||
stream.onRunStep({ id: 'step', stepDetails: { type: 'tool_calls', tool_calls: [call] } });
|
||||
stream.finish();
|
||||
expect(toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}');
|
||||
expect(
|
||||
deltas
|
||||
.flatMap((d) => d.tool_calls ?? [])
|
||||
.map((c) => c.function?.arguments ?? '')
|
||||
.join(''),
|
||||
).toBe('{"city":"Madrid"}');
|
||||
},
|
||||
);
|
||||
|
||||
it.each([false, true])(
|
||||
'assembles name substrings before the consumer freezes them (snapshot=%s)',
|
||||
(snapshot) => {
|
||||
const toolCalls = new Map<number, ToolCall>();
|
||||
const deltas: ChatCompletionChunkChoice['delta'][] = [];
|
||||
const stream = createOpenAIToolCallStream({ toolCalls, emit: (delta) => deltas.push(delta) });
|
||||
if (snapshot)
|
||||
stream.onRunStep({
|
||||
id: 'step',
|
||||
stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'a', name: 'get_', args: {} }] },
|
||||
});
|
||||
for (const chunk of [
|
||||
{ id: 'a', name: 'get_', index: 0, args: '{}' },
|
||||
{ name: 'weather', index: 0 },
|
||||
]) {
|
||||
stream.onRunStepDelta({ id: 'step', delta: { type: 'tool_calls', tool_calls: [chunk] } });
|
||||
}
|
||||
stream.finish();
|
||||
expect(toolCalls.get(0)?.function.name).toBe('get_weather');
|
||||
expect(
|
||||
deltas
|
||||
.flatMap((d) => d.tool_calls ?? [])
|
||||
.filter((c) => c.id)
|
||||
.map((c) => c.function?.name),
|
||||
).toEqual(['get_weather']);
|
||||
},
|
||||
);
|
||||
|
||||
it('synthesizes stable, distinct IDs for id-less complete calls and replayed declarations', () => {
|
||||
const toolCalls = new Map<number, ToolCall>();
|
||||
const stream = createOpenAIToolCallStream({ toolCalls });
|
||||
const calls: Agents.ToolCall[] = [
|
||||
{ name: 'lookup', args: {} },
|
||||
{ name: 'lookup', args: { city: 'Paris' } },
|
||||
];
|
||||
const event = { id: 'step', stepDetails: { type: 'tool_calls', tool_calls: calls } };
|
||||
stream.onRunStep(event);
|
||||
stream.onRunStep(event);
|
||||
stream.onRunStep({ ...event, id: 'later' });
|
||||
stream.finish();
|
||||
expect(toolCalls.size).toBe(4);
|
||||
expect(new Set([...toolCalls.values()].map((c) => c.id)).size).toBe(4);
|
||||
expect([...toolCalls.values()].every((c) => !!c.id)).toBe(true);
|
||||
});
|
||||
it.each(['native', 'wire'])('assembles split names and IDs with %s delta fields', (shape) => {
|
||||
const toolCalls = new Map<number, ToolCall>();
|
||||
const stream = createOpenAIToolCallStream({ toolCalls });
|
||||
for (const [id, name, args] of [
|
||||
['call_', 'get_', '{"city":'],
|
||||
['123', 'weather', '"Paris"}'],
|
||||
]) {
|
||||
stream.onRunStepDelta({
|
||||
id: 'step',
|
||||
delta: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [
|
||||
shape === 'native'
|
||||
? { index: 4, id, name, args }
|
||||
: { index: 4, id, function: { name, arguments: args } },
|
||||
],
|
||||
},
|
||||
});
|
||||
}
|
||||
stream.finish();
|
||||
expect([...toolCalls.values()]).toEqual([
|
||||
{
|
||||
id: 'call_123',
|
||||
type: 'function',
|
||||
function: { name: 'get_weather', arguments: '{"city":"Paris"}' },
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([true, false])(
|
||||
'keeps id-less calls distinct with mixed raw data (explicit index=%s)',
|
||||
(explicit) => {
|
||||
const toolCalls = new Map<number, ToolCall>();
|
||||
const stream = createOpenAIToolCallStream({ toolCalls });
|
||||
stream.onRunStep({
|
||||
id: 'step',
|
||||
stepDetails: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [
|
||||
{ name: 'get_', args: {}, ...(explicit && { index: 5 }) },
|
||||
{ name: 'lookup', args: { city: 'Paris' }, ...(explicit && { index: 8 }) },
|
||||
],
|
||||
},
|
||||
});
|
||||
const index = explicit ? 5 : 0;
|
||||
stream.onRunStepDelta({
|
||||
id: 'step',
|
||||
delta: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [{ index, name: 'get_', args: '{"city":"Madrid"}' }],
|
||||
},
|
||||
});
|
||||
stream.onRunStepDelta({
|
||||
id: 'step',
|
||||
delta: { type: 'tool_calls', tool_calls: [{ index, name: 'weather' }] },
|
||||
});
|
||||
stream.finish();
|
||||
expect([...toolCalls.values()].map((call) => call.function)).toEqual([
|
||||
{ name: 'get_weather', arguments: '{"city":"Madrid"}' },
|
||||
{ name: 'lookup', arguments: '{"city":"Paris"}' },
|
||||
]);
|
||||
expect(new Set([...toolCalls.values()].map((call) => call.id)).size).toBe(2);
|
||||
},
|
||||
);
|
||||
|
||||
it('correlates an id-less declaration with a later identified chunk rather than adding a call', () => {
|
||||
const toolCalls = new Map<number, ToolCall>();
|
||||
const stream = createOpenAIToolCallStream({ toolCalls });
|
||||
stream.onRunStep({
|
||||
id: 'step',
|
||||
stepDetails: { type: 'tool_calls', tool_calls: [{ name: 'lookup', args: {} }] },
|
||||
});
|
||||
stream.onRunStepDelta({
|
||||
id: 'step',
|
||||
delta: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [{ id: 'provider', name: 'lookup', index: 9, args: '{}' }],
|
||||
},
|
||||
});
|
||||
stream.finish();
|
||||
expect(toolCalls.size).toBe(1);
|
||||
expect(toolCalls.get(0)?.id).toBe('provider');
|
||||
});
|
||||
|
||||
it('assembles name fragments before and after a full snapshot without appending the snapshot', () => {
|
||||
const toolCalls = new Map<number, ToolCall>();
|
||||
const stream = createOpenAIToolCallStream({ toolCalls });
|
||||
stream.onRunStepDelta({
|
||||
id: 'step',
|
||||
delta: { type: 'tool_calls', tool_calls: [{ id: 'a', index: 0, name: 'get_', args: '{}' }] },
|
||||
});
|
||||
stream.onRunStep({
|
||||
id: 'step',
|
||||
stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'a', name: 'get_weather', args: {} }] },
|
||||
});
|
||||
stream.onRunStepDelta({
|
||||
id: 'step',
|
||||
delta: { type: 'tool_calls', tool_calls: [{ index: 0, name: 'weather' }] },
|
||||
});
|
||||
stream.finish();
|
||||
expect(toolCalls.get(0)?.function.name).toBe('get_weather');
|
||||
});
|
||||
|
||||
it('seals before invoking an emitter that reenters finish or throws', () => {
|
||||
const toolCalls = new Map<number, ToolCall>();
|
||||
const emit = jest.fn(() => {
|
||||
stream.finish();
|
||||
throw new Error('transport failed');
|
||||
});
|
||||
const stream = createOpenAIToolCallStream({ toolCalls, emit });
|
||||
stream.onRunStep({
|
||||
id: 'step',
|
||||
stepDetails: { type: 'tool_calls', tool_calls: [{ name: 'lookup', args: {} }] },
|
||||
});
|
||||
expect(() => stream.finish()).toThrow('transport failed');
|
||||
expect(() => stream.finish()).not.toThrow();
|
||||
expect(emit).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not emit any tool identity if another call is malformed', () => {
|
||||
const toolCalls = new Map<number, ToolCall>();
|
||||
const emit = jest.fn();
|
||||
const stream = createOpenAIToolCallStream({ toolCalls, emit });
|
||||
stream.onRunStep({
|
||||
id: 'step',
|
||||
stepDetails: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [
|
||||
{ name: 'lookup', args: {} },
|
||||
{ name: 'lookup', args: 'NOT_JSON' },
|
||||
],
|
||||
},
|
||||
});
|
||||
expect(() => stream.finish()).toThrow('Invalid tool call arguments');
|
||||
expect(emit).not.toHaveBeenCalled();
|
||||
expect(toolCalls.size).toBe(0);
|
||||
});
|
||||
it('matches late id-less snapshot arrays to already indexed raw calls', () => {
|
||||
const toolCalls = new Map<number, ToolCall>();
|
||||
const stream = createOpenAIToolCallStream({ toolCalls });
|
||||
for (const index of [0, 1])
|
||||
stream.onRunStepDelta({
|
||||
id: 'step',
|
||||
delta: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [{ index, name: 'lookup', args: JSON.stringify({ index }) }],
|
||||
},
|
||||
});
|
||||
stream.onRunStep({
|
||||
id: 'step',
|
||||
stepDetails: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [
|
||||
{ name: 'lookup', args: { index: 0 } },
|
||||
{ name: 'lookup', args: { index: 1 } },
|
||||
],
|
||||
},
|
||||
});
|
||||
stream.finish();
|
||||
expect(toolCalls.size).toBe(2);
|
||||
expect([...toolCalls.values()].map((call) => call.function.arguments)).toEqual([
|
||||
'{"index":0}',
|
||||
'{"index":1}',
|
||||
]);
|
||||
});
|
||||
|
||||
it('separates fully identified raw calls when no provider index is supplied', () => {
|
||||
const toolCalls = new Map<number, ToolCall>();
|
||||
const stream = createOpenAIToolCallStream({ toolCalls });
|
||||
for (const id of ['a', 'b'])
|
||||
stream.onRunStepDelta({
|
||||
id: 'step',
|
||||
delta: { type: 'tool_calls', tool_calls: [{ id, name: 'lookup', args: '{}' }] },
|
||||
});
|
||||
stream.finish();
|
||||
expect([...toolCalls.values()].map((call) => call.id)).toEqual(['a', 'b']);
|
||||
});
|
||||
|
||||
it('does not guess that identical name fragments are replayed data', () => {
|
||||
const toolCalls = new Map<number, ToolCall>();
|
||||
const stream = createOpenAIToolCallStream({ toolCalls });
|
||||
for (const name of ['a', 'a'])
|
||||
stream.onRunStepDelta({
|
||||
id: 'step',
|
||||
delta: { type: 'tool_calls', tool_calls: [{ index: 0, name }] },
|
||||
});
|
||||
stream.onRunStepDelta({
|
||||
id: 'step',
|
||||
delta: { type: 'tool_calls', tool_calls: [{ index: 0, args: '{}' }] },
|
||||
});
|
||||
stream.finish();
|
||||
expect(toolCalls.get(0)?.function.name).toBe('aa');
|
||||
});
|
||||
it.each([true, false])(
|
||||
'does not alias parallel shared ID prefixes (snapshots=%s)',
|
||||
(snapshots) => {
|
||||
const toolCalls = new Map<number, ToolCall>();
|
||||
const stream = createOpenAIToolCallStream({ toolCalls });
|
||||
if (snapshots)
|
||||
stream.onRunStep({
|
||||
id: 'step',
|
||||
stepDetails: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [
|
||||
{ id: 'call_0', name: 'get_weather', args: {} },
|
||||
{ id: 'call_1', name: 'get_weather', args: {} },
|
||||
],
|
||||
},
|
||||
});
|
||||
for (const index of [0, 1])
|
||||
stream.onRunStepDelta({
|
||||
id: 'step',
|
||||
delta: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [{ index, id: 'call_', name: 'get_', args: '{"i":' }],
|
||||
},
|
||||
});
|
||||
for (const index of [1, 0])
|
||||
stream.onRunStepDelta({
|
||||
id: 'step',
|
||||
delta: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [{ index, id: String(index), name: 'weather', args: `${index}}` }],
|
||||
},
|
||||
});
|
||||
stream.finish();
|
||||
expect([...toolCalls.values()]).toEqual(
|
||||
[0, 1].map((i) => ({
|
||||
id: `call_${i}`,
|
||||
type: 'function',
|
||||
function: { name: 'get_weather', arguments: JSON.stringify({ i }) },
|
||||
})),
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
302
packages/api/src/agents/openai/toolCalls.graph.spec.ts
Normal file
302
packages/api/src/agents/openai/toolCalls.graph.spec.ts
Normal file
|
|
@ -0,0 +1,302 @@
|
|||
import { response } from 'express';
|
||||
import { AIMessageChunk } from '@librechat/agents/langchain/messages';
|
||||
import {
|
||||
ChatModelStreamHandler,
|
||||
HandlerRegistry,
|
||||
Providers,
|
||||
StandardGraph,
|
||||
} from '@librechat/agents';
|
||||
import type { Response } from 'express';
|
||||
import type { ChatCompletionChunk, ToolCall } from './types';
|
||||
import {
|
||||
createOpenAIHandlers,
|
||||
createOpenAIStreamTracker,
|
||||
createOpenAIContentAggregator,
|
||||
createOpenAIToolCallStream,
|
||||
OpenAIRunStepHandler,
|
||||
OpenAIRunStepDeltaHandler,
|
||||
sendFinalChunk,
|
||||
} from './handlers';
|
||||
|
||||
/** No graph event fixtures: the SDK generates run steps and deltas from real
|
||||
* AIMessageChunk instances. Only the provider transport is omitted. */
|
||||
describe('tool-call projection with real SDK graph dispatch', () => {
|
||||
it.each([true, false])(
|
||||
'preserves interleaved calls across SDK steps (stream=%s)',
|
||||
async (streaming) => {
|
||||
const frames: string[] = [];
|
||||
const tracker = createOpenAIStreamTracker();
|
||||
const context = { requestId: 'test', created: 1, model: 'fixture' };
|
||||
const res: Response = Object.create(response);
|
||||
jest.spyOn(res, 'write').mockImplementation((frame) => {
|
||||
frames.push(String(frame));
|
||||
return true;
|
||||
});
|
||||
const config = { tracker, context, res };
|
||||
const stream = createOpenAIToolCallStream({ toolCalls: tracker.toolCalls });
|
||||
const handlers = streaming
|
||||
? createOpenAIHandlers(config)
|
||||
: {
|
||||
on_run_step: new OpenAIRunStepHandler(stream),
|
||||
on_run_step_delta: new OpenAIRunStepDeltaHandler(stream),
|
||||
};
|
||||
const graph = new StandardGraph({
|
||||
runId: 'test',
|
||||
agents: [{ agentId: 'agent', provider: Providers.OPENAI, tools: [] }],
|
||||
});
|
||||
graph.config = { configurable: { run_id: 'test', thread_id: 'thread' } };
|
||||
graph.handlerRegistry = new HandlerRegistry();
|
||||
for (const [event, handler] of Object.entries(handlers)) {
|
||||
graph.handlerRegistry.register(event, handler);
|
||||
}
|
||||
const producer = new ChatModelStreamHandler();
|
||||
for (const chunk of [
|
||||
{ index: 0, id: 'a', name: 'get_time', args: '' },
|
||||
{ index: 1, id: 'b', name: 'get_time', args: '' },
|
||||
{ index: 0, args: '{"city":' },
|
||||
{ index: 1, args: '{"city":"Paris"}' },
|
||||
{ index: 0, args: '"Madrid"}' },
|
||||
]) {
|
||||
await producer.handle(
|
||||
'on_chat_model_stream',
|
||||
{
|
||||
chunk: new AIMessageChunk({
|
||||
content: '',
|
||||
tool_call_chunks: [{ ...chunk, type: 'tool_call_chunk' }],
|
||||
}),
|
||||
},
|
||||
{ langgraph_node: 'agent=agent', langgraph_step: 1 },
|
||||
graph,
|
||||
);
|
||||
}
|
||||
expect(frames).toEqual([]);
|
||||
if (streaming) tracker.finishToolCalls?.();
|
||||
else stream.finish();
|
||||
expect([...tracker.toolCalls.values()].map((call) => call.function.arguments)).toEqual([
|
||||
'{"city":"Madrid"}',
|
||||
'{"city":"Paris"}',
|
||||
]);
|
||||
if (streaming) {
|
||||
const received = new Map<number, ToolCall>();
|
||||
for (const frame of frames) {
|
||||
const chunk: ChatCompletionChunk = JSON.parse(frame.slice(6));
|
||||
for (const part of chunk.choices[0].delta.tool_calls ?? []) {
|
||||
const existing = received.get(part.index);
|
||||
if (existing) {
|
||||
existing.function.arguments += part.function?.arguments ?? '';
|
||||
} else {
|
||||
expect(part.id).toBeDefined();
|
||||
expect(part.function?.name).toBe('get_time');
|
||||
received.set(part.index, {
|
||||
id: part.id!,
|
||||
type: 'function',
|
||||
function: {
|
||||
name: part.function!.name!,
|
||||
arguments: part.function?.arguments ?? '',
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
expect([...received.values()]).toEqual([...tracker.toolCalls.values()]);
|
||||
await producer.handle(
|
||||
'on_chat_model_stream',
|
||||
{
|
||||
chunk: new AIMessageChunk({ content: 'Both tools completed. Here is the answer.' }),
|
||||
},
|
||||
{ langgraph_node: 'agent=agent', langgraph_step: 3 },
|
||||
graph,
|
||||
);
|
||||
expect(tracker.hasText).toBe(true);
|
||||
sendFinalChunk(config, 'stop');
|
||||
const final: ChatCompletionChunk = JSON.parse(frames[frames.length - 2].slice(6));
|
||||
expect(final.choices[0].finish_reason).toBe('stop');
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it('isolates provider index zero across agents, invocations and checkpoint namespaces', () => {
|
||||
const tracker = createOpenAIStreamTracker();
|
||||
const stream = createOpenAIToolCallStream({ toolCalls: tracker.toolCalls });
|
||||
const metadata = [
|
||||
{ langgraph_node: 'agent=a', langgraph_step: 1 },
|
||||
{ langgraph_node: 'agent=b', langgraph_step: 1 },
|
||||
{ langgraph_node: 'agent=a', langgraph_step: 3 },
|
||||
{ langgraph_node: 'agent=a', langgraph_step: 3, checkpoint_ns: 'child' },
|
||||
];
|
||||
for (const [i, meta] of metadata.entries()) {
|
||||
stream.onRunStep(
|
||||
{
|
||||
id: `step_${i}`,
|
||||
stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'call_0', name: 'get_time' }] },
|
||||
},
|
||||
meta,
|
||||
);
|
||||
stream.onRunStepDelta(
|
||||
{
|
||||
id: `step_${i}`,
|
||||
delta: { type: 'tool_calls', tool_calls: [{ index: 0, id: 'call_0', name: 'get_time' }] },
|
||||
},
|
||||
meta,
|
||||
);
|
||||
}
|
||||
for (const [i, meta] of metadata.entries()) {
|
||||
stream.onRunStepDelta(
|
||||
{
|
||||
id: `step_${i}`,
|
||||
delta: { type: 'tool_calls', tool_calls: [{ index: 0, args: JSON.stringify({ i }) }] },
|
||||
},
|
||||
meta,
|
||||
);
|
||||
}
|
||||
stream.finish();
|
||||
expect([...tracker.toolCalls.values()].map((call) => call.function.arguments)).toEqual(
|
||||
metadata.map((_, i) => JSON.stringify({ i })),
|
||||
);
|
||||
expect(new Set([...tracker.toolCalls.values()].map((call) => call.id)).size).toBe(4);
|
||||
});
|
||||
it.each([true, false])(
|
||||
'projects complete-only SDK messages at response completion (stream=%s)',
|
||||
async (streaming) => {
|
||||
const frames: string[] = [];
|
||||
const res: Response = Object.create(response);
|
||||
jest.spyOn(res, 'write').mockImplementation((frame) => {
|
||||
frames.push(String(frame));
|
||||
return true;
|
||||
});
|
||||
const tracker = createOpenAIStreamTracker();
|
||||
const aggregator = createOpenAIContentAggregator();
|
||||
const config = {
|
||||
tracker,
|
||||
res,
|
||||
context: { requestId: 'complete', created: 1, model: 'fixture' },
|
||||
};
|
||||
const handlers = createOpenAIHandlers(streaming ? config : { aggregator });
|
||||
const graph = new StandardGraph({
|
||||
runId: 'complete',
|
||||
agents: [{ agentId: 'agent', provider: Providers.OPENAI, tools: [] }],
|
||||
});
|
||||
graph.config = { configurable: { run_id: 'complete', thread_id: 'thread' } };
|
||||
graph.handlerRegistry = new HandlerRegistry();
|
||||
for (const [event, handler] of Object.entries(handlers))
|
||||
graph.handlerRegistry.register(event, handler);
|
||||
await new ChatModelStreamHandler().handle(
|
||||
'on_chat_model_stream',
|
||||
{
|
||||
chunk: new AIMessageChunk({
|
||||
content: '',
|
||||
tool_calls: [{ id: 'a', name: 'get_time', args: { city: 'Madrid' } }],
|
||||
}),
|
||||
},
|
||||
{ langgraph_node: 'agent=agent', langgraph_step: 1 },
|
||||
graph,
|
||||
);
|
||||
const target = streaming ? tracker : aggregator;
|
||||
expect(target.toolCalls.size).toBe(0);
|
||||
if (streaming) sendFinalChunk(config);
|
||||
else target.finishToolCalls?.();
|
||||
expect(target.toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}');
|
||||
if (streaming) {
|
||||
const chunks: ChatCompletionChunk[] = frames
|
||||
.filter((frame) => frame !== 'data: [DONE]\n\n')
|
||||
.map((frame) => JSON.parse(frame.slice(6)));
|
||||
expect(
|
||||
chunks
|
||||
.flatMap((chunk) => chunk.choices[0].delta.tool_calls ?? [])
|
||||
.map((call) => call.function?.arguments)
|
||||
.join(''),
|
||||
).toBe('{"city":"Madrid"}');
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it('uses graph-owned stream segments without stealing late fragments from the old segment', () => {
|
||||
const graph = new StandardGraph({
|
||||
runId: 'segments',
|
||||
agents: [{ agentId: 'a', provider: Providers.OPENAI, tools: [] }],
|
||||
});
|
||||
graph.config = { configurable: { run_id: 'segments', thread_id: 'thread' } };
|
||||
const meta = { langgraph_node: 'agent=a', langgraph_step: 1 };
|
||||
const tracker = createOpenAIStreamTracker();
|
||||
const stream = createOpenAIToolCallStream({ toolCalls: tracker.toolCalls });
|
||||
const start = new OpenAIRunStepHandler(stream);
|
||||
const delta = new OpenAIRunStepDeltaHandler(stream);
|
||||
for (let i = 0; i < 2; i++) {
|
||||
start.handle(
|
||||
'on_run_step',
|
||||
{
|
||||
id: `step_${i}`,
|
||||
stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'call_0', name: 'get_time' }] },
|
||||
},
|
||||
meta,
|
||||
graph,
|
||||
);
|
||||
delta.handle(
|
||||
'on_run_step_delta',
|
||||
{
|
||||
id: `step_${i}`,
|
||||
delta: { type: 'tool_calls', tool_calls: [{ id: 'call_0', index: 0, name: 'get_time' }] },
|
||||
},
|
||||
meta,
|
||||
graph,
|
||||
);
|
||||
graph.advanceStreamSegment();
|
||||
}
|
||||
for (let i = 0; i < 2; i++)
|
||||
delta.handle(
|
||||
'on_run_step_delta',
|
||||
{
|
||||
id: `step_${i}`,
|
||||
delta: { type: 'tool_calls', tool_calls: [{ index: 0, args: JSON.stringify({ i }) }] },
|
||||
},
|
||||
meta,
|
||||
graph,
|
||||
);
|
||||
stream.finish();
|
||||
expect([...tracker.toolCalls.values()].map((call) => call.function.arguments)).toEqual([
|
||||
'{"i":0}',
|
||||
'{"i":1}',
|
||||
]);
|
||||
});
|
||||
it('assembles the name before emitting when real SDK chunks split it', async () => {
|
||||
const tracker = createOpenAIStreamTracker();
|
||||
const emitted: ChatCompletionChunk['choices'][number]['delta'][] = [];
|
||||
const stream = createOpenAIToolCallStream({
|
||||
toolCalls: tracker.toolCalls,
|
||||
emit: (delta) => emitted.push(delta),
|
||||
});
|
||||
const graph = new StandardGraph({
|
||||
runId: 'split',
|
||||
agents: [{ agentId: 'a', provider: Providers.OPENAI, tools: [] }],
|
||||
});
|
||||
graph.config = { configurable: { run_id: 'split', thread_id: 'split' } };
|
||||
graph.handlerRegistry = new HandlerRegistry();
|
||||
const handlers: ReturnType<typeof createOpenAIHandlers> = {
|
||||
on_run_step: new OpenAIRunStepHandler(stream),
|
||||
on_run_step_delta: new OpenAIRunStepDeltaHandler(stream),
|
||||
};
|
||||
for (const [event, handler] of Object.entries(handlers))
|
||||
graph.handlerRegistry.register(event, handler);
|
||||
const producer = new ChatModelStreamHandler();
|
||||
for (const chunk of [
|
||||
{ id: 'a', index: 0, name: 'get_', args: '' },
|
||||
{ index: 0, args: '{}' },
|
||||
{ index: 0, name: 'weather' },
|
||||
])
|
||||
await producer.handle(
|
||||
'on_chat_model_stream',
|
||||
{
|
||||
chunk: new AIMessageChunk({
|
||||
content: '',
|
||||
tool_call_chunks: [{ ...chunk, type: 'tool_call_chunk' }],
|
||||
}),
|
||||
},
|
||||
{ langgraph_node: 'agent=a', langgraph_step: 1 },
|
||||
graph,
|
||||
);
|
||||
expect(emitted).toEqual([]);
|
||||
stream.finish();
|
||||
expect(tracker.toolCalls.get(0)?.function.name).toBe('get_weather');
|
||||
expect(emitted[0].tool_calls?.[0].function?.name).toBe('get_weather');
|
||||
});
|
||||
});
|
||||
616
packages/api/src/agents/openai/toolCalls.spec.ts
Normal file
616
packages/api/src/agents/openai/toolCalls.spec.ts
Normal file
|
|
@ -0,0 +1,616 @@
|
|||
import type { ChatCompletionChunkChoice, OpenAIResponseContext, ToolCall } from './types';
|
||||
import type { RunStepData, RunStepDeltaData } from './handlers';
|
||||
import {
|
||||
sendFinalChunk,
|
||||
completeOpenAIToolCalls,
|
||||
createOpenAIStreamTracker,
|
||||
createOpenAIToolCallStream,
|
||||
createOpenAIContentAggregator,
|
||||
} from './handlers';
|
||||
import { buildNonStreamingResponse } from './service';
|
||||
|
||||
type Delta = ChatCompletionChunkChoice['delta'];
|
||||
|
||||
/**
|
||||
* Event payloads below are the shapes `@librechat/agents` actually dispatches
|
||||
* for a Bedrock Converse stream: the run step opens a call with its id and name
|
||||
* and carries no tool-call index, the first delta fragment repeats id and name
|
||||
* with the provider's content-block index, and later fragments carry only that
|
||||
* index and an argument slice. A step's own `index` is its position in the
|
||||
* response content, which is why it appears here as an unrelated number.
|
||||
*/
|
||||
function runStep(stepId: string, contentIndex: number, id: string, name: string): RunStepData {
|
||||
return {
|
||||
id: stepId,
|
||||
index: contentIndex,
|
||||
stepDetails: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [{ name, args: {}, id, type: 'tool_call' }],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function opensCall(
|
||||
stepId: string,
|
||||
providerIndex: number,
|
||||
id: string,
|
||||
name: string,
|
||||
): RunStepDeltaData {
|
||||
return {
|
||||
id: stepId,
|
||||
delta: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [{ name, id, index: providerIndex, type: 'tool_call_chunk' }],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function streamsArgs(stepId: string, providerIndex: number, args: string): RunStepDeltaData {
|
||||
return {
|
||||
id: stepId,
|
||||
delta: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [{ args, index: providerIndex, type: 'tool_call_chunk' }],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The accumulation an OpenAI-compatible client performs, including the rule that
|
||||
* produced the reported `AI_InvalidResponseDataError: Expected 'id' to be a
|
||||
* string`: the first chunk seen at an index must declare the call.
|
||||
*/
|
||||
function accumulateLikeClient(deltas: Delta[]) {
|
||||
const calls = new Map<number, { id: string; name: string; arguments: string }>();
|
||||
for (const delta of deltas) {
|
||||
for (const fragment of delta.tool_calls ?? []) {
|
||||
const index = fragment.index as number;
|
||||
const open = calls.get(index);
|
||||
if (open === undefined) {
|
||||
if (!fragment.id || !fragment.function?.name) {
|
||||
throw new Error(`first chunk at index ${index} declares no id and name`);
|
||||
}
|
||||
calls.set(index, {
|
||||
id: fragment.id,
|
||||
name: fragment.function.name,
|
||||
arguments: fragment.function.arguments ?? '',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
open.arguments += fragment.function?.arguments ?? '';
|
||||
}
|
||||
}
|
||||
return [...calls.entries()].map(([index, call]) => ({ index, ...call }));
|
||||
}
|
||||
|
||||
function streamingBridge() {
|
||||
const deltas: Delta[] = [];
|
||||
const toolCalls = new Map<number, ToolCall>();
|
||||
const stream = createOpenAIToolCallStream({
|
||||
toolCalls,
|
||||
emit: (delta) => deltas.push(delta),
|
||||
});
|
||||
return { stream, deltas, toolCalls };
|
||||
}
|
||||
|
||||
describe('outward tool call indexes', () => {
|
||||
it('keeps one index from the declaration through the argument fragments', () => {
|
||||
const { stream, deltas, toolCalls } = streamingBridge();
|
||||
|
||||
stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time'));
|
||||
stream.onRunStepDelta(opensCall('step_1', 1, 'call_a', 'get_time'));
|
||||
stream.onRunStepDelta(streamsArgs('step_1', 1, '{"city":'));
|
||||
stream.onRunStepDelta(streamsArgs('step_1', 1, '"Madrid"}'));
|
||||
|
||||
expect(deltas).toEqual([]);
|
||||
stream.finish();
|
||||
expect(deltas.flatMap((delta) => delta.tool_calls ?? []).map((call) => call.index)).toEqual([
|
||||
0, 0,
|
||||
]);
|
||||
expect(accumulateLikeClient(deltas)).toEqual([
|
||||
{ index: 0, id: 'call_a', name: 'get_time', arguments: '{"city":"Madrid"}' },
|
||||
]);
|
||||
expect([...toolCalls.values()]).toEqual([
|
||||
{
|
||||
id: 'call_a',
|
||||
type: 'function',
|
||||
function: { name: 'get_time', arguments: '{"city":"Madrid"}' },
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('declares a call once when the run step and the first fragment both name it', () => {
|
||||
const { stream, deltas } = streamingBridge();
|
||||
|
||||
stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time'));
|
||||
stream.onRunStepDelta(opensCall('step_1', 1, 'call_a', 'get_time'));
|
||||
|
||||
stream.finish();
|
||||
const declarations = deltas
|
||||
.flatMap((delta) => delta.tool_calls ?? [])
|
||||
.filter((call) => call.id !== undefined);
|
||||
expect(declarations).toEqual([
|
||||
{ index: 0, id: 'call_a', type: 'function', function: { name: 'get_time', arguments: '' } },
|
||||
]);
|
||||
});
|
||||
|
||||
it('separates parallel calls that arrive as consecutive single-call steps', () => {
|
||||
const { stream, deltas, toolCalls } = streamingBridge();
|
||||
|
||||
stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time'));
|
||||
stream.onRunStepDelta(opensCall('step_1', 0, 'call_a', 'get_time'));
|
||||
stream.onRunStepDelta(streamsArgs('step_1', 0, '{"city":"Madrid"}'));
|
||||
stream.onRunStep(runStep('step_2', 2, 'call_b', 'get_time'));
|
||||
stream.onRunStepDelta(opensCall('step_2', 1, 'call_b', 'get_time'));
|
||||
stream.onRunStepDelta(streamsArgs('step_2', 1, '{"city":"Paris"}'));
|
||||
|
||||
stream.finish();
|
||||
expect(accumulateLikeClient(deltas)).toEqual([
|
||||
{ index: 0, id: 'call_a', name: 'get_time', arguments: '{"city":"Madrid"}' },
|
||||
{ index: 1, id: 'call_b', name: 'get_time', arguments: '{"city":"Paris"}' },
|
||||
]);
|
||||
expect([...toolCalls.keys()]).toEqual([0, 1]);
|
||||
});
|
||||
|
||||
it('allocates a new index when a later model invocation reuses provider index zero', () => {
|
||||
const { stream, deltas, toolCalls } = streamingBridge();
|
||||
|
||||
stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time'));
|
||||
stream.onRunStepDelta(opensCall('step_1', 0, 'call_a', 'get_time'));
|
||||
stream.onRunStepDelta(streamsArgs('step_1', 0, '{"city":"Madrid"}'));
|
||||
stream.onRunStep(runStep('step_3', 3, 'call_b', 'get_time'));
|
||||
stream.onRunStepDelta(opensCall('step_3', 0, 'call_b', 'get_time'));
|
||||
stream.onRunStepDelta(streamsArgs('step_3', 0, '{"city":"Paris"}'));
|
||||
|
||||
stream.finish();
|
||||
expect(accumulateLikeClient(deltas)).toEqual([
|
||||
{ index: 0, id: 'call_a', name: 'get_time', arguments: '{"city":"Madrid"}' },
|
||||
{ index: 1, id: 'call_b', name: 'get_time', arguments: '{"city":"Paris"}' },
|
||||
]);
|
||||
expect(toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}');
|
||||
expect(toolCalls.get(1)?.function.arguments).toBe('{"city":"Paris"}');
|
||||
});
|
||||
|
||||
it('attributes unidentified argument fragments to the call its step opened', () => {
|
||||
const { stream, deltas } = streamingBridge();
|
||||
|
||||
stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time'));
|
||||
stream.onRunStepDelta(streamsArgs('step_1', 7, '{"city":"Madrid"}'));
|
||||
|
||||
stream.finish();
|
||||
expect(accumulateLikeClient(deltas)).toEqual([
|
||||
{ index: 0, id: 'call_a', name: 'get_time', arguments: '{"city":"Madrid"}' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('drops a fragment it cannot attribute rather than charging another call', () => {
|
||||
const { stream, deltas, toolCalls } = streamingBridge();
|
||||
|
||||
stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time'));
|
||||
stream.onRunStepDelta(opensCall('step_1', 0, 'call_a', 'get_time'));
|
||||
stream.onRunStep(runStep('step_2', 2, 'call_b', 'get_time'));
|
||||
stream.onRunStepDelta(opensCall('step_2', 1, 'call_b', 'get_time'));
|
||||
stream.onRunStepDelta({
|
||||
id: 'step_unknown',
|
||||
delta: { type: 'tool_calls', tool_calls: [{ index: 4, args: '{"city":"Lisbon"}' }] },
|
||||
});
|
||||
|
||||
expect(() => stream.finish()).toThrow('Unattributable tool call arguments');
|
||||
expect(toolCalls.size).toBe(0);
|
||||
expect(deltas).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([true, false])(
|
||||
'isolates repeated provider IDs across interleaved steps (stream=%s)',
|
||||
(streaming) => {
|
||||
const deltas: Delta[] = [];
|
||||
const toolCalls = new Map<number, ToolCall>();
|
||||
const stream = createOpenAIToolCallStream({
|
||||
toolCalls,
|
||||
emit: streaming ? (delta) => deltas.push(delta) : undefined,
|
||||
});
|
||||
stream.onRunStep(runStep('agent_a_turn_1', 1, 'call_0', 'get_time'));
|
||||
stream.onRunStepDelta(opensCall('agent_a_turn_1', 0, 'call_0', 'get_time'));
|
||||
stream.onRunStepDelta(streamsArgs('agent_a_turn_1', 0, '{"city":'));
|
||||
stream.onRunStep(runStep('agent_b_turn_1', 2, 'call_0', 'get_time'));
|
||||
stream.onRunStepDelta(opensCall('agent_b_turn_1', 0, 'call_0', 'get_time'));
|
||||
stream.onRunStepDelta(streamsArgs('agent_b_turn_1', 0, '{"city":"Paris"}'));
|
||||
stream.onRunStepDelta(streamsArgs('agent_a_turn_1', 0, '"Madrid"}'));
|
||||
stream.onRunStep(runStep('agent_a_turn_2', 3, 'call_0', 'get_time'));
|
||||
stream.onRunStepDelta(opensCall('agent_a_turn_2', 0, 'call_0', 'get_time'));
|
||||
stream.onRunStepDelta(streamsArgs('agent_a_turn_2', 0, '{"city":"Lisbon"}'));
|
||||
stream.finish();
|
||||
expect([...toolCalls.keys()]).toEqual([0, 1, 2]);
|
||||
expect([...toolCalls.values()].map((call) => call.function.arguments)).toEqual([
|
||||
'{"city":"Madrid"}',
|
||||
'{"city":"Paris"}',
|
||||
'{"city":"Lisbon"}',
|
||||
]);
|
||||
if (streaming) {
|
||||
expect(accumulateLikeClient(deltas).map((call) => call.arguments)).toEqual([
|
||||
'{"city":"Madrid"}',
|
||||
'{"city":"Paris"}',
|
||||
'{"city":"Lisbon"}',
|
||||
]);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it('declares OpenAI-shaped names without waiting for an identifying delta', () => {
|
||||
const { stream, deltas, toolCalls } = streamingBridge();
|
||||
stream.onRunStep({
|
||||
id: 'step_function',
|
||||
stepDetails: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [
|
||||
{
|
||||
id: 'call_a',
|
||||
type: 'function',
|
||||
function: { name: 'get_time', arguments: '{"city":"Madrid"}' },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
stream.finish();
|
||||
expect(toolCalls.get(0)?.function.name).toBe('get_time');
|
||||
expect(accumulateLikeClient(deltas)).toMatchObject([
|
||||
{ index: 0, id: 'call_a', name: 'get_time' },
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([true, false])(
|
||||
'binds multiple declared calls before ID-less fragments (explicit indexes=%s)',
|
||||
(explicit) => {
|
||||
const { stream, deltas, toolCalls } = streamingBridge();
|
||||
const indexes = explicit ? [4, 8] : [0, 1];
|
||||
stream.onRunStep({
|
||||
id: 'parallel',
|
||||
index: 12,
|
||||
stepDetails: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [
|
||||
{ id: 'call_a', name: 'get_time', ...(explicit && { index: indexes[0] }) },
|
||||
{ id: 'call_b', name: 'get_time', ...(explicit && { index: indexes[1] }) },
|
||||
],
|
||||
},
|
||||
});
|
||||
stream.onRunStepDelta(streamsArgs('parallel', indexes[1], '{"city":"Paris"}'));
|
||||
stream.onRunStepDelta(streamsArgs('parallel', indexes[0], '{"city":"Madrid"}'));
|
||||
stream.finish();
|
||||
stream.finish();
|
||||
expect(accumulateLikeClient(deltas).map((call) => call.arguments)).toEqual([
|
||||
'{"city":"Madrid"}',
|
||||
'{"city":"Paris"}',
|
||||
]);
|
||||
expect([...toolCalls.values()].map((call) => call.function.arguments)).toEqual([
|
||||
'{"city":"Madrid"}',
|
||||
'{"city":"Paris"}',
|
||||
]);
|
||||
},
|
||||
);
|
||||
|
||||
it('does not re-declare a replayed step or bind unrelated indexes to a known call', () => {
|
||||
const { stream, deltas, toolCalls } = streamingBridge();
|
||||
stream.onRunStep(runStep('step', 1, 'call_a', 'get_time'));
|
||||
stream.onRunStep(runStep('step', 1, 'call_a', 'get_time'));
|
||||
stream.onRunStepDelta(opensCall('step', 3, 'call_a', 'get_time'));
|
||||
stream.onRunStepDelta(streamsArgs('step', 9, '{"wrong":true}'));
|
||||
stream.onRunStepDelta(streamsArgs('step', 3, '{}'));
|
||||
expect(() => stream.finish()).toThrow('Unattributable tool call arguments');
|
||||
expect(deltas).toEqual([]);
|
||||
expect(toolCalls.size).toBe(0);
|
||||
});
|
||||
|
||||
it('buffers identified arguments until a later name-only fragment can declare the call', () => {
|
||||
const { stream, deltas, toolCalls } = streamingBridge();
|
||||
stream.onRunStepDelta({
|
||||
id: 'step',
|
||||
delta: { type: 'tool_calls', tool_calls: [{ id: 'call_a', index: 3, args: '{"city":' }] },
|
||||
});
|
||||
expect(deltas).toEqual([]);
|
||||
stream.onRunStepDelta({
|
||||
id: 'step',
|
||||
delta: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [{ index: 3, function: { name: 'get_time', arguments: '"Madrid"}' } }],
|
||||
},
|
||||
});
|
||||
stream.finish();
|
||||
expect(accumulateLikeClient(deltas)).toEqual([
|
||||
{ index: 0, id: 'call_a', name: 'get_time', arguments: '{"city":"Madrid"}' },
|
||||
]);
|
||||
expect(toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}');
|
||||
});
|
||||
|
||||
it('prefers identified provider indexes to declaration position', () => {
|
||||
const { stream, deltas } = streamingBridge();
|
||||
stream.onRunStep({
|
||||
id: 'step',
|
||||
stepDetails: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [
|
||||
{ id: 'call_a', name: 'get_time' },
|
||||
{ id: 'call_b', name: 'get_time' },
|
||||
],
|
||||
},
|
||||
});
|
||||
stream.onRunStepDelta(opensCall('step', 1, 'call_a', 'get_time'));
|
||||
stream.onRunStepDelta(opensCall('step', 0, 'call_b', 'get_time'));
|
||||
stream.onRunStepDelta(streamsArgs('step', 0, '{"city":"Paris"}'));
|
||||
stream.onRunStepDelta(streamsArgs('step', 1, '{"city":"Madrid"}'));
|
||||
stream.finish();
|
||||
expect(accumulateLikeClient(deltas).map((call) => call.arguments)).toEqual([
|
||||
'{"city":"Madrid"}',
|
||||
'{"city":"Paris"}',
|
||||
]);
|
||||
});
|
||||
|
||||
it('never merges unnamed-index fragments into a multi-call step', () => {
|
||||
const { stream, toolCalls } = streamingBridge();
|
||||
stream.onRunStep({
|
||||
id: 'step',
|
||||
stepDetails: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [
|
||||
{ id: 'a', name: 'get_time' },
|
||||
{ id: 'b', name: 'get_time' },
|
||||
],
|
||||
},
|
||||
});
|
||||
stream.onRunStepDelta({
|
||||
id: 'step',
|
||||
delta: { type: 'tool_calls', tool_calls: [{ args: 'unattributable' }] },
|
||||
});
|
||||
expect(() => stream.finish()).toThrow('Unattributable tool call arguments');
|
||||
expect(toolCalls.size).toBe(0);
|
||||
});
|
||||
|
||||
it('gives id-based consumers unique IDs and starts each response with fresh state', () => {
|
||||
const { stream, toolCalls } = streamingBridge();
|
||||
stream.onRunStep(runStep('s1', 1, 'call_0', 'get_time'));
|
||||
stream.onRunStep(runStep('s2', 2, 'call_0', 'get_time'));
|
||||
stream.onRunStep(runStep('s3', 3, 'call_0_1', 'get_time'));
|
||||
stream.finish();
|
||||
expect(new Set([...toolCalls.values()].map((call) => call.id)).size).toBe(3);
|
||||
const second = streamingBridge();
|
||||
second.stream.onRunStep(runStep('s1', 1, 'call_0', 'get_time'));
|
||||
second.stream.finish();
|
||||
expect([...second.toolCalls.keys()]).toEqual([0]);
|
||||
expect(second.toolCalls.get(0)?.id).toBe('call_0');
|
||||
});
|
||||
|
||||
it('accumulates the same calls for a non-streaming response with no emitter', () => {
|
||||
const aggregator = createOpenAIContentAggregator();
|
||||
const stream = createOpenAIToolCallStream({ toolCalls: aggregator.toolCalls });
|
||||
|
||||
stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time'));
|
||||
stream.onRunStepDelta(opensCall('step_1', 0, 'call_a', 'get_time'));
|
||||
stream.onRunStepDelta(streamsArgs('step_1', 0, '{"city":"Madrid"}'));
|
||||
stream.onRunStep(runStep('step_3', 3, 'call_b', 'get_time'));
|
||||
stream.onRunStepDelta(opensCall('step_3', 0, 'call_b', 'get_time'));
|
||||
stream.onRunStepDelta(streamsArgs('step_3', 0, '{"city":"Paris"}'));
|
||||
|
||||
stream.finish();
|
||||
const response = buildNonStreamingResponse(
|
||||
{ requestId: 'chatcmpl-test', created: 1778317637, model: 'agent_test' },
|
||||
'Checking both.',
|
||||
'',
|
||||
aggregator.toolCalls,
|
||||
{ prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
|
||||
);
|
||||
|
||||
expect(response.choices[0].message.tool_calls).toEqual([
|
||||
{
|
||||
id: 'call_a',
|
||||
type: 'function',
|
||||
function: { name: 'get_time', arguments: '{"city":"Madrid"}' },
|
||||
},
|
||||
{
|
||||
id: 'call_b',
|
||||
type: 'function',
|
||||
function: { name: 'get_time', arguments: '{"city":"Paris"}' },
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('complete snapshots and streamed fragments', () => {
|
||||
function snapshot(step = 'complete', args = '{"city":"Madrid"}') {
|
||||
return {
|
||||
id: step,
|
||||
stepDetails: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [{ id: 'a', function: { name: 'get_time', arguments: args } }],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it.each(['before', 'after', 'between'])(
|
||||
'never concatenates a snapshot %s raw fragments',
|
||||
(order) => {
|
||||
const { stream, deltas, toolCalls } = streamingBridge();
|
||||
if (order === 'before') stream.onRunStep(snapshot());
|
||||
stream.onRunStepDelta(opensCall('complete', 0, 'a', 'get_time'));
|
||||
stream.onRunStepDelta(streamsArgs('complete', 0, '{"city":'));
|
||||
if (order === 'between') stream.onRunStep(snapshot());
|
||||
stream.onRunStepDelta(streamsArgs('complete', 0, '"Madrid"}'));
|
||||
if (order === 'after') stream.onRunStep(snapshot());
|
||||
stream.finish();
|
||||
expect(accumulateLikeClient(deltas)[0].arguments).toBe('{"city":"Madrid"}');
|
||||
expect(toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}');
|
||||
},
|
||||
);
|
||||
|
||||
it('captures a native snapshot without retaining the producer object', () => {
|
||||
const { stream, toolCalls } = streamingBridge();
|
||||
const args = { city: 'Madrid' };
|
||||
stream.onRunStep({
|
||||
id: 'step',
|
||||
stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'a', name: 'get_time', args }] },
|
||||
});
|
||||
args.city = 'Paris';
|
||||
stream.finish();
|
||||
expect(toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}');
|
||||
});
|
||||
|
||||
it('preserves empty native objects and does not flush a snapshot on identifying-only chunks', () => {
|
||||
const { stream, deltas } = streamingBridge();
|
||||
stream.onRunStep({
|
||||
id: 'complete',
|
||||
stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'a', name: 'noop', args: {} }] },
|
||||
});
|
||||
stream.onRunStepDelta(opensCall('complete', 0, 'a', 'noop'));
|
||||
expect(deltas).toEqual([]);
|
||||
stream.finish();
|
||||
expect(accumulateLikeClient(deltas)[0].arguments).toBe('{}');
|
||||
});
|
||||
|
||||
it('flushes replayed declarations only once and ignores writes after completion', () => {
|
||||
const { stream, deltas } = streamingBridge();
|
||||
stream.onRunStep(snapshot());
|
||||
stream.onRunStep(snapshot());
|
||||
stream.finish();
|
||||
stream.finish();
|
||||
stream.onRunStep(snapshot('late'));
|
||||
stream.onRunStepDelta(streamsArgs('complete', 0, 'late'));
|
||||
expect(accumulateLikeClient(deltas)).toEqual([
|
||||
{ index: 0, id: 'a', name: 'get_time', arguments: '{"city":"Madrid"}' },
|
||||
]);
|
||||
});
|
||||
|
||||
it.each(['abort', 'reject', 'sync throw'])(
|
||||
'does not flush fallback snapshots on %s',
|
||||
async (failure) => {
|
||||
const signal = new AbortController();
|
||||
const deltas: Delta[] = [];
|
||||
const toolCalls = new Map<number, ToolCall>();
|
||||
const stream = createOpenAIToolCallStream({
|
||||
toolCalls,
|
||||
signal: signal.signal,
|
||||
emit: (delta) => deltas.push(delta),
|
||||
});
|
||||
stream.onRunStep(snapshot());
|
||||
const error = new Error('provider failed');
|
||||
const run = completeOpenAIToolCalls(stream, () => {
|
||||
if (failure === 'sync throw') throw error;
|
||||
if (failure === 'reject') return Promise.reject(error);
|
||||
signal.abort();
|
||||
return Promise.resolve();
|
||||
});
|
||||
if (failure === 'abort') await expect(run).rejects.toMatchObject({ name: 'AbortError' });
|
||||
else await expect(run).rejects.toBe(error);
|
||||
stream.onRunStep(snapshot('late'));
|
||||
stream.onRunStepDelta(streamsArgs('complete', 0, '{"city":"Paris"}'));
|
||||
expect(() => stream.finish()).toThrow('Agent response aborted');
|
||||
expect(toolCalls.size).toBe(0);
|
||||
expect(deltas).toEqual([]);
|
||||
const retry = streamingBridge();
|
||||
retry.stream.onRunStep(snapshot());
|
||||
retry.stream.finish();
|
||||
expect(retry.toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}');
|
||||
},
|
||||
);
|
||||
|
||||
it('does not report success when argument fragments could not be attributed', () => {
|
||||
const { stream } = streamingBridge();
|
||||
stream.onRunStepDelta(streamsArgs('unknown', 9, '{}'));
|
||||
expect(() => stream.finish()).toThrow('Unattributable tool call arguments');
|
||||
});
|
||||
|
||||
it('never repairs truncated streamed arguments with a complete snapshot', () => {
|
||||
const { stream, toolCalls } = streamingBridge();
|
||||
stream.onRunStep(snapshot());
|
||||
stream.onRunStepDelta(streamsArgs('complete', 0, '{"city":'));
|
||||
expect(() => stream.finish()).toThrow('Invalid tool call arguments');
|
||||
expect(toolCalls.size).toBe(0);
|
||||
});
|
||||
|
||||
it('fails incomplete identity rather than reporting a successful missing call', () => {
|
||||
const { stream } = streamingBridge();
|
||||
stream.onRunStepDelta({
|
||||
id: 'complete',
|
||||
delta: { type: 'tool_calls', tool_calls: [{ id: 'a', index: 0, args: '{}' }] },
|
||||
});
|
||||
expect(() => stream.finish()).toThrow('Incomplete tool call');
|
||||
});
|
||||
|
||||
it('validates all terminal snapshots before flushing any of them', () => {
|
||||
const { stream, deltas } = streamingBridge();
|
||||
stream.onRunStep(snapshot());
|
||||
stream.onRunStep(snapshot('bad', 'PRIVATE-INCOMPLETE-INPUT'));
|
||||
expect(() => stream.finish()).toThrow('Invalid tool call arguments in agent response');
|
||||
expect(deltas).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps a raw stream authoritative over an earlier parsed placeholder', () => {
|
||||
const { stream, deltas } = streamingBridge();
|
||||
stream.onRunStep(snapshot('complete', '{}'));
|
||||
stream.onRunStepDelta(streamsArgs('complete', 0, '{"city":"Madrid"}'));
|
||||
stream.finish();
|
||||
expect(accumulateLikeClient(deltas)[0].arguments).toBe('{"city":"Madrid"}');
|
||||
});
|
||||
|
||||
it.each(['wire', 'native'])('retains complete %s declarations without deltas', (shape) => {
|
||||
const { stream, deltas, toolCalls } = streamingBridge();
|
||||
stream.onRunStep({
|
||||
id: 'complete',
|
||||
stepDetails: {
|
||||
type: 'tool_calls',
|
||||
tool_calls: [
|
||||
shape === 'wire'
|
||||
? { id: 'a', function: { name: 'get_time', arguments: '{"city":"Madrid"}' } }
|
||||
: { id: 'a', name: 'get_time', args: { city: 'Madrid' } },
|
||||
],
|
||||
},
|
||||
});
|
||||
stream.finish();
|
||||
expect(toolCalls.get(0)?.function.arguments).toBe('{"city":"Madrid"}');
|
||||
expect(accumulateLikeClient(deltas)[0].arguments).toBe('{"city":"Madrid"}');
|
||||
});
|
||||
});
|
||||
|
||||
describe('finish reason for a response that called tools', () => {
|
||||
const context: OpenAIResponseContext = {
|
||||
requestId: 'chatcmpl-test',
|
||||
created: 1778317637,
|
||||
model: 'agent_test',
|
||||
};
|
||||
|
||||
it('preserves stop after a server-executed tool followed by final text', () => {
|
||||
const tracker = createOpenAIStreamTracker();
|
||||
const written: string[] = [];
|
||||
const stream = createOpenAIToolCallStream({
|
||||
toolCalls: tracker.toolCalls,
|
||||
emit: () => undefined,
|
||||
});
|
||||
|
||||
stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time'));
|
||||
|
||||
stream.finish();
|
||||
tracker.addText();
|
||||
sendFinalChunk({
|
||||
context,
|
||||
tracker,
|
||||
res: { write: (chunk: string) => written.push(chunk) } as never,
|
||||
});
|
||||
|
||||
const final = JSON.parse(written[0].replace(/^data: /, ''));
|
||||
expect(final.choices[0].finish_reason).toBe('stop');
|
||||
});
|
||||
|
||||
it('preserves stop in the final non-streaming answer after server tools', () => {
|
||||
const toolCalls = new Map<number, ToolCall>();
|
||||
const stream = createOpenAIToolCallStream({ toolCalls });
|
||||
|
||||
stream.onRunStep(runStep('step_1', 1, 'call_a', 'get_time'));
|
||||
|
||||
stream.finish();
|
||||
const response = buildNonStreamingResponse(context, 'Checking.', '', toolCalls, {
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
total_tokens: 0,
|
||||
});
|
||||
|
||||
expect(response.choices[0].finish_reason).toBe('stop');
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue