mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-08-27 04:07:05 +00:00
🚦 feat: Configurable Circuit Breakers for Runaway Streamed Tool Args (#14613)
* 🚦 feat: Configurable Circuit Breakers for Runaway Streamed Tool Args * docs: forewarn create_file about the streamed tool-argument limit The breaker failing a near-limit write should not be the model's first exposure to the bound. Both create_file variants now state the default 64 KB per-call limit and the incremental pattern (create the first section, extend with edit_file) in the tool description and the content parameter description. * fix: keep skill create_file description under the provider advisory cap The limit-guidance paragraph pushed the skill-aware description to 1169 chars, past the 1024-char advisory bound where providers may truncate. The skill variant now carries the guidance only in its content parameter description, which sits closest to the generated payload and is not at truncation risk; the shorter code-sandbox variant keeps the full paragraph. * 🚦 feat: per-tool streamed-arg limits with a create_file default Thirty days of production data show create_file is the only tool class with legitimate near-limit arguments (p99 80.6 KiB; every other tool p99 under 10 KiB). Rather than loosening the global 64 KiB cap for all tools, the yaml gains maxToolCallArgBytesByTool (per-tool overrides, keyed by model-facing tool name, 0 disables that tool's guard) and LibreChat ships { create_file: 131072 } by default; yaml entries merge over and can replace it. Pairs with maxToolCallArgBytesByTool support in the agents SDK and stays inert until the dependency bump. * test: pass per-tool spec configs as plain Partial literals The as-TAgentsEndpoint casts fail TS2352 for object-valued fields: comparability does not grant nested literals the implicit index signature that plain assignability does, so casts carrying maxToolCallArgBytesByTool never sufficiently overlap. The mapper already accepts Partial<TAgentsEndpoint>, so the new cases pass uncast literals instead. * chore(deps): bump @librechat/agents to 3.3.12
This commit is contained in:
Danny Avila
GitHub
parent
b807292997
commit
ccf4301093
9 changed files with 182 additions and 12 deletions
|
|
@ -46,7 +46,7 @@
|
|||
"@azure/storage-blob": "^12.30.0",
|
||||
"@google/genai": "^2.8.0",
|
||||
"@keyv/redis": "^4.3.3",
|
||||
"@librechat/agents": "^3.3.11",
|
||||
"@librechat/agents": "^3.3.12",
|
||||
"@librechat/api": "*",
|
||||
"@librechat/data-schemas": "*",
|
||||
"@microsoft/microsoft-graph-client": "^3.0.7",
|
||||
|
|
|
|||
|
|
@ -407,6 +407,17 @@ endpoints:
|
|||
# recursionLimit: 50
|
||||
# # (optional) Max recursion depth for agents, defaults to 25
|
||||
# maxRecursionLimit: 100
|
||||
# # (optional) Abort a run once a single streamed tool call's arguments exceed this many bytes.
|
||||
# # Guards against runaway malformed tool-call generation. Defaults to 65536 (64 KiB); 0 disables.
|
||||
# maxToolCallArgBytes: 65536
|
||||
# # (optional) Abort a run once a single model generation emits more than this many stream events.
|
||||
# # Defense in depth against looping provider streams. Disabled by default.
|
||||
# maxDeltaEventsPerTurn: 100000
|
||||
# # (optional) Per-tool overrides for maxToolCallArgBytes, keyed by tool name; 0 disables that
|
||||
# # tool's guard. LibreChat ships { create_file: 131072 } so document-sized file writes are not
|
||||
# # cut off; entries here merge over (and can replace) that default.
|
||||
# maxToolCallArgBytesByTool:
|
||||
# create_file: 131072
|
||||
# # (optional) Disable the builder interface for agents
|
||||
# disableBuilder: false
|
||||
# # (optional) When conversation titles are generated:
|
||||
|
|
|
|||
49
package-lock.json
generated
49
package-lock.json
generated
|
|
@ -61,7 +61,7 @@
|
|||
"@azure/storage-blob": "^12.30.0",
|
||||
"@google/genai": "^2.8.0",
|
||||
"@keyv/redis": "^4.3.3",
|
||||
"@librechat/agents": "^3.3.11",
|
||||
"@librechat/agents": "^3.3.12",
|
||||
"@librechat/api": "*",
|
||||
"@librechat/data-schemas": "*",
|
||||
"@microsoft/microsoft-graph-client": "^3.0.7",
|
||||
|
|
@ -10615,9 +10615,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@librechat/agents": {
|
||||
"version": "3.3.11",
|
||||
"resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-3.3.11.tgz",
|
||||
"integrity": "sha512-rCCUQbi6b2HcUfq11a/1FXR9JveCmV/30HH64TkVyVHoUkL7QlSd99rddaYRjnAcsGlWb1zY+0oHEO1jQ3m93A==",
|
||||
"version": "3.3.12",
|
||||
"resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-3.3.12.tgz",
|
||||
"integrity": "sha512-kTB50KMlEXR3jMLmGMZzzSkne019k57Wy3Fv0dGX7I4G/oxJMgqZsw70KqiAMYokNOQnc1DoWGmIcZLgJYcewA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sdk": "^0.115.0",
|
||||
|
|
@ -10653,6 +10653,7 @@
|
|||
"okapibm25": "^1.4.1",
|
||||
"openai": "^6.46.0",
|
||||
"reo-census": "^1.2.9",
|
||||
"socks-proxy-agent": "^8.0.5",
|
||||
"uuid": "^11.1.1"
|
||||
},
|
||||
"engines": {
|
||||
|
|
@ -38978,12 +38979,50 @@
|
|||
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/smart-buffer": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz",
|
||||
"integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 6.0.0",
|
||||
"npm": ">= 3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/smob": {
|
||||
"version": "1.4.1",
|
||||
"resolved": "https://registry.npmjs.org/smob/-/smob-1.4.1.tgz",
|
||||
"integrity": "sha512-9LK+E7Hv5R9u4g4C3p+jjLstaLe11MDsL21UpYaCNmapvMkYhqCV4A/f/3gyH8QjMyh6l68q9xC85vihY9ahMQ==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/socks": {
|
||||
"version": "2.8.9",
|
||||
"resolved": "https://registry.npmjs.org/socks/-/socks-2.8.9.tgz",
|
||||
"integrity": "sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ip-address": "^10.1.1",
|
||||
"smart-buffer": "^4.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 10.0.0",
|
||||
"npm": ">= 3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/socks-proxy-agent": {
|
||||
"version": "8.0.5",
|
||||
"resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz",
|
||||
"integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"agent-base": "^7.1.2",
|
||||
"debug": "^4.3.4",
|
||||
"socks": "^2.8.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 14"
|
||||
}
|
||||
},
|
||||
"node_modules/source-map": {
|
||||
"version": "0.6.1",
|
||||
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
|
||||
|
|
@ -42658,7 +42697,7 @@
|
|||
"@azure/storage-blob": "^12.30.0",
|
||||
"@google/genai": "^2.8.0",
|
||||
"@keyv/redis": "^4.3.3",
|
||||
"@librechat/agents": "^3.3.11",
|
||||
"@librechat/agents": "^3.3.12",
|
||||
"@librechat/data-schemas": "*",
|
||||
"@modelcontextprotocol/sdk": "^1.29.0",
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
|
|
|
|||
|
|
@ -107,7 +107,7 @@
|
|||
"@azure/storage-blob": "^12.30.0",
|
||||
"@google/genai": "^2.8.0",
|
||||
"@keyv/redis": "^4.3.3",
|
||||
"@librechat/agents": "^3.3.11",
|
||||
"@librechat/agents": "^3.3.12",
|
||||
"@librechat/data-schemas": "*",
|
||||
"@modelcontextprotocol/sdk": "^1.29.0",
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import type { TAgentsEndpoint } from 'librechat-data-provider';
|
||||
import { resolveRecursionLimit, resolveSubagentMaxTurns } from './config';
|
||||
import { resolveRecursionLimit, resolveStreamLimits, resolveSubagentMaxTurns } from './config';
|
||||
|
||||
describe('resolveRecursionLimit', () => {
|
||||
it('returns default 50 when no config or agent provided', () => {
|
||||
|
|
@ -111,3 +111,61 @@ describe('resolveSubagentMaxTurns', () => {
|
|||
expect(turns * 3).toBeLessThanOrEqual(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveStreamLimits', () => {
|
||||
const CREATE_FILE_DEFAULT = { create_file: 131072 };
|
||||
|
||||
it('ships only the create_file override when no yaml fields are set, so SDK defaults apply', () => {
|
||||
expect(resolveStreamLimits(undefined)).toEqual({
|
||||
maxToolCallArgBytesByTool: CREATE_FILE_DEFAULT,
|
||||
});
|
||||
expect(resolveStreamLimits({} as TAgentsEndpoint)).toEqual({
|
||||
maxToolCallArgBytesByTool: CREATE_FILE_DEFAULT,
|
||||
});
|
||||
});
|
||||
|
||||
it('maps both global yaml fields onto the SDK streamLimits shape', () => {
|
||||
const config = {
|
||||
maxToolCallArgBytes: 131072,
|
||||
maxDeltaEventsPerTurn: 100000,
|
||||
} as TAgentsEndpoint;
|
||||
expect(resolveStreamLimits(config)).toEqual({
|
||||
maxToolCallArgBytes: 131072,
|
||||
maxToolCallArgBytesByTool: CREATE_FILE_DEFAULT,
|
||||
maxDeltaEventsPerTurn: 100000,
|
||||
});
|
||||
});
|
||||
|
||||
it('passes each global field independently, omitting the unset one', () => {
|
||||
expect(resolveStreamLimits({ maxToolCallArgBytes: 1024 } as TAgentsEndpoint)).toEqual({
|
||||
maxToolCallArgBytes: 1024,
|
||||
maxToolCallArgBytesByTool: CREATE_FILE_DEFAULT,
|
||||
});
|
||||
expect(resolveStreamLimits({ maxDeltaEventsPerTurn: 5000 } as TAgentsEndpoint)).toEqual({
|
||||
maxToolCallArgBytesByTool: CREATE_FILE_DEFAULT,
|
||||
maxDeltaEventsPerTurn: 5000,
|
||||
});
|
||||
});
|
||||
|
||||
it('passes an explicit 0 through so admins can disable the SDK default', () => {
|
||||
expect(resolveStreamLimits({ maxToolCallArgBytes: 0 } as TAgentsEndpoint)).toEqual({
|
||||
maxToolCallArgBytes: 0,
|
||||
maxToolCallArgBytesByTool: CREATE_FILE_DEFAULT,
|
||||
});
|
||||
});
|
||||
|
||||
it('merges yaml per-tool entries over the shipped create_file default', () => {
|
||||
expect(resolveStreamLimits({ maxToolCallArgBytesByTool: { my_mcp_tool: 32768 } })).toEqual({
|
||||
maxToolCallArgBytesByTool: { create_file: 131072, my_mcp_tool: 32768 },
|
||||
});
|
||||
});
|
||||
|
||||
it('lets a yaml create_file entry replace the shipped default, including 0 to disable', () => {
|
||||
expect(resolveStreamLimits({ maxToolCallArgBytesByTool: { create_file: 262144 } })).toEqual({
|
||||
maxToolCallArgBytesByTool: { create_file: 262144 },
|
||||
});
|
||||
expect(resolveStreamLimits({ maxToolCallArgBytesByTool: { create_file: 0 } })).toEqual({
|
||||
maxToolCallArgBytesByTool: { create_file: 0 },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import type { TAgentsEndpoint } from 'librechat-data-provider';
|
||||
import { CREATE_FILE_TOOL_NAME } from '~/agents/tools';
|
||||
|
||||
const DEFAULT_RECURSION_LIMIT = 50;
|
||||
|
||||
|
|
@ -59,3 +60,43 @@ export function resolveSubagentMaxTurns(
|
|||
const limit = resolveRecursionLimit(agentsEConfig, agent);
|
||||
return Math.floor(limit / SUBAGENT_RECURSION_MULTIPLIER);
|
||||
}
|
||||
|
||||
/** Mirrors `StreamLimits` in `@librechat/agents` (agents#381). */
|
||||
export interface StreamLimitsConfig {
|
||||
maxToolCallArgBytes?: number;
|
||||
maxToolCallArgBytesByTool?: Record<string, number>;
|
||||
maxDeltaEventsPerTurn?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* LibreChat's shipped per-tool override: create_file legitimately streams
|
||||
* whole documents as its content argument (production p99 of 80.6 KiB versus
|
||||
* under 10 KiB for every other tool class), so it runs at twice the SDK's
|
||||
* 64 KiB global default instead of loosening the cap for all tools.
|
||||
*/
|
||||
const CREATE_FILE_MAX_TOOL_CALL_ARG_BYTES = 131_072;
|
||||
|
||||
/**
|
||||
* Maps the librechat.yaml stream circuit-breaker fields
|
||||
* (`endpoints.agents.maxToolCallArgBytes` / `maxToolCallArgBytesByTool` /
|
||||
* `maxDeltaEventsPerTurn`) to the SDK's `RunConfig.streamLimits`. Unset
|
||||
* global fields keep the SDK defaults (64 KiB per streamed tool call's
|
||||
* arguments, per-turn delta event cap off), while the per-tool map always
|
||||
* ships the create_file override; a yaml entry for the same tool wins. Value
|
||||
* normalization (0 disables, NaN falls back) lives in the SDK's
|
||||
* `resolveStreamLimits`.
|
||||
*/
|
||||
export function resolveStreamLimits(
|
||||
agentsEConfig: Partial<TAgentsEndpoint> | undefined,
|
||||
): StreamLimitsConfig {
|
||||
const maxToolCallArgBytes = agentsEConfig?.maxToolCallArgBytes;
|
||||
const maxDeltaEventsPerTurn = agentsEConfig?.maxDeltaEventsPerTurn;
|
||||
return {
|
||||
...(maxToolCallArgBytes != null && { maxToolCallArgBytes }),
|
||||
maxToolCallArgBytesByTool: {
|
||||
[CREATE_FILE_TOOL_NAME]: CREATE_FILE_MAX_TOOL_CALL_ARG_BYTES,
|
||||
...agentsEConfig?.maxToolCallArgBytesByTool,
|
||||
},
|
||||
...(maxDeltaEventsPerTurn != null && { maxDeltaEventsPerTurn }),
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ import { applyCustomHandoffPromptKeyCompatibility } from '~/agents/handoffPrompt
|
|||
import { stripIntentFromToolRegistry, stripIntentFromToolDefinitions } from '~/agents/intent';
|
||||
import { isSteeringSupported, isSteerPreemptSupported } from '~/agents/steering/runtime';
|
||||
import { getLLMConfig as getAnthropicLLMConfig } from '~/endpoints/anthropic/llm';
|
||||
import { resolveStreamLimits, resolveSubagentMaxTurns } from '~/agents/config';
|
||||
import { CREATE_FILE_TOOL_NAME, EDIT_FILE_TOOL_NAME } from '~/agents/tools';
|
||||
import { getProviderConfig } from '~/endpoints/config/providers';
|
||||
import { extractDefaultParams } from '~/endpoints/openai/llm';
|
||||
|
|
@ -59,7 +60,6 @@ import { resolveHeaders, createSafeUser } from '~/utils/env';
|
|||
import { getAgentCheckpointer } from '~/agents/checkpointer';
|
||||
import { getOpenAIConfig } from '~/endpoints/openai/config';
|
||||
import { buildHITLRunWiring } from '~/agents/hitl/runtime';
|
||||
import { resolveSubagentMaxTurns } from '~/agents/config';
|
||||
import { buildLangfuseConfig } from '~/langfuse/config';
|
||||
import { resolveConfigHeaders } from '~/utils/headers';
|
||||
import { applyTestRunHook } from '~/agents/testHook';
|
||||
|
|
@ -1540,6 +1540,8 @@ export async function createRun({
|
|||
}
|
||||
}
|
||||
|
||||
const streamLimits = resolveStreamLimits(agentsEndpointConfig);
|
||||
|
||||
/**
|
||||
* Built as a variable (not an inline literal) so the extra
|
||||
* `subagentUsageSink` field passes assignability against SDK versions
|
||||
|
|
@ -1643,6 +1645,12 @@ export async function createRun({
|
|||
// live, so gating both on the same capability keeps them in lockstep.
|
||||
...(steering?.preemption != null &&
|
||||
isSteerPreemptSupported() && { preemption: steering.preemption }),
|
||||
// Stream circuit breakers (librechat.yaml endpoints.agents.maxToolCallArgBytes /
|
||||
// maxDeltaEventsPerTurn). Omitted when unset so the SDK defaults apply: a runaway
|
||||
// streamed tool-call argument aborts the run at 64 KiB, the per-turn delta event
|
||||
// cap stays off. Requires @librechat/agents with streamLimits support (agents#381);
|
||||
// older versions ignore the field.
|
||||
...(streamLimits && { streamLimits }),
|
||||
};
|
||||
const run = await Run.create(runConfig);
|
||||
|
||||
|
|
|
|||
|
|
@ -181,7 +181,8 @@ const SKILL_CREATE_FILE_PARAMETERS: LCTool['parameters'] = Object.freeze({
|
|||
},
|
||||
content: {
|
||||
type: 'string',
|
||||
description: 'Complete file contents.',
|
||||
description:
|
||||
'Complete file contents. Keep a single call well under the streamed tool-argument limit (64 KB by default); build larger files incrementally with edit_file.',
|
||||
},
|
||||
overwrite: {
|
||||
type: 'boolean',
|
||||
|
|
@ -202,7 +203,8 @@ const CODE_CREATE_FILE_PARAMETERS: LCTool['parameters'] = Object.freeze({
|
|||
},
|
||||
content: {
|
||||
type: 'string',
|
||||
description: 'Complete file contents.',
|
||||
description:
|
||||
'Complete file contents. Keep a single call well under the streamed tool-argument limit (64 KB by default); build larger files incrementally with edit_file.',
|
||||
},
|
||||
overwrite: {
|
||||
type: 'boolean',
|
||||
|
|
@ -295,7 +297,9 @@ const CODE_CREATE_FILE_DESCRIPTION = `Create a new file, or overwrite an existin
|
|||
|
||||
Use for new files and full rewrites where the change is larger than half the file. Requires overwrite: true to replace existing files. Refuses otherwise.
|
||||
|
||||
Targets code-execution sandbox paths. Prefer /mnt/data/{file} for files that should remain available to later sandbox calls.`;
|
||||
Targets code-execution sandbox paths. Prefer /mnt/data/{file} for files that should remain available to later sandbox calls.
|
||||
|
||||
Very long content can exceed the streamed tool-argument limit (64 KB by default) and fail the call. For large files, create the file with its first section, then extend it with edit_file.`;
|
||||
|
||||
const SKILL_EDIT_FILE_DESCRIPTION = `Apply targeted text replacements to an existing file.
|
||||
|
||||
|
|
|
|||
|
|
@ -916,6 +916,15 @@ export const agentsEndpointSchema = baseEndpointSchema
|
|||
recursionLimit: z.number().optional(),
|
||||
disableBuilder: z.boolean().optional().default(false),
|
||||
maxRecursionLimit: z.number().optional(),
|
||||
/** Max cumulative bytes a single streamed tool call's arguments may reach before the run
|
||||
* aborts. Defaults to 64 KiB in the agents SDK; `0` disables the guard. */
|
||||
maxToolCallArgBytes: z.number().optional(),
|
||||
/** Max streamed chunk events per model generation before the run aborts. Off by default. */
|
||||
maxDeltaEventsPerTurn: z.number().optional(),
|
||||
/** Per-tool overrides of `maxToolCallArgBytes`, keyed by model-facing tool name; `0`
|
||||
* disables the guard for that tool only. Merged over LibreChat's shipped default of
|
||||
* `{ create_file: 131072 }`. */
|
||||
maxToolCallArgBytesByTool: z.record(z.number()).optional(),
|
||||
maxCitations: z.number().min(1).max(50).optional().default(30),
|
||||
maxCitationsPerFile: z.number().min(1).max(10).optional().default(7),
|
||||
minRelevanceScore: z.number().min(0.0).max(1.0).optional().default(0.45),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue