From 33c8801ba6e515f8e3514174192ccc0cb970e78e Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 6 Aug 2026 12:39:12 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=8C=8A=20feat:=20Wire=20Adaptive=20Stream?= =?UTF-8?q?=20Smoothing=20Across=20Google,=20Bedrock,=20and=20Zero-Disable?= =?UTF-8?q?=20Semantics=20(#14660)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🌊 feat: Wire Adaptive Stream Smoothing Across Google, Bedrock, and Zero-Disable Semantics With @librechat/agents 3.4.0 smoothing defaults ON (25ms adaptive) for every provider; this completes the LibreChat side: - google: fix the unguarded endpoints.all clobber and wire streamRate into llmConfig._lc_stream_delay — previously read from config and silently dropped, making google streamRate a no-op end to end - bedrock: wire streamRate (endpoint + endpoints.all) — previously absent entirely, so bedrock smoothing was unreachable from config - anthropic/openai: nullish guards so streamRate: 0 survives as the explicit smoothing disable; drop the azure 30/17 hardcoded fallback the SDK default now supersedes - delete dead createHandleLLMNewToken (no call sites since #6886, and LangChain backgrounds callbacks so a sleep there never paced anything) - schema: streamRate gains .min(0) and docs; example docs updated, including pairing STREAM_DELTA_COALESCE_MS with the smoothing tick * 🩹 fix: Address Review — Keep Published Shim, Typed Delay Access, Scoped Docs - restore createHandleLLMNewToken as a @deprecated compatibility shim: it ships in the public @librechat/api root, so removal is reserved for a major release - assign llmConfig._lc_stream_delay via the SDK's typed property (3.4.0 StreamSmoothingOptions) instead of Record casts at all five sites - scope the 25ms-default wording to agents SDK-backed providers (legacy Assistants/Ollama still per-chunk sleep at DEFAULT_STREAM_RATE=1) and clarify coalescing guidance when streamRate: 0 --- .env.example | 6 ++ librechat.example.yaml | 4 +- .../api/src/endpoints/anthropic/initialize.ts | 8 +-- .../src/endpoints/bedrock/initialize.spec.ts | 41 +++++++++++++ .../api/src/endpoints/bedrock/initialize.ts | 8 +++ .../api/src/endpoints/custom/initialize.ts | 2 +- .../src/endpoints/google/initialize.spec.ts | 57 +++++++++++++++++++ .../api/src/endpoints/google/initialize.ts | 10 +++- .../api/src/endpoints/openai/initialize.ts | 8 +-- packages/api/src/utils/generators.ts | 8 +++ packages/data-provider/src/config.ts | 9 ++- 11 files changed, 147 insertions(+), 14 deletions(-) diff --git a/.env.example b/.env.example index 2f2eb94067..a38ee4d660 100644 --- a/.env.example +++ b/.env.example @@ -1001,6 +1001,12 @@ HELP_AND_FAQ_URL=https://librechat.ai # Redis CPU at high token rates) at the cost of up to one window of added delivery # latency. Enable only after EVERY replica runs a build with batch-frame support; # older subscribers drop coalesced frames. Values are capped at 1000. +# Keep the window <= the stream-smoothing cadence (`streamRate`, default 25ms): +# each smoothing tick emits its pieces in one burst, so a tick-sized window +# captures exactly one batch per tick; a larger window re-batches the paced +# deltas and quantizes the smoothed cadence at delivery. With smoothing +# disabled (`streamRate: 0`) there is no cadence to preserve — the window is +# then purely the Redis-cost vs delivery-latency tradeoff described above. # STREAM_DELTA_COALESCE_MS=25 # Single Redis instance diff --git a/librechat.example.yaml b/librechat.example.yaml index b8f6ce4bbb..423894017c 100644 --- a/librechat.example.yaml +++ b/librechat.example.yaml @@ -494,7 +494,9 @@ endpoints: # Anthropic endpoint configuration with Vertex AI support # Use this to run Anthropic Claude models through Google Cloud Vertex AI # anthropic: - # # (optional) Stream rate limiting in milliseconds + # # (optional) Override the adaptive stream-smoothing cadence in milliseconds. + # # Agents SDK-backed providers smooth at 25ms by default; set 0 to disable + # # smoothing. (Legacy Assistants/Ollama paths sleep this long per chunk instead.) # streamRate: 20 # # (optional) Title model for conversation titles # titleModel: claude-3.5-haiku # Use the visible model name (key from models config) diff --git a/packages/api/src/endpoints/anthropic/initialize.ts b/packages/api/src/endpoints/anthropic/initialize.ts index 94a86294a2..232ff80405 100644 --- a/packages/api/src/endpoints/anthropic/initialize.ts +++ b/packages/api/src/endpoints/anthropic/initialize.ts @@ -85,12 +85,12 @@ export async function initializeAnthropic({ const result = getLLMConfig(credentials, clientOptions); - if (anthropicConfig?.streamRate) { - (result.llmConfig as Record)._lc_stream_delay = anthropicConfig.streamRate; + if (anthropicConfig?.streamRate != null) { + result.llmConfig._lc_stream_delay = anthropicConfig.streamRate; } - if (allConfig?.streamRate) { - (result.llmConfig as Record)._lc_stream_delay = allConfig.streamRate; + if (allConfig?.streamRate != null) { + result.llmConfig._lc_stream_delay = allConfig.streamRate; } return result; diff --git a/packages/api/src/endpoints/bedrock/initialize.spec.ts b/packages/api/src/endpoints/bedrock/initialize.spec.ts index 3e7c4eaf63..8bbd3d5f64 100644 --- a/packages/api/src/endpoints/bedrock/initialize.spec.ts +++ b/packages/api/src/endpoints/bedrock/initialize.spec.ts @@ -1106,3 +1106,44 @@ describe('initializeBedrock', () => { }); }); }); + +describe('initializeBedrock streamRate resolution', () => { + beforeEach(() => { + process.env.BEDROCK_AWS_ACCESS_KEY_ID = 'test-access-key'; + process.env.BEDROCK_AWS_SECRET_ACCESS_KEY = 'test-secret-key'; + process.env.BEDROCK_AWS_DEFAULT_REGION = 'us-east-1'; + }); + + async function delayFor(config: Record): Promise { + const result = await initializeBedrock(createMockParams({ config })); + return (result.llmConfig as Record)._lc_stream_delay; + } + + it('wires `endpoints.bedrock.streamRate` into llmConfig._lc_stream_delay', async () => { + await expect( + delayFor({ endpoints: { [EModelEndpoint.bedrock]: { streamRate: 25 } } }), + ).resolves.toBe(25); + }); + + it('preserves the endpoint streamRate when `endpoints.all` exists without one', async () => { + await expect( + delayFor({ + endpoints: { [EModelEndpoint.bedrock]: { streamRate: 25 }, all: { activityLabel: true } }, + }), + ).resolves.toBe(25); + }); + + it('lets `endpoints.all.streamRate` (including 0) override the endpoint value', async () => { + await expect( + delayFor({ + endpoints: { [EModelEndpoint.bedrock]: { streamRate: 25 }, all: { streamRate: 0 } }, + }), + ).resolves.toBe(0); + }); + + it('leaves the delay unset when neither level configures a streamRate', async () => { + await expect( + delayFor({ endpoints: { all: { activityLabel: true } } }), + ).resolves.toBeUndefined(); + }); +}); diff --git a/packages/api/src/endpoints/bedrock/initialize.ts b/packages/api/src/endpoints/bedrock/initialize.ts index 09307dfba0..7686da24dd 100644 --- a/packages/api/src/endpoints/bedrock/initialize.ts +++ b/packages/api/src/endpoints/bedrock/initialize.ts @@ -325,6 +325,14 @@ export async function initializeBedrock({ } } + const streamRate = + appConfig?.endpoints?.all?.streamRate != null + ? appConfig.endpoints.all.streamRate + : (bedrockConfig?.streamRate as number | undefined); + if (streamRate != null) { + llmConfig._lc_stream_delay = streamRate; + } + return { llmConfig, configOptions, diff --git a/packages/api/src/endpoints/custom/initialize.ts b/packages/api/src/endpoints/custom/initialize.ts index 0d334878ec..a896ec6966 100644 --- a/packages/api/src/endpoints/custom/initialize.ts +++ b/packages/api/src/endpoints/custom/initialize.ts @@ -346,7 +346,7 @@ export async function initializeCustom({ const streamRate = clientOptions.streamRate as number | undefined; if (streamRate != null) { - (options.llmConfig as Record)._lc_stream_delay = streamRate; + options.llmConfig._lc_stream_delay = streamRate; } return options; diff --git a/packages/api/src/endpoints/google/initialize.spec.ts b/packages/api/src/endpoints/google/initialize.spec.ts index a6239805ef..44cf9994b5 100644 --- a/packages/api/src/endpoints/google/initialize.spec.ts +++ b/packages/api/src/endpoints/google/initialize.spec.ts @@ -165,3 +165,60 @@ describe('initializeGoogle', () => { }); }); }); + +describe('initializeGoogle streamRate resolution', () => { + beforeEach(() => { + jest.clearAllMocks(); + process.env.GOOGLE_KEY = 'test-api-key'; + }); + + async function initWithConfig( + endpointsConfig: Record, + ): Promise> { + const req = createReq(); + (req as unknown as { config: Record }).config = { + endpoints: endpointsConfig, + }; + const result = await initializeGoogle({ + req, + endpoint: EModelEndpoint.google, + model_parameters: { model: 'gemini-2.5-flash' }, + db: createDb(), + }); + return result.llmConfig as Record; + } + + it('wires the endpoint streamRate into llmConfig._lc_stream_delay', async () => { + const llmConfig = await initWithConfig({ [EModelEndpoint.google]: { streamRate: 25 } }); + expect(llmConfig._lc_stream_delay).toBe(25); + }); + + it('preserves the endpoint streamRate when `endpoints.all` exists without one', async () => { + const llmConfig = await initWithConfig({ + [EModelEndpoint.google]: { streamRate: 25 }, + all: { activityLabel: true }, + }); + expect(llmConfig._lc_stream_delay).toBe(25); + }); + + it('lets `endpoints.all.streamRate` override the endpoint value', async () => { + const llmConfig = await initWithConfig({ + [EModelEndpoint.google]: { streamRate: 25 }, + all: { streamRate: 10 }, + }); + expect(llmConfig._lc_stream_delay).toBe(10); + }); + + it('lets `endpoints.all.streamRate: 0` disable smoothing explicitly', async () => { + const llmConfig = await initWithConfig({ + [EModelEndpoint.google]: { streamRate: 25 }, + all: { streamRate: 0 }, + }); + expect(llmConfig._lc_stream_delay).toBe(0); + }); + + it('leaves the delay unset when neither level configures a streamRate', async () => { + const llmConfig = await initWithConfig({ all: { activityLabel: true } }); + expect(llmConfig._lc_stream_delay).toBeUndefined(); + }); +}); diff --git a/packages/api/src/endpoints/google/initialize.ts b/packages/api/src/endpoints/google/initialize.ts index 833956ad1a..4543171875 100644 --- a/packages/api/src/endpoints/google/initialize.ts +++ b/packages/api/src/endpoints/google/initialize.ts @@ -84,7 +84,7 @@ export async function initializeGoogle({ clientOptions.titleModel = googleConfig.titleModel; } - if (allConfig) { + if (allConfig?.streamRate != null) { clientOptions.streamRate = allConfig.streamRate; } @@ -124,5 +124,11 @@ export async function initializeGoogle({ ...clientOptions, }; - return getGoogleConfig(credentials, clientOptions); + const result = getGoogleConfig(credentials, clientOptions); + + if (clientOptions.streamRate != null) { + result.llmConfig._lc_stream_delay = clientOptions.streamRate; + } + + return result; } diff --git a/packages/api/src/endpoints/openai/initialize.ts b/packages/api/src/endpoints/openai/initialize.ts index 9e9418e037..3423a87584 100644 --- a/packages/api/src/endpoints/openai/initialize.ts +++ b/packages/api/src/endpoints/openai/initialize.ts @@ -194,21 +194,19 @@ export async function initializeOpenAI({ (options as InitializeResultBase).useLegacyContent = true; } - const azureRate = modelName?.includes('gpt-4') ? 30 : 17; - let streamRate: number | undefined; if (isAzureOpenAI && azureConfig) { - streamRate = azureConfig.streamRate ?? azureRate; + streamRate = azureConfig.streamRate; } else if (!isAzureOpenAI && openAIConfig) { streamRate = openAIConfig.streamRate; } - if (allConfig?.streamRate) { + if (allConfig?.streamRate != null) { streamRate = allConfig.streamRate; } - if (streamRate) { + if (streamRate != null) { options.llmConfig._lc_stream_delay = streamRate; } diff --git a/packages/api/src/utils/generators.ts b/packages/api/src/utils/generators.ts index 2e0416cb89..2466e4d3b3 100644 --- a/packages/api/src/utils/generators.ts +++ b/packages/api/src/utils/generators.ts @@ -92,6 +92,14 @@ export function createStreamEventHandlers(res: ServerResponse): { }; } +/** + * @deprecated No longer used internally: stream pacing lives in + * `@librechat/agents` (adaptive smoothing via `_lc_stream_delay` / + * `streamRate`), and LangChain runs callback handlers in the background, so + * a sleep here never paced token delivery. Kept as a compatibility shim for + * external `@librechat/api` consumers; removal is reserved for a major + * release. + */ export function createHandleLLMNewToken(streamRate: number) { return async function (): Promise { if (streamRate) { diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 33328ab0f6..73861605f3 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -591,7 +591,14 @@ export const defaultAssistantsVersion = { }; export const baseEndpointSchema = z.object({ - streamRate: z.number().optional(), + /** + * Milliseconds between visible streamed chunks. Agents SDK-backed + * providers (openAI, custom, anthropic, google, bedrock, agents) smooth + * adaptively at 25ms by default; set to override the cadence, 0 to + * disable smoothing. Legacy Assistants and Ollama paths instead sleep + * this long per provider chunk (default 1ms), with no adaptive smoothing. + */ + streamRate: z.number().min(0).optional(), baseURL: z.string().optional(), /** * Custom request headers forwarded to the provider on every request. Values