🌊 feat: Wire Adaptive Stream Smoothing Across Google, Bedrock, and Zero-Disable Semantics (#14660)

* 🌊 feat: Wire Adaptive Stream Smoothing Across Google, Bedrock, and Zero-Disable Semantics

With @librechat/agents 3.4.0 smoothing defaults ON (25ms adaptive) for
every provider; this completes the LibreChat side:

- google: fix the unguarded endpoints.all clobber and wire streamRate
  into llmConfig._lc_stream_delay — previously read from config and
  silently dropped, making google streamRate a no-op end to end
- bedrock: wire streamRate (endpoint + endpoints.all) — previously
  absent entirely, so bedrock smoothing was unreachable from config
- anthropic/openai: nullish guards so streamRate: 0 survives as the
  explicit smoothing disable; drop the azure 30/17 hardcoded fallback
  the SDK default now supersedes
- delete dead createHandleLLMNewToken (no call sites since #6886, and
  LangChain backgrounds callbacks so a sleep there never paced anything)
- schema: streamRate gains .min(0) and docs; example docs updated,
  including pairing STREAM_DELTA_COALESCE_MS with the smoothing tick

* 🩹 fix: Address Review — Keep Published Shim, Typed Delay Access, Scoped Docs

- restore createHandleLLMNewToken as a @deprecated compatibility shim:
  it ships in the public @librechat/api root, so removal is reserved for
  a major release
- assign llmConfig._lc_stream_delay via the SDK's typed property (3.4.0
  StreamSmoothingOptions) instead of Record<string, unknown> casts at
  all five sites
- scope the 25ms-default wording to agents SDK-backed providers (legacy
  Assistants/Ollama still per-chunk sleep at DEFAULT_STREAM_RATE=1) and
  clarify coalescing guidance when streamRate: 0

.env.example

@@ -1001,6 +1001,12 @@ HELP_AND_FAQ_URL=https://librechat.ai
 # Redis CPU at high token rates) at the cost of up to one window of added delivery
 # latency. Enable only after EVERY replica runs a build with batch-frame support;
 # older subscribers drop coalesced frames. Values are capped at 1000.
+# Keep the window <= the stream-smoothing cadence (`streamRate`, default 25ms):
+# each smoothing tick emits its pieces in one burst, so a tick-sized window
+# captures exactly one batch per tick; a larger window re-batches the paced
+# deltas and quantizes the smoothed cadence at delivery. With smoothing
+# disabled (`streamRate: 0`) there is no cadence to preserve — the window is
+# then purely the Redis-cost vs delivery-latency tradeoff described above.
 # STREAM_DELTA_COALESCE_MS=25
 
 # Single Redis instance

librechat.example.yaml

@@ -494,7 +494,9 @@ endpoints:
   # Anthropic endpoint configuration with Vertex AI support
   # Use this to run Anthropic Claude models through Google Cloud Vertex AI
   # anthropic:
-  #   # (optional) Stream rate limiting in milliseconds
+  #   # (optional) Override the adaptive stream-smoothing cadence in milliseconds.
+  #   # Agents SDK-backed providers smooth at 25ms by default; set 0 to disable
+  #   # smoothing. (Legacy Assistants/Ollama paths sleep this long per chunk instead.)
   #   streamRate: 20
   #   # (optional) Title model for conversation titles
   #   titleModel: claude-3.5-haiku  # Use the visible model name (key from models config)

packages/api/src/endpoints/anthropic/initialize.ts

@@ -85,12 +85,12 @@ export async function initializeAnthropic({
 
   const result = getLLMConfig(credentials, clientOptions);
 
-  if (anthropicConfig?.streamRate) {
-    (result.llmConfig as Record<string, unknown>)._lc_stream_delay = anthropicConfig.streamRate;
+  if (anthropicConfig?.streamRate != null) {
+    result.llmConfig._lc_stream_delay = anthropicConfig.streamRate;
   }
 
-  if (allConfig?.streamRate) {
-    (result.llmConfig as Record<string, unknown>)._lc_stream_delay = allConfig.streamRate;
+  if (allConfig?.streamRate != null) {
+    result.llmConfig._lc_stream_delay = allConfig.streamRate;
   }
 
   return result;

packages/api/src/endpoints/bedrock/initialize.spec.ts

@@ -1106,3 +1106,44 @@ describe('initializeBedrock', () => {
     });
   });
 });
+
+describe('initializeBedrock streamRate resolution', () => {
+  beforeEach(() => {
+    process.env.BEDROCK_AWS_ACCESS_KEY_ID = 'test-access-key';
+    process.env.BEDROCK_AWS_SECRET_ACCESS_KEY = 'test-secret-key';
+    process.env.BEDROCK_AWS_DEFAULT_REGION = 'us-east-1';
+  });
+
+  async function delayFor(config: Record<string, unknown>): Promise<unknown> {
+    const result = await initializeBedrock(createMockParams({ config }));
+    return (result.llmConfig as Record<string, unknown>)._lc_stream_delay;
+  }
+
+  it('wires `endpoints.bedrock.streamRate` into llmConfig._lc_stream_delay', async () => {
+    await expect(
+      delayFor({ endpoints: { [EModelEndpoint.bedrock]: { streamRate: 25 } } }),
+    ).resolves.toBe(25);
+  });
+
+  it('preserves the endpoint streamRate when `endpoints.all` exists without one', async () => {
+    await expect(
+      delayFor({
+        endpoints: { [EModelEndpoint.bedrock]: { streamRate: 25 }, all: { activityLabel: true } },
+      }),
+    ).resolves.toBe(25);
+  });
+
+  it('lets `endpoints.all.streamRate` (including 0) override the endpoint value', async () => {
+    await expect(
+      delayFor({
+        endpoints: { [EModelEndpoint.bedrock]: { streamRate: 25 }, all: { streamRate: 0 } },
+      }),
+    ).resolves.toBe(0);
+  });
+
+  it('leaves the delay unset when neither level configures a streamRate', async () => {
+    await expect(
+      delayFor({ endpoints: { all: { activityLabel: true } } }),
+    ).resolves.toBeUndefined();
+  });
+});

packages/api/src/endpoints/bedrock/initialize.ts

@@ -325,6 +325,14 @@ export async function initializeBedrock({
     }
   }
 
+  const streamRate =
+    appConfig?.endpoints?.all?.streamRate != null
+      ? appConfig.endpoints.all.streamRate
+      : (bedrockConfig?.streamRate as number | undefined);
+  if (streamRate != null) {
+    llmConfig._lc_stream_delay = streamRate;
+  }
+
   return {
     llmConfig,
     configOptions,

packages/api/src/endpoints/custom/initialize.ts

@@ -346,7 +346,7 @@ export async function initializeCustom({
 
   const streamRate = clientOptions.streamRate as number | undefined;
   if (streamRate != null) {
-    (options.llmConfig as Record<string, unknown>)._lc_stream_delay = streamRate;
+    options.llmConfig._lc_stream_delay = streamRate;
   }
 
   return options;

packages/api/src/endpoints/google/initialize.spec.ts

@@ -165,3 +165,60 @@ describe('initializeGoogle', () => {
     });
   });
 });
+
+describe('initializeGoogle streamRate resolution', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+    process.env.GOOGLE_KEY = 'test-api-key';
+  });
+
+  async function initWithConfig(
+    endpointsConfig: Record<string, unknown>,
+  ): Promise<Record<string, unknown>> {
+    const req = createReq();
+    (req as unknown as { config: Record<string, unknown> }).config = {
+      endpoints: endpointsConfig,
+    };
+    const result = await initializeGoogle({
+      req,
+      endpoint: EModelEndpoint.google,
+      model_parameters: { model: 'gemini-2.5-flash' },
+      db: createDb(),
+    });
+    return result.llmConfig as Record<string, unknown>;
+  }
+
+  it('wires the endpoint streamRate into llmConfig._lc_stream_delay', async () => {
+    const llmConfig = await initWithConfig({ [EModelEndpoint.google]: { streamRate: 25 } });
+    expect(llmConfig._lc_stream_delay).toBe(25);
+  });
+
+  it('preserves the endpoint streamRate when `endpoints.all` exists without one', async () => {
+    const llmConfig = await initWithConfig({
+      [EModelEndpoint.google]: { streamRate: 25 },
+      all: { activityLabel: true },
+    });
+    expect(llmConfig._lc_stream_delay).toBe(25);
+  });
+
+  it('lets `endpoints.all.streamRate` override the endpoint value', async () => {
+    const llmConfig = await initWithConfig({
+      [EModelEndpoint.google]: { streamRate: 25 },
+      all: { streamRate: 10 },
+    });
+    expect(llmConfig._lc_stream_delay).toBe(10);
+  });
+
+  it('lets `endpoints.all.streamRate: 0` disable smoothing explicitly', async () => {
+    const llmConfig = await initWithConfig({
+      [EModelEndpoint.google]: { streamRate: 25 },
+      all: { streamRate: 0 },
+    });
+    expect(llmConfig._lc_stream_delay).toBe(0);
+  });
+
+  it('leaves the delay unset when neither level configures a streamRate', async () => {
+    const llmConfig = await initWithConfig({ all: { activityLabel: true } });
+    expect(llmConfig._lc_stream_delay).toBeUndefined();
+  });
+});

packages/api/src/endpoints/google/initialize.ts

@@ -84,7 +84,7 @@ export async function initializeGoogle({
     clientOptions.titleModel = googleConfig.titleModel;
   }
 
-  if (allConfig) {
+  if (allConfig?.streamRate != null) {
     clientOptions.streamRate = allConfig.streamRate;
   }
 
@@ -124,5 +124,11 @@ export async function initializeGoogle({
     ...clientOptions,
   };
 
-  return getGoogleConfig(credentials, clientOptions);
+  const result = getGoogleConfig(credentials, clientOptions);
+
+  if (clientOptions.streamRate != null) {
+    result.llmConfig._lc_stream_delay = clientOptions.streamRate;
+  }
+
+  return result;
 }

packages/api/src/endpoints/openai/initialize.ts

@@ -194,21 +194,19 @@ export async function initializeOpenAI({
     (options as InitializeResultBase).useLegacyContent = true;
   }
 
-  const azureRate = modelName?.includes('gpt-4') ? 30 : 17;
-
   let streamRate: number | undefined;
 
   if (isAzureOpenAI && azureConfig) {
-    streamRate = azureConfig.streamRate ?? azureRate;
+    streamRate = azureConfig.streamRate;
   } else if (!isAzureOpenAI && openAIConfig) {
     streamRate = openAIConfig.streamRate;
   }
 
-  if (allConfig?.streamRate) {
+  if (allConfig?.streamRate != null) {
     streamRate = allConfig.streamRate;
   }
 
-  if (streamRate) {
+  if (streamRate != null) {
     options.llmConfig._lc_stream_delay = streamRate;
   }
 

packages/api/src/utils/generators.ts

@@ -92,6 +92,14 @@ export function createStreamEventHandlers(res: ServerResponse): {
   };
 }
 
+/**
+ * @deprecated No longer used internally: stream pacing lives in
+ * `@librechat/agents` (adaptive smoothing via `_lc_stream_delay` /
+ * `streamRate`), and LangChain runs callback handlers in the background, so
+ * a sleep here never paced token delivery. Kept as a compatibility shim for
+ * external `@librechat/api` consumers; removal is reserved for a major
+ * release.
+ */
 export function createHandleLLMNewToken(streamRate: number) {
   return async function (): Promise<void> {
     if (streamRate) {

packages/data-provider/src/config.ts

@@ -591,7 +591,14 @@ export const defaultAssistantsVersion = {
 };
 
 export const baseEndpointSchema = z.object({
-  streamRate: z.number().optional(),
+  /**
+   * Milliseconds between visible streamed chunks. Agents SDK-backed
+   * providers (openAI, custom, anthropic, google, bedrock, agents) smooth
+   * adaptively at 25ms by default; set to override the cadence, 0 to
+   * disable smoothing. Legacy Assistants and Ollama paths instead sleep
+   * this long per provider chunk (default 1ms), with no adaptive smoothing.
+   */
+  streamRate: z.number().min(0).optional(),
   baseURL: z.string().optional(),
   /**
    * Custom request headers forwarded to the provider on every request. Values