From 2f67471b65205acf4d00f403c997fc60226458cf Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 13:04:43 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=90=20perf:=20Enable=20Redis=20Stream?= =?UTF-8?q?=20Coalescing=20by=20Default=20(#16093)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🚐 perf: Enable Redis Stream Coalescing by Default * fix: Preserve Legacy Chunk Frames in Coalesced Publications * fix: Align Coalesced Publication and Append Size Thresholds --------- Co-authored-by: Lia --- .env.example | 12 +- UPGRADING.md | 24 +++ .../api/src/stream/GenerationJobManager.ts | 5 +- .../stream/__tests__/deltaCoalescing.spec.ts | 106 ++++++++-- ...deltaCoalescing.stream_integration.spec.ts | 200 ++++++++++++++++-- .../src/stream/__tests__/helpers/publisher.ts | 15 +- .../implementations/RedisEventTransport.ts | 53 +++-- .../api/src/stream/internal/coalescing.ts | 6 +- 8 files changed, 368 insertions(+), 53 deletions(-) diff --git a/.env.example b/.env.example index b310ffe2cc..fa196f5e3c 100644 --- a/.env.example +++ b/.env.example @@ -1232,11 +1232,13 @@ HELP_AND_FAQ_URL=https://librechat.ai # SCHEDULES_DISABLED=true # Coalesce streamed model/tool-argument deltas into windowed Redis publications (ms). -# Unset or 0 (default) publishes per delta. 25 is recommended: it batches the publish -# EVAL and the durable append across the window (fewer Redis round trips and lower -# Redis CPU at high token rates) at the cost of up to one window of added delivery -# latency. Enable only after EVERY replica runs a build with batch-frame support; -# older subscribers drop coalesced frames. Values are capped at 1000. +# Defaults to 25ms when unset; explicit 0 publishes per delta. Batches both publish +# and durable append operations (fewer Redis round trips and repeated guard/TTL work at high +# token rates), adding up to one window of buffering latency and crash-loss exposure +# for unflushed deltas. In-memory streams are unchanged. Values are capped at 1000. +# Batched publications retain individual chunk frames for older subscribers; Pub/Sub +# message count is unchanged. Incoming chunk_batch frames remain supported for existing +# opt-in producers. See UPGRADING.md for compatibility guidance. # Keep the window <= the stream-smoothing cadence (`streamRate`, default 25ms): # each smoothing tick emits its pieces in one burst, so a tick-sized window # captures exactly one batch per tick; a larger window re-batches the paced diff --git a/UPGRADING.md b/UPGRADING.md index 51eab7cc67..aecd00630c 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -1,5 +1,29 @@ # Upgrading LibreChat +## Redis streaming now coalesces deltas by default + +Redis-backed streams now batch model and tool-argument deltas in a **25 ms** +window when `STREAM_DELTA_COALESCE_MS` is unset. Explicit `0` keeps per-delta +publication; existing explicit values retain their behavior. In-memory streams +are unchanged, including deployments using `USE_REDIS_STREAMS=false`. + +The same window batches durable appends and publications. It reduces Redis +round trips and repeated guard/TTL work, but not Pub/Sub message count at the cost of up to one window of buffering latency and possible loss +of unflushed deltas on process crash. Terminal and non-coalescable barriers still +flush pending batches before proceeding. + +Coalescing batches Redis requests, not the subscriber wire format: each event +still publishes as an individually sequenced `chunk` frame. Subscribers from +before batch-frame support can read these publications without a preparatory +configuration change. New subscribers also retain `chunk_batch` decoding for +interoperation with existing opt-in batching producers. + +This removes the new default's batch-frame compatibility hazard; it does not +promise compatibility across unrelated generation-protocol changes. If an +existing producer already emits `chunk_batch` frames through explicit opt-in, +keep those producers away from subscribers that predate batch-frame support, +or disable coalescing on those producers before introducing older subscribers. + ## Tenant index migration (v0.8.7 and earlier databases) Upgrading an existing database can log `Index build failed` for User, Role, diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts index 7c65136c22..a547193cc0 100644 --- a/packages/api/src/stream/GenerationJobManager.ts +++ b/packages/api/src/stream/GenerationJobManager.ts @@ -6734,10 +6734,13 @@ class GenerationJobManagerClass { * One decision drives both durable-log and publish batching: the append and * the sequence allocation for an event must stay tightly coupled in time, or * the resume frontier (chunk-log snapshot → sequence-counter sync) misreads - * a window's tail as already-delivered or as duplicates. + * a window's tail as already-delivered or as duplicates. Before first-subscriber + * admission, each append is awaited for snapshot safety; batching that path + * would add a full window of producer latency to every delta. */ const coalescableDelta = this._deltaCoalescingEnabled && + (runtime.hasSubscriber || runtime.everHadSubscriber) && !runtime.startupTelemetry && options?.durable !== true && options?.deliveredSteer == null && diff --git a/packages/api/src/stream/__tests__/deltaCoalescing.spec.ts b/packages/api/src/stream/__tests__/deltaCoalescing.spec.ts index 42ebfde2b7..c2822d3f5e 100644 --- a/packages/api/src/stream/__tests__/deltaCoalescing.spec.ts +++ b/packages/api/src/stream/__tests__/deltaCoalescing.spec.ts @@ -4,6 +4,7 @@ import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTr import { registerChunkPublicationCapability } from '~/stream/internal/chunkPublication'; import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore'; import { GenerationJobManagerClass } from '~/stream/GenerationJobManager'; +import { resolveCoalesceWindowMs } from '~/stream/internal/coalescing'; jest.spyOn(console, 'log').mockImplementation(); @@ -37,6 +38,7 @@ describe('delta coalescing manager gating and backpressure', () => { async function createManager(options: { windowMs: string | undefined; advertise: boolean; + isRedis?: boolean; }): Promise<{ manager: GenerationJobManagerClass; publishCalls: Array; @@ -64,7 +66,7 @@ describe('delta coalescing manager gating and backpressure', () => { }); const manager = new GenerationJobManagerClass(); - manager.configure({ jobStore, eventTransport, isRedis: true }); + manager.configure({ jobStore, eventTransport, isRedis: options.isRedis ?? true }); manager.initialize(); return { manager, @@ -83,6 +85,7 @@ describe('delta coalescing manager gating and backpressure', () => { advertise: false, }); await manager.createJob('gate-off', 'user-1', 'gate-off'); + await manager.subscribe('gate-off', jest.fn()); const emission = manager.emitChunk('gate-off', DELTA_EVENT); await new Promise((resolve) => setImmediate(resolve)); @@ -105,27 +108,62 @@ describe('delta coalescing manager gating and backpressure', () => { await manager.destroy(); }); - it('sends coalesce hints and stops awaiting receipts when both services advertise support', async () => { - const { manager, publishCalls, settleAll } = await createManager({ - windowMs: '25', - advertise: true, - }); - await manager.createJob('gate-on', 'user-1', 'gate-on'); + it.each([undefined, '25'])( + 'sends coalesce hints without awaiting receipts with capable Redis services and window %s', + async (windowMs) => { + const { manager, publishCalls, settleAll } = await createManager({ + windowMs, + advertise: true, + }); + await manager.createJob('gate-on', 'user-1', 'gate-on'); + await manager.subscribe('gate-on', jest.fn()); - await manager.emitChunk('gate-on', DELTA_EVENT); - expect(publishCalls).toHaveLength(1); - expect(publishCalls[0][3]).toEqual({ coalesce: true }); + await manager.emitChunk('gate-on', DELTA_EVENT); + expect(publishCalls).toHaveLength(1); + expect(publishCalls[0][3]).toEqual({ coalesce: true }); - settleAll(0); - await manager.destroy(); - }); + settleAll(0); + await manager.destroy(); + }, + ); - it('keeps the awaited path when the window is unset even with capable services', async () => { + it('does not batch the awaited append path before first subscriber admission', async () => { const { manager, publishCalls, settleAll } = await createManager({ windowMs: undefined, advertise: true, }); + await manager.createJob('before-admission', 'user-1', 'before-admission'); + + const emission = manager.emitChunk('before-admission', DELTA_EVENT); + await new Promise((resolve) => setImmediate(resolve)); + expect(publishCalls[0]).toHaveLength(3); + settleAll(0); + await emission; + await manager.destroy(); + }); + + it('continues batching after an admitted subscriber disconnects', async () => { + const { manager, publishCalls, settleAll } = await createManager({ + windowMs: undefined, + advertise: true, + }); + await manager.createJob('after-admission', 'user-1', 'after-admission'); + const subscription = await manager.subscribe('after-admission', jest.fn()); + subscription?.unsubscribe(); + + await manager.emitChunk('after-admission', DELTA_EVENT); + expect(publishCalls[0][3]).toEqual({ coalesce: true }); + settleAll(0); + await manager.destroy(); + }); + + it('keeps the awaited path with an explicit zero even with capable services', async () => { + const { manager, publishCalls, settleAll } = await createManager({ + windowMs: '0', + advertise: true, + }); await manager.createJob('window-off', 'user-1', 'window-off'); + await manager.subscribe('window-off', jest.fn()); const emission = manager.emitChunk('window-off', DELTA_EVENT); await new Promise((resolve) => setImmediate(resolve)); @@ -136,12 +174,52 @@ describe('delta coalescing manager gating and backpressure', () => { await manager.destroy(); }); + it('keeps attached in-memory streams on the awaited path with the default window', async () => { + const { manager, publishCalls, settleAll } = await createManager({ + windowMs: undefined, + advertise: true, + isRedis: false, + }); + await manager.createJob('memory-default', 'user-1', 'memory-default'); + const subscription = await manager.subscribe('memory-default', jest.fn()); + + const emission = manager.emitChunk('memory-default', DELTA_EVENT); + await new Promise((resolve) => setImmediate(resolve)); + expect(publishCalls[0]).toHaveLength(3); + settleAll(0); + await emission; + subscription?.unsubscribe(); + await manager.destroy(); + }); + + it.each([ + [undefined, 25], + ['0', 0], + ['', 0], + ['invalid', 0], + ['NaN', 0], + ['Infinity', 0], + ['-1', 0], + ['0.5', 0], + ['25', 25], + ['50.9', 50], + ['2000', 1000], + ])('resolves window %s to %s ms', (value, expected) => { + if (value === undefined) { + delete process.env.STREAM_DELTA_COALESCE_MS; + } else { + process.env.STREAM_DELTA_COALESCE_MS = value; + } + expect(resolveCoalesceWindowMs()).toBe(expected); + }); + it('applies backpressure once outstanding coalesced receipts hit the cap', async () => { const { manager, publishCalls, settleAll } = await createManager({ windowMs: '25', advertise: true, }); await manager.createJob('backpressure', 'user-1', 'backpressure'); + await manager.subscribe('backpressure', jest.fn()); /** The cap is 256: every emission below it resolves without awaiting the * (deliberately unsettled) publication receipts. */ diff --git a/packages/api/src/stream/__tests__/deltaCoalescing.stream_integration.spec.ts b/packages/api/src/stream/__tests__/deltaCoalescing.stream_integration.spec.ts index 5db02086cc..0359de0697 100644 --- a/packages/api/src/stream/__tests__/deltaCoalescing.stream_integration.spec.ts +++ b/packages/api/src/stream/__tests__/deltaCoalescing.stream_integration.spec.ts @@ -1,4 +1,4 @@ -/* eslint jest/no-standalone-expect: ["error", { "additionalTestBlockFunctions": ["testRedis"] }] */ +/* eslint jest/no-standalone-expect: ["error", { "additionalTestBlockFunctions": ["testRedis", "testRedis.each"] }] */ import type { Redis, Cluster } from 'ioredis'; import type { emitChunkWithReceipt as EmitChunkWithReceipt } from '~/stream/internal/chunkPublication'; import type { ServerSentEvent } from '~/types'; @@ -11,12 +11,12 @@ jest.spyOn(console, 'log').mockImplementation(); * Integration tests for streaming-delta coalescing (STREAM_DELTA_COALESCE_MS > 0). * * Coalescing batches eligible delta publications (and their durable appends) into - * one windowed frame while preserving per-event sequences, barrier ordering, + * one Redis request with legacy frames while preserving per-event sequences, barrier ordering, * terminal flushing, and the generation fence. * * Run with: USE_REDIS=true npx jest deltaCoalescing.stream_integration */ -describe('Delta coalescing integration', () => { +describe.each([undefined, '25'])('Delta coalescing integration (window %s)', (windowMs) => { let originalEnv: NodeJS.ProcessEnv; let ioredisClient: Redis | Cluster | null = null; const testPrefix = 'DeltaCoalescing-Integration-Test'; @@ -28,7 +28,11 @@ describe('Delta coalescing integration', () => { process.env.USE_REDIS = process.env.USE_REDIS ?? 'true'; process.env.REDIS_URI = process.env.REDIS_URI ?? 'redis://127.0.0.1:6379'; process.env.REDIS_KEY_PREFIX = testPrefix; - process.env.STREAM_DELTA_COALESCE_MS = '25'; + if (windowMs === undefined) { + delete process.env.STREAM_DELTA_COALESCE_MS; + } else { + process.env.STREAM_DELTA_COALESCE_MS = windowMs; + } jest.resetModules(); const redisModule = await import('~/cache/redisClients'); @@ -98,7 +102,7 @@ describe('Delta coalescing integration', () => { } testRedis( - 'delivers a coalesced window as individually sequenced chunks in order', + 'publishes a window in one EVAL as legacy chunks readable without batch support', async () => { const { RedisEventTransport, emitChunkWithReceipt } = await importFreshTransportModules(); const subscriber = (ioredisClient as Redis).duplicate(); @@ -107,10 +111,15 @@ describe('Delta coalescing integration', () => { const received: unknown[] = []; const rawSubscriber = (ioredisClient as Redis).duplicate(); - const rawFrames: Array<{ type: string; count?: number }> = []; + const rawFrames: Array<{ type: string; seq: number; data: { data: { i: number } } }> = []; + const legacyReceived: number[] = []; rawSubscriber.on('message', (_channel: string, message: string) => { - const parsed = JSON.parse(message) as { type: string; events?: unknown[] }; - rawFrames.push({ type: parsed.type, count: parsed.events?.length }); + const parsed = JSON.parse(message) as (typeof rawFrames)[number]; + rawFrames.push(parsed); + /** A pre-batching subscriber only recognizes the individual chunk envelope. */ + if (parsed.type === 'chunk') { + legacyReceived.push(parsed.data.data.i); + } }); await rawSubscriber.subscribe(`stream:{${streamId}}:events`); @@ -119,6 +128,7 @@ describe('Delta coalescing integration', () => { }); await subscription.ready; + const evalSpy = jest.spyOn(ioredisClient!, 'eval'); const receipts = await Promise.all( Array.from({ length: 5 }, (_, i) => emitChunkWithReceipt( @@ -131,7 +141,10 @@ describe('Delta coalescing integration', () => { ), ); - await waitFor(() => received.length === 5); + await waitFor(() => received.length === 5 && rawFrames.length === 5); + expect(evalSpy).toHaveBeenCalledTimes(1); + evalSpy.mockRestore(); + expect(legacyReceived).toEqual([0, 1, 2, 3, 4]); expect(received.map((event) => (event as { data: { i: number } }).data.i)).toEqual([ 0, 1, 2, 3, 4, ]); @@ -140,9 +153,8 @@ describe('Delta coalescing integration', () => { for (let i = 1; i < sequences.length; i++) { expect(sequences[i]).toBe(sequences[0] + i); } - const batchFrames = rawFrames.filter((frame) => frame.type === 'chunk_batch'); - expect(batchFrames).toHaveLength(1); - expect(batchFrames[0].count).toBe(5); + expect(rawFrames.map((frame) => frame.type)).toEqual(Array(5).fill('chunk')); + expect(rawFrames.map((frame) => frame.seq)).toEqual(sequences); subscription.unsubscribe(); transport.destroy(); @@ -151,6 +163,170 @@ describe('Delta coalescing integration', () => { 15000, ); + testRedis.each([ + { name: 'below byte cap, untagged', offset: -1, tagged: false, count: 2, character: 'x' }, + { name: 'at byte cap, untagged', offset: 0, tagged: false, count: 2, character: 'x' }, + { name: 'above byte cap, untagged', offset: 1, tagged: false, count: 2, character: 'x' }, + { name: 'below byte cap, tagged', offset: -1, tagged: true, count: 2, character: 'x' }, + { name: 'at byte cap, tagged', offset: 0, tagged: true, count: 2, character: 'x' }, + { name: 'above byte cap, tagged', offset: 1, tagged: true, count: 2, character: 'x' }, + { name: 'below byte cap, non-ASCII', offset: -1, tagged: true, count: 2, character: '雪' }, + { name: 'at event cap', offset: -1000, tagged: true, count: 64, character: 'x' }, + ])( + 'aligns append and publication flushes $name', + async ({ offset, tagged, count, character }) => { + const { RedisEventTransport, emitChunkWithReceipt } = await importFreshTransportModules(); + const { RedisJobStore } = await import('../implementations/RedisJobStore'); + const { MAX_COALESCED_BYTES, MAX_COALESCED_EVENTS } = await import('../internal/coalescing'); + const store = new RedisJobStore(ioredisClient!); + const transport = new RedisEventTransport( + ioredisClient!, + (ioredisClient as Redis).duplicate(), + ); + const reader = (ioredisClient as Redis).duplicate(); + const streamId = `coalesce-boundary-${Date.now()}`; + const job = await store.createJob(streamId, 'user-1', streamId); + await reader.ping(); + const generationId = tagged ? job.createdAt : undefined; + const eventWithText = (text: string) => ({ event: 'on_message_delta', data: { text } }); + const targetSize = MAX_COALESCED_BYTES + offset; + const emptySize = JSON.stringify(eventWithText('')).length; + const baseSize = Math.floor(targetSize / count); + const events = Array.from({ length: count }, (_, index) => + eventWithText( + character.repeat( + (index === count - 1 ? targetSize - baseSize * (count - 1) : baseSize) - emptySize, + ), + ), + ); + expect(events.reduce((size, event) => size + JSON.stringify(event).length, 0)).toBe( + targetSize, + ); + const immediate = targetSize >= MAX_COALESCED_BYTES || count >= MAX_COALESCED_EVENTS; + const evalSpy = jest.spyOn(ioredisClient!, 'eval'); + const appends: Array> = []; + const publications: Array> = []; + + /** Freeze only timers: real Redis I/O continues, while scheduler speed cannot + * conceal one coalescer flushing before the other's window expires. */ + jest.useFakeTimers({ + doNotFake: [ + 'Date', + 'hrtime', + 'nextTick', + 'performance', + 'queueMicrotask', + 'setImmediate', + 'clearImmediate', + ], + }); + try { + for (const event of events) { + appends.push( + store.appendChunk(streamId, event, generationId, undefined, { coalesce: true }), + ); + publications.push( + emitChunkWithReceipt(transport, streamId, event, generationId, { coalesce: true }), + ); + } + /** Append script has eight keys; publication has three. Both must issue on + * the same boundary, with durable append first, or neither may issue yet. */ + expect(evalSpy.mock.calls.map((call) => call[1])).toEqual(immediate ? [8, 3] : []); + if (!immediate) { + expect(await reader.xlen(`stream:{${streamId}}:chunks`)).toBe(0); + expect(await reader.get(`stream:{${streamId}}:seq`)).toBeNull(); + await jest.advanceTimersByTimeAsync(25); + } + expect(await Promise.all(appends)).toEqual(Array(count).fill(true)); + expect(await Promise.all(publications)).toEqual(Array.from({ length: count }, (_, i) => i)); + expect(evalSpy.mock.calls.map((call) => call[1])).toEqual([8, 3]); + + /** A different connection reads only durable Redis state, never the owner's + * local pending buffer. Its log and publication frontier must agree. */ + const entries = await reader.xrange(`stream:{${streamId}}:chunks`, '-', '+'); + expect( + entries.map(([, fields]) => JSON.parse(fields[fields.indexOf('event') + 1])), + ).toEqual(events); + expect(await reader.get(`stream:{${streamId}}:seq`)).toBe(String(count)); + await jest.advanceTimersByTimeAsync(25); + expect(evalSpy).toHaveBeenCalledTimes(2); + } finally { + evalSpy.mockRestore(); + transport.destroy(); + await store.destroy(); + reader.disconnect(); + jest.useRealTimers(); + } + }, + ); + + testRedis('preserves JSON payloads and generation tags in coalesced legacy frames', async () => { + const { RedisEventTransport, emitChunkWithReceipt } = await importFreshTransportModules(); + const { RedisJobStore } = await import('../implementations/RedisJobStore'); + const store = new RedisJobStore(ioredisClient!); + const streamId = `coalesce-json-${Date.now()}`; + const job = await store.createJob(streamId, 'user-1', streamId); + const rawSubscriber = (ioredisClient as Redis).duplicate(); + const transport = new RedisEventTransport(ioredisClient!, (ioredisClient as Redis).duplicate()); + const frames: Array<{ type: string; seq: number; data?: object | null; generationId: number }> = + []; + rawSubscriber.on('message', (_channel: string, message: string) => { + frames.push(JSON.parse(message) as (typeof frames)[number]); + }); + await rawSubscriber.subscribe(`stream:{${streamId}}:events`); + const payloads = [ + { arrays: [[], [null, []]], precise: 1.2345678901234567, text: '雪 "quoted" \n' }, + null, + undefined, + ]; + const receipts = await Promise.all( + payloads.map((payload) => + emitChunkWithReceipt(transport, streamId, payload, job.createdAt, { coalesce: true }), + ), + ); + await waitFor(() => frames.length === payloads.length); + expect(frames).toEqual( + payloads.map((data, index) => ({ + type: 'chunk', + seq: receipts[index], + generationId: job.createdAt, + ...(data !== undefined && { data }), + })), + ); + expect(frames[2]).not.toHaveProperty('data'); + rawSubscriber.disconnect(); + transport.destroy(); + await store.destroy(); + }); + + testRedis('still receives chunk_batch frames from existing opt-in producers', async () => { + const { RedisEventTransport } = await importFreshTransportModules(); + const transport = new RedisEventTransport(ioredisClient!, (ioredisClient as Redis).duplicate()); + const streamId = `coalesce-older-producer-${Date.now()}`; + const received: unknown[] = []; + let done = false; + const subscription = transport.subscribe(streamId, { + onChunk: (event) => received.push(event), + onDone: () => { + done = true; + }, + }); + await subscription.ready; + const events = [{ delta: 'first' }, { delta: 'second' }]; + await ioredisClient!.publish( + `stream:{${streamId}}:events`, + JSON.stringify({ type: 'chunk_batch', baseSeq: 0, events }), + ); + await ioredisClient!.publish( + `stream:{${streamId}}:events`, + JSON.stringify({ type: 'done', seq: 2, data: { final: true } }), + ); + await waitFor(() => done); + expect(received).toEqual(events); + subscription.unsubscribe(); + transport.destroy(); + }); + testRedis( 'a non-coalescable publication is a barrier that preserves emission order', async () => { diff --git a/packages/api/src/stream/__tests__/helpers/publisher.ts b/packages/api/src/stream/__tests__/helpers/publisher.ts index 9c2c65efab..58e676d143 100644 --- a/packages/api/src/stream/__tests__/helpers/publisher.ts +++ b/packages/api/src/stream/__tests__/helpers/publisher.ts @@ -76,13 +76,20 @@ export function createMockPublisher(): MockPublisher { _expectedGenerationId: string, _allowRetainedEpoch: string, _generationEpochGraceTtl: string, + _requireActiveJob: string, + sequenceCount = '1', + ...chunkSuffixes: string[] ) => { if (_numKeys === 1) { const frontier = await publisher.get(seqKey); await publisher.publish(jobKey, _generationEpochKey); return frontier ?? '0'; } + const count = Number(sequenceCount); const val = (await publisher.incr(seqKey)) as number; + for (let i = 1; i < count; i++) { + await publisher.incr(seqKey); + } let ttl = Number(ttlSeconds); const seqTtl = (await publisher.ttl(seqKey)) as number; if (seqTtl < Math.floor(ttl / 2)) { @@ -93,7 +100,13 @@ export function createMockPublisher(): MockPublisher { await publisher.expire(seqKey, ttl); } const seq = val - 1; - await publisher.publish(channel, `${prefix}${seq}${suffix}`); + if (chunkSuffixes.length > 0) { + for (let i = 0; i < count; i++) { + await publisher.publish(channel, `${prefix}${seq + i}${chunkSuffixes[i]}`); + } + } else { + await publisher.publish(channel, `${prefix}${seq}${suffix}`); + } return seq; }, ); diff --git a/packages/api/src/stream/implementations/RedisEventTransport.ts b/packages/api/src/stream/implementations/RedisEventTransport.ts index 274c089fd5..5854447576 100644 --- a/packages/api/src/stream/implementations/RedisEventTransport.ts +++ b/packages/api/src/stream/implementations/RedisEventTransport.ts @@ -80,7 +80,7 @@ interface PubSubMessage { /** * Producer-side buffer of coalescable chunk publications for one stream. - * Payloads are pre-serialized at enqueue so a flush only joins strings, and + * Payload suffixes are pre-serialized at enqueue so a flush only passes strings, and * each resolver settles its caller's receipt with `baseSeq + index`. */ interface PendingChunkBatch { @@ -147,12 +147,13 @@ interface PreemptRegistration { * allowRetainedEpoch ("0" | "1"), * generationEpochGraceTtl, * requireActiveJob ("0" | "1"), - * sequenceCount + * sequenceCount, + * ...chunkSuffixes (optional, one per event for a coalesced publication) * ] * RETURNS: the 0-indexed first seq assigned to this frame, or -1 when the generation - * guard fails. A single-event frame passes count 1 and splices the seq; a coalesced - * frame passes its event count, reserves that many consecutive sequences in one INCRBY, - * and splices the base — event i in the frame owns base + i. + * guard fails. A single frame splices its seq into prefix/suffix. A coalesced + * publication reserves consecutive sequences in one INCRBY and publishes one + * legacy chunk frame per suffix, keeping pre-batching subscribers compatible. * * During a rolling deployment, a job created by the previous version can expire without * leaving a generation marker. A tagged terminal event may claim that absent marker only @@ -187,7 +188,13 @@ const PUBLISH_SEQ_LUA = 'redis.call("EXPIRE", KEYS[1], ttl) ' + 'end ' + 'local seq = val - count ' + + 'if #ARGV > 9 then ' + + 'for i = 1, count do ' + + 'redis.call("PUBLISH", ARGV[1], ARGV[2] .. string.format("%d", seq + i - 1) .. ARGV[9 + i]) ' + + 'end ' + + 'else ' + 'redis.call("PUBLISH", ARGV[1], ARGV[2] .. string.format("%d", seq) .. ARGV[3]) ' + + 'end ' + 'return seq'; /** A normal generation guard correctly rejects events from an old epoch once @@ -470,6 +477,7 @@ export class RedisEventTransport implements IEventTransport { expectedGenerationId?: number, allowRetainedEpoch = false, requireActiveJob = false, + chunkSuffixes: string[] = [], ): Promise { const seq = await this.publisher.eval( PUBLISH_SEQ_LUA, @@ -486,6 +494,7 @@ export class RedisEventTransport implements IEventTransport { String(GENERATION_EPOCH_GRACE_TTL_SECONDS), requireActiveJob ? '1' : '0', String(count), + ...chunkSuffixes, ); return seq as number; } @@ -548,9 +557,19 @@ export class RedisEventTransport implements IEventTransport { } const batch = pending; - const encoded = JSON.stringify(event); - batch.events.push(encoded); - batch.bytes += encoded.length; + const payload = JSON.stringify(event); + const [, envelopeSuffix] = RedisEventTransport.buildPayloadParts({ + type: EventTypes.CHUNK, + ...(generationId != null && { generationId }), + }); + batch.events.push( + payload === undefined ? envelopeSuffix : `,"data":${payload}${envelopeSuffix}`, + ); + /** Match RedisJobStore's raw JSON length, excluding the publication envelope. + * Counting its wrapper would flush publications before their durable appends + * near the byte cap, letting a remote resume skip not-yet-durable events. + * Reuse the serialized payload so size accounting adds no second traversal. */ + batch.bytes += payload?.length ?? 0; const receipt = new Promise((resolve) => { batch.resolvers.push(resolve); }); @@ -566,12 +585,12 @@ export class RedisEventTransport implements IEventTransport { } /** - * Publish the stream's pending coalesced chunks as one CHUNK_BATCH frame. + * Publish the pending window in one EVAL using legacy CHUNK frames. * - * One INCRBY reserves a consecutive sequence per buffered event, so subscribers - * unpack the frame into individually sequenced chunks and the reorder buffer is - * none the wiser. The events array is spliced from pre-serialized payloads for - * the same reason single frames are: no server-side re-encoding. + * One INCRBY reserves consecutive sequences and Lua publishes each serialized + * suffix with its own sequence. This saves round trips and repeated guard/TTL + * work without requiring older subscribers to understand CHUNK_BATCH. Payloads + * are spliced, not decoded/re-encoded by Lua, preserving JSON semantics. */ private flushCoalescedChunks(streamId: string): Promise { const pending = this.pendingBatches.get(streamId); @@ -585,19 +604,17 @@ export class RedisEventTransport implements IEventTransport { } const { generationId, events, resolvers } = pending; - const prefix = `{"type":${JSON.stringify(EventTypes.CHUNK_BATCH)},"baseSeq":`; - const suffix = - (generationId != null ? `,"generationId":${generationId}` : '') + - `,"events":[${events.join(',')}]}`; + const prefix = `{"type":${JSON.stringify(EventTypes.CHUNK)},"seq":`; return this.evalPublishSequenced( streamId, prefix, - suffix, + '', events.length, generationId, false, generationId != null, + events, ).then( (baseSeq) => { if (baseSeq === -1) { diff --git a/packages/api/src/stream/internal/coalescing.ts b/packages/api/src/stream/internal/coalescing.ts index 5440566688..8114ffa002 100644 --- a/packages/api/src/stream/internal/coalescing.ts +++ b/packages/api/src/stream/internal/coalescing.ts @@ -2,6 +2,8 @@ const MAX_COALESCE_WINDOW_MS = 1000; /** Coalesced-batch safety caps: a full buffer flushes immediately, ahead of the window. */ export const MAX_COALESCED_EVENTS = 64; +/** Shared raw-event JSON string-length budget, excluding transport envelopes. + * Keep both coalescers on the same accounting basis, including for non-ASCII JSON. */ export const MAX_COALESCED_BYTES = 128 * 1024; /** @@ -14,10 +16,10 @@ export const MAX_COALESCED_BYTES = 128 * 1024; * resume frontier assumes an event is never visible in the durable chunk log * meaningfully earlier than its sequence lands on the shared counter, so * batching one side without the other reopens that race for the full window - * instead of a same-tick skew. 0 (the default) disables coalescing. + * instead of a same-tick skew. Unset uses 25ms; explicit 0 disables coalescing. */ export function resolveCoalesceWindowMs(): number { - const raw = Number(process.env.STREAM_DELTA_COALESCE_MS ?? 0); + const raw = Number(process.env.STREAM_DELTA_COALESCE_MS ?? 25); if (!Number.isFinite(raw) || raw <= 0) { return 0; }