mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-21 23:43:35 +00:00
🚐 perf: Enable Redis Stream Coalescing by Default (#16093)
* 🚐 perf: Enable Redis Stream Coalescing by Default
* fix: Preserve Legacy Chunk Frames in Coalesced Publications
* fix: Align Coalesced Publication and Append Size Thresholds
---------
Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
parent
9acf9f0ac8
commit
2f67471b65
8 changed files with 368 additions and 53 deletions
12
.env.example
12
.env.example
|
|
@ -1232,11 +1232,13 @@ HELP_AND_FAQ_URL=https://librechat.ai
|
|||
# SCHEDULES_DISABLED=true
|
||||
|
||||
# Coalesce streamed model/tool-argument deltas into windowed Redis publications (ms).
|
||||
# Unset or 0 (default) publishes per delta. 25 is recommended: it batches the publish
|
||||
# EVAL and the durable append across the window (fewer Redis round trips and lower
|
||||
# Redis CPU at high token rates) at the cost of up to one window of added delivery
|
||||
# latency. Enable only after EVERY replica runs a build with batch-frame support;
|
||||
# older subscribers drop coalesced frames. Values are capped at 1000.
|
||||
# Defaults to 25ms when unset; explicit 0 publishes per delta. Batches both publish
|
||||
# and durable append operations (fewer Redis round trips and repeated guard/TTL work at high
|
||||
# token rates), adding up to one window of buffering latency and crash-loss exposure
|
||||
# for unflushed deltas. In-memory streams are unchanged. Values are capped at 1000.
|
||||
# Batched publications retain individual chunk frames for older subscribers; Pub/Sub
|
||||
# message count is unchanged. Incoming chunk_batch frames remain supported for existing
|
||||
# opt-in producers. See UPGRADING.md for compatibility guidance.
|
||||
# Keep the window <= the stream-smoothing cadence (`streamRate`, default 25ms):
|
||||
# each smoothing tick emits its pieces in one burst, so a tick-sized window
|
||||
# captures exactly one batch per tick; a larger window re-batches the paced
|
||||
|
|
|
|||
24
UPGRADING.md
24
UPGRADING.md
|
|
@ -1,5 +1,29 @@
|
|||
# Upgrading LibreChat
|
||||
|
||||
## Redis streaming now coalesces deltas by default
|
||||
|
||||
Redis-backed streams now batch model and tool-argument deltas in a **25 ms**
|
||||
window when `STREAM_DELTA_COALESCE_MS` is unset. Explicit `0` keeps per-delta
|
||||
publication; existing explicit values retain their behavior. In-memory streams
|
||||
are unchanged, including deployments using `USE_REDIS_STREAMS=false`.
|
||||
|
||||
The same window batches durable appends and publications. It reduces Redis
|
||||
round trips and repeated guard/TTL work, but not Pub/Sub message count at the cost of up to one window of buffering latency and possible loss
|
||||
of unflushed deltas on process crash. Terminal and non-coalescable barriers still
|
||||
flush pending batches before proceeding.
|
||||
|
||||
Coalescing batches Redis requests, not the subscriber wire format: each event
|
||||
still publishes as an individually sequenced `chunk` frame. Subscribers from
|
||||
before batch-frame support can read these publications without a preparatory
|
||||
configuration change. New subscribers also retain `chunk_batch` decoding for
|
||||
interoperation with existing opt-in batching producers.
|
||||
|
||||
This removes the new default's batch-frame compatibility hazard; it does not
|
||||
promise compatibility across unrelated generation-protocol changes. If an
|
||||
existing producer already emits `chunk_batch` frames through explicit opt-in,
|
||||
keep those producers away from subscribers that predate batch-frame support,
|
||||
or disable coalescing on those producers before introducing older subscribers.
|
||||
|
||||
## Tenant index migration (v0.8.7 and earlier databases)
|
||||
|
||||
Upgrading an existing database can log `Index build failed` for User, Role,
|
||||
|
|
|
|||
|
|
@ -6734,10 +6734,13 @@ class GenerationJobManagerClass {
|
|||
* One decision drives both durable-log and publish batching: the append and
|
||||
* the sequence allocation for an event must stay tightly coupled in time, or
|
||||
* the resume frontier (chunk-log snapshot → sequence-counter sync) misreads
|
||||
* a window's tail as already-delivered or as duplicates.
|
||||
* a window's tail as already-delivered or as duplicates. Before first-subscriber
|
||||
* admission, each append is awaited for snapshot safety; batching that path
|
||||
* would add a full window of producer latency to every delta.
|
||||
*/
|
||||
const coalescableDelta =
|
||||
this._deltaCoalescingEnabled &&
|
||||
(runtime.hasSubscriber || runtime.everHadSubscriber) &&
|
||||
!runtime.startupTelemetry &&
|
||||
options?.durable !== true &&
|
||||
options?.deliveredSteer == null &&
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTr
|
|||
import { registerChunkPublicationCapability } from '~/stream/internal/chunkPublication';
|
||||
import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';
|
||||
import { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
|
||||
import { resolveCoalesceWindowMs } from '~/stream/internal/coalescing';
|
||||
|
||||
jest.spyOn(console, 'log').mockImplementation();
|
||||
|
||||
|
|
@ -37,6 +38,7 @@ describe('delta coalescing manager gating and backpressure', () => {
|
|||
async function createManager(options: {
|
||||
windowMs: string | undefined;
|
||||
advertise: boolean;
|
||||
isRedis?: boolean;
|
||||
}): Promise<{
|
||||
manager: GenerationJobManagerClass;
|
||||
publishCalls: Array<unknown[]>;
|
||||
|
|
@ -64,7 +66,7 @@ describe('delta coalescing manager gating and backpressure', () => {
|
|||
});
|
||||
|
||||
const manager = new GenerationJobManagerClass();
|
||||
manager.configure({ jobStore, eventTransport, isRedis: true });
|
||||
manager.configure({ jobStore, eventTransport, isRedis: options.isRedis ?? true });
|
||||
manager.initialize();
|
||||
return {
|
||||
manager,
|
||||
|
|
@ -83,6 +85,7 @@ describe('delta coalescing manager gating and backpressure', () => {
|
|||
advertise: false,
|
||||
});
|
||||
await manager.createJob('gate-off', 'user-1', 'gate-off');
|
||||
await manager.subscribe('gate-off', jest.fn());
|
||||
|
||||
const emission = manager.emitChunk('gate-off', DELTA_EVENT);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
|
@ -105,27 +108,62 @@ describe('delta coalescing manager gating and backpressure', () => {
|
|||
await manager.destroy();
|
||||
});
|
||||
|
||||
it('sends coalesce hints and stops awaiting receipts when both services advertise support', async () => {
|
||||
const { manager, publishCalls, settleAll } = await createManager({
|
||||
windowMs: '25',
|
||||
advertise: true,
|
||||
});
|
||||
await manager.createJob('gate-on', 'user-1', 'gate-on');
|
||||
it.each([undefined, '25'])(
|
||||
'sends coalesce hints without awaiting receipts with capable Redis services and window %s',
|
||||
async (windowMs) => {
|
||||
const { manager, publishCalls, settleAll } = await createManager({
|
||||
windowMs,
|
||||
advertise: true,
|
||||
});
|
||||
await manager.createJob('gate-on', 'user-1', 'gate-on');
|
||||
await manager.subscribe('gate-on', jest.fn());
|
||||
|
||||
await manager.emitChunk('gate-on', DELTA_EVENT);
|
||||
expect(publishCalls).toHaveLength(1);
|
||||
expect(publishCalls[0][3]).toEqual({ coalesce: true });
|
||||
await manager.emitChunk('gate-on', DELTA_EVENT);
|
||||
expect(publishCalls).toHaveLength(1);
|
||||
expect(publishCalls[0][3]).toEqual({ coalesce: true });
|
||||
|
||||
settleAll(0);
|
||||
await manager.destroy();
|
||||
});
|
||||
settleAll(0);
|
||||
await manager.destroy();
|
||||
},
|
||||
);
|
||||
|
||||
it('keeps the awaited path when the window is unset even with capable services', async () => {
|
||||
it('does not batch the awaited append path before first subscriber admission', async () => {
|
||||
const { manager, publishCalls, settleAll } = await createManager({
|
||||
windowMs: undefined,
|
||||
advertise: true,
|
||||
});
|
||||
await manager.createJob('before-admission', 'user-1', 'before-admission');
|
||||
|
||||
const emission = manager.emitChunk('before-admission', DELTA_EVENT);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(publishCalls[0]).toHaveLength(3);
|
||||
settleAll(0);
|
||||
await emission;
|
||||
await manager.destroy();
|
||||
});
|
||||
|
||||
it('continues batching after an admitted subscriber disconnects', async () => {
|
||||
const { manager, publishCalls, settleAll } = await createManager({
|
||||
windowMs: undefined,
|
||||
advertise: true,
|
||||
});
|
||||
await manager.createJob('after-admission', 'user-1', 'after-admission');
|
||||
const subscription = await manager.subscribe('after-admission', jest.fn());
|
||||
subscription?.unsubscribe();
|
||||
|
||||
await manager.emitChunk('after-admission', DELTA_EVENT);
|
||||
expect(publishCalls[0][3]).toEqual({ coalesce: true });
|
||||
settleAll(0);
|
||||
await manager.destroy();
|
||||
});
|
||||
|
||||
it('keeps the awaited path with an explicit zero even with capable services', async () => {
|
||||
const { manager, publishCalls, settleAll } = await createManager({
|
||||
windowMs: '0',
|
||||
advertise: true,
|
||||
});
|
||||
await manager.createJob('window-off', 'user-1', 'window-off');
|
||||
await manager.subscribe('window-off', jest.fn());
|
||||
|
||||
const emission = manager.emitChunk('window-off', DELTA_EVENT);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
|
@ -136,12 +174,52 @@ describe('delta coalescing manager gating and backpressure', () => {
|
|||
await manager.destroy();
|
||||
});
|
||||
|
||||
it('keeps attached in-memory streams on the awaited path with the default window', async () => {
|
||||
const { manager, publishCalls, settleAll } = await createManager({
|
||||
windowMs: undefined,
|
||||
advertise: true,
|
||||
isRedis: false,
|
||||
});
|
||||
await manager.createJob('memory-default', 'user-1', 'memory-default');
|
||||
const subscription = await manager.subscribe('memory-default', jest.fn());
|
||||
|
||||
const emission = manager.emitChunk('memory-default', DELTA_EVENT);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(publishCalls[0]).toHaveLength(3);
|
||||
settleAll(0);
|
||||
await emission;
|
||||
subscription?.unsubscribe();
|
||||
await manager.destroy();
|
||||
});
|
||||
|
||||
it.each([
|
||||
[undefined, 25],
|
||||
['0', 0],
|
||||
['', 0],
|
||||
['invalid', 0],
|
||||
['NaN', 0],
|
||||
['Infinity', 0],
|
||||
['-1', 0],
|
||||
['0.5', 0],
|
||||
['25', 25],
|
||||
['50.9', 50],
|
||||
['2000', 1000],
|
||||
])('resolves window %s to %s ms', (value, expected) => {
|
||||
if (value === undefined) {
|
||||
delete process.env.STREAM_DELTA_COALESCE_MS;
|
||||
} else {
|
||||
process.env.STREAM_DELTA_COALESCE_MS = value;
|
||||
}
|
||||
expect(resolveCoalesceWindowMs()).toBe(expected);
|
||||
});
|
||||
|
||||
it('applies backpressure once outstanding coalesced receipts hit the cap', async () => {
|
||||
const { manager, publishCalls, settleAll } = await createManager({
|
||||
windowMs: '25',
|
||||
advertise: true,
|
||||
});
|
||||
await manager.createJob('backpressure', 'user-1', 'backpressure');
|
||||
await manager.subscribe('backpressure', jest.fn());
|
||||
|
||||
/** The cap is 256: every emission below it resolves without awaiting the
|
||||
* (deliberately unsettled) publication receipts. */
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* eslint jest/no-standalone-expect: ["error", { "additionalTestBlockFunctions": ["testRedis"] }] */
|
||||
/* eslint jest/no-standalone-expect: ["error", { "additionalTestBlockFunctions": ["testRedis", "testRedis.each"] }] */
|
||||
import type { Redis, Cluster } from 'ioredis';
|
||||
import type { emitChunkWithReceipt as EmitChunkWithReceipt } from '~/stream/internal/chunkPublication';
|
||||
import type { ServerSentEvent } from '~/types';
|
||||
|
|
@ -11,12 +11,12 @@ jest.spyOn(console, 'log').mockImplementation();
|
|||
* Integration tests for streaming-delta coalescing (STREAM_DELTA_COALESCE_MS > 0).
|
||||
*
|
||||
* Coalescing batches eligible delta publications (and their durable appends) into
|
||||
* one windowed frame while preserving per-event sequences, barrier ordering,
|
||||
* one Redis request with legacy frames while preserving per-event sequences, barrier ordering,
|
||||
* terminal flushing, and the generation fence.
|
||||
*
|
||||
* Run with: USE_REDIS=true npx jest deltaCoalescing.stream_integration
|
||||
*/
|
||||
describe('Delta coalescing integration', () => {
|
||||
describe.each([undefined, '25'])('Delta coalescing integration (window %s)', (windowMs) => {
|
||||
let originalEnv: NodeJS.ProcessEnv;
|
||||
let ioredisClient: Redis | Cluster | null = null;
|
||||
const testPrefix = 'DeltaCoalescing-Integration-Test';
|
||||
|
|
@ -28,7 +28,11 @@ describe('Delta coalescing integration', () => {
|
|||
process.env.USE_REDIS = process.env.USE_REDIS ?? 'true';
|
||||
process.env.REDIS_URI = process.env.REDIS_URI ?? 'redis://127.0.0.1:6379';
|
||||
process.env.REDIS_KEY_PREFIX = testPrefix;
|
||||
process.env.STREAM_DELTA_COALESCE_MS = '25';
|
||||
if (windowMs === undefined) {
|
||||
delete process.env.STREAM_DELTA_COALESCE_MS;
|
||||
} else {
|
||||
process.env.STREAM_DELTA_COALESCE_MS = windowMs;
|
||||
}
|
||||
|
||||
jest.resetModules();
|
||||
const redisModule = await import('~/cache/redisClients');
|
||||
|
|
@ -98,7 +102,7 @@ describe('Delta coalescing integration', () => {
|
|||
}
|
||||
|
||||
testRedis(
|
||||
'delivers a coalesced window as individually sequenced chunks in order',
|
||||
'publishes a window in one EVAL as legacy chunks readable without batch support',
|
||||
async () => {
|
||||
const { RedisEventTransport, emitChunkWithReceipt } = await importFreshTransportModules();
|
||||
const subscriber = (ioredisClient as Redis).duplicate();
|
||||
|
|
@ -107,10 +111,15 @@ describe('Delta coalescing integration', () => {
|
|||
|
||||
const received: unknown[] = [];
|
||||
const rawSubscriber = (ioredisClient as Redis).duplicate();
|
||||
const rawFrames: Array<{ type: string; count?: number }> = [];
|
||||
const rawFrames: Array<{ type: string; seq: number; data: { data: { i: number } } }> = [];
|
||||
const legacyReceived: number[] = [];
|
||||
rawSubscriber.on('message', (_channel: string, message: string) => {
|
||||
const parsed = JSON.parse(message) as { type: string; events?: unknown[] };
|
||||
rawFrames.push({ type: parsed.type, count: parsed.events?.length });
|
||||
const parsed = JSON.parse(message) as (typeof rawFrames)[number];
|
||||
rawFrames.push(parsed);
|
||||
/** A pre-batching subscriber only recognizes the individual chunk envelope. */
|
||||
if (parsed.type === 'chunk') {
|
||||
legacyReceived.push(parsed.data.data.i);
|
||||
}
|
||||
});
|
||||
await rawSubscriber.subscribe(`stream:{${streamId}}:events`);
|
||||
|
||||
|
|
@ -119,6 +128,7 @@ describe('Delta coalescing integration', () => {
|
|||
});
|
||||
await subscription.ready;
|
||||
|
||||
const evalSpy = jest.spyOn(ioredisClient!, 'eval');
|
||||
const receipts = await Promise.all(
|
||||
Array.from({ length: 5 }, (_, i) =>
|
||||
emitChunkWithReceipt(
|
||||
|
|
@ -131,7 +141,10 @@ describe('Delta coalescing integration', () => {
|
|||
),
|
||||
);
|
||||
|
||||
await waitFor(() => received.length === 5);
|
||||
await waitFor(() => received.length === 5 && rawFrames.length === 5);
|
||||
expect(evalSpy).toHaveBeenCalledTimes(1);
|
||||
evalSpy.mockRestore();
|
||||
expect(legacyReceived).toEqual([0, 1, 2, 3, 4]);
|
||||
expect(received.map((event) => (event as { data: { i: number } }).data.i)).toEqual([
|
||||
0, 1, 2, 3, 4,
|
||||
]);
|
||||
|
|
@ -140,9 +153,8 @@ describe('Delta coalescing integration', () => {
|
|||
for (let i = 1; i < sequences.length; i++) {
|
||||
expect(sequences[i]).toBe(sequences[0] + i);
|
||||
}
|
||||
const batchFrames = rawFrames.filter((frame) => frame.type === 'chunk_batch');
|
||||
expect(batchFrames).toHaveLength(1);
|
||||
expect(batchFrames[0].count).toBe(5);
|
||||
expect(rawFrames.map((frame) => frame.type)).toEqual(Array(5).fill('chunk'));
|
||||
expect(rawFrames.map((frame) => frame.seq)).toEqual(sequences);
|
||||
|
||||
subscription.unsubscribe();
|
||||
transport.destroy();
|
||||
|
|
@ -151,6 +163,170 @@ describe('Delta coalescing integration', () => {
|
|||
15000,
|
||||
);
|
||||
|
||||
testRedis.each([
|
||||
{ name: 'below byte cap, untagged', offset: -1, tagged: false, count: 2, character: 'x' },
|
||||
{ name: 'at byte cap, untagged', offset: 0, tagged: false, count: 2, character: 'x' },
|
||||
{ name: 'above byte cap, untagged', offset: 1, tagged: false, count: 2, character: 'x' },
|
||||
{ name: 'below byte cap, tagged', offset: -1, tagged: true, count: 2, character: 'x' },
|
||||
{ name: 'at byte cap, tagged', offset: 0, tagged: true, count: 2, character: 'x' },
|
||||
{ name: 'above byte cap, tagged', offset: 1, tagged: true, count: 2, character: 'x' },
|
||||
{ name: 'below byte cap, non-ASCII', offset: -1, tagged: true, count: 2, character: '雪' },
|
||||
{ name: 'at event cap', offset: -1000, tagged: true, count: 64, character: 'x' },
|
||||
])(
|
||||
'aligns append and publication flushes $name',
|
||||
async ({ offset, tagged, count, character }) => {
|
||||
const { RedisEventTransport, emitChunkWithReceipt } = await importFreshTransportModules();
|
||||
const { RedisJobStore } = await import('../implementations/RedisJobStore');
|
||||
const { MAX_COALESCED_BYTES, MAX_COALESCED_EVENTS } = await import('../internal/coalescing');
|
||||
const store = new RedisJobStore(ioredisClient!);
|
||||
const transport = new RedisEventTransport(
|
||||
ioredisClient!,
|
||||
(ioredisClient as Redis).duplicate(),
|
||||
);
|
||||
const reader = (ioredisClient as Redis).duplicate();
|
||||
const streamId = `coalesce-boundary-${Date.now()}`;
|
||||
const job = await store.createJob(streamId, 'user-1', streamId);
|
||||
await reader.ping();
|
||||
const generationId = tagged ? job.createdAt : undefined;
|
||||
const eventWithText = (text: string) => ({ event: 'on_message_delta', data: { text } });
|
||||
const targetSize = MAX_COALESCED_BYTES + offset;
|
||||
const emptySize = JSON.stringify(eventWithText('')).length;
|
||||
const baseSize = Math.floor(targetSize / count);
|
||||
const events = Array.from({ length: count }, (_, index) =>
|
||||
eventWithText(
|
||||
character.repeat(
|
||||
(index === count - 1 ? targetSize - baseSize * (count - 1) : baseSize) - emptySize,
|
||||
),
|
||||
),
|
||||
);
|
||||
expect(events.reduce((size, event) => size + JSON.stringify(event).length, 0)).toBe(
|
||||
targetSize,
|
||||
);
|
||||
const immediate = targetSize >= MAX_COALESCED_BYTES || count >= MAX_COALESCED_EVENTS;
|
||||
const evalSpy = jest.spyOn(ioredisClient!, 'eval');
|
||||
const appends: Array<Promise<boolean>> = [];
|
||||
const publications: Array<ReturnType<typeof emitChunkWithReceipt>> = [];
|
||||
|
||||
/** Freeze only timers: real Redis I/O continues, while scheduler speed cannot
|
||||
* conceal one coalescer flushing before the other's window expires. */
|
||||
jest.useFakeTimers({
|
||||
doNotFake: [
|
||||
'Date',
|
||||
'hrtime',
|
||||
'nextTick',
|
||||
'performance',
|
||||
'queueMicrotask',
|
||||
'setImmediate',
|
||||
'clearImmediate',
|
||||
],
|
||||
});
|
||||
try {
|
||||
for (const event of events) {
|
||||
appends.push(
|
||||
store.appendChunk(streamId, event, generationId, undefined, { coalesce: true }),
|
||||
);
|
||||
publications.push(
|
||||
emitChunkWithReceipt(transport, streamId, event, generationId, { coalesce: true }),
|
||||
);
|
||||
}
|
||||
/** Append script has eight keys; publication has three. Both must issue on
|
||||
* the same boundary, with durable append first, or neither may issue yet. */
|
||||
expect(evalSpy.mock.calls.map((call) => call[1])).toEqual(immediate ? [8, 3] : []);
|
||||
if (!immediate) {
|
||||
expect(await reader.xlen(`stream:{${streamId}}:chunks`)).toBe(0);
|
||||
expect(await reader.get(`stream:{${streamId}}:seq`)).toBeNull();
|
||||
await jest.advanceTimersByTimeAsync(25);
|
||||
}
|
||||
expect(await Promise.all(appends)).toEqual(Array(count).fill(true));
|
||||
expect(await Promise.all(publications)).toEqual(Array.from({ length: count }, (_, i) => i));
|
||||
expect(evalSpy.mock.calls.map((call) => call[1])).toEqual([8, 3]);
|
||||
|
||||
/** A different connection reads only durable Redis state, never the owner's
|
||||
* local pending buffer. Its log and publication frontier must agree. */
|
||||
const entries = await reader.xrange(`stream:{${streamId}}:chunks`, '-', '+');
|
||||
expect(
|
||||
entries.map(([, fields]) => JSON.parse(fields[fields.indexOf('event') + 1])),
|
||||
).toEqual(events);
|
||||
expect(await reader.get(`stream:{${streamId}}:seq`)).toBe(String(count));
|
||||
await jest.advanceTimersByTimeAsync(25);
|
||||
expect(evalSpy).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
evalSpy.mockRestore();
|
||||
transport.destroy();
|
||||
await store.destroy();
|
||||
reader.disconnect();
|
||||
jest.useRealTimers();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
testRedis('preserves JSON payloads and generation tags in coalesced legacy frames', async () => {
|
||||
const { RedisEventTransport, emitChunkWithReceipt } = await importFreshTransportModules();
|
||||
const { RedisJobStore } = await import('../implementations/RedisJobStore');
|
||||
const store = new RedisJobStore(ioredisClient!);
|
||||
const streamId = `coalesce-json-${Date.now()}`;
|
||||
const job = await store.createJob(streamId, 'user-1', streamId);
|
||||
const rawSubscriber = (ioredisClient as Redis).duplicate();
|
||||
const transport = new RedisEventTransport(ioredisClient!, (ioredisClient as Redis).duplicate());
|
||||
const frames: Array<{ type: string; seq: number; data?: object | null; generationId: number }> =
|
||||
[];
|
||||
rawSubscriber.on('message', (_channel: string, message: string) => {
|
||||
frames.push(JSON.parse(message) as (typeof frames)[number]);
|
||||
});
|
||||
await rawSubscriber.subscribe(`stream:{${streamId}}:events`);
|
||||
const payloads = [
|
||||
{ arrays: [[], [null, []]], precise: 1.2345678901234567, text: '雪 "quoted" \n' },
|
||||
null,
|
||||
undefined,
|
||||
];
|
||||
const receipts = await Promise.all(
|
||||
payloads.map((payload) =>
|
||||
emitChunkWithReceipt(transport, streamId, payload, job.createdAt, { coalesce: true }),
|
||||
),
|
||||
);
|
||||
await waitFor(() => frames.length === payloads.length);
|
||||
expect(frames).toEqual(
|
||||
payloads.map((data, index) => ({
|
||||
type: 'chunk',
|
||||
seq: receipts[index],
|
||||
generationId: job.createdAt,
|
||||
...(data !== undefined && { data }),
|
||||
})),
|
||||
);
|
||||
expect(frames[2]).not.toHaveProperty('data');
|
||||
rawSubscriber.disconnect();
|
||||
transport.destroy();
|
||||
await store.destroy();
|
||||
});
|
||||
|
||||
testRedis('still receives chunk_batch frames from existing opt-in producers', async () => {
|
||||
const { RedisEventTransport } = await importFreshTransportModules();
|
||||
const transport = new RedisEventTransport(ioredisClient!, (ioredisClient as Redis).duplicate());
|
||||
const streamId = `coalesce-older-producer-${Date.now()}`;
|
||||
const received: unknown[] = [];
|
||||
let done = false;
|
||||
const subscription = transport.subscribe(streamId, {
|
||||
onChunk: (event) => received.push(event),
|
||||
onDone: () => {
|
||||
done = true;
|
||||
},
|
||||
});
|
||||
await subscription.ready;
|
||||
const events = [{ delta: 'first' }, { delta: 'second' }];
|
||||
await ioredisClient!.publish(
|
||||
`stream:{${streamId}}:events`,
|
||||
JSON.stringify({ type: 'chunk_batch', baseSeq: 0, events }),
|
||||
);
|
||||
await ioredisClient!.publish(
|
||||
`stream:{${streamId}}:events`,
|
||||
JSON.stringify({ type: 'done', seq: 2, data: { final: true } }),
|
||||
);
|
||||
await waitFor(() => done);
|
||||
expect(received).toEqual(events);
|
||||
subscription.unsubscribe();
|
||||
transport.destroy();
|
||||
});
|
||||
|
||||
testRedis(
|
||||
'a non-coalescable publication is a barrier that preserves emission order',
|
||||
async () => {
|
||||
|
|
|
|||
|
|
@ -76,13 +76,20 @@ export function createMockPublisher(): MockPublisher {
|
|||
_expectedGenerationId: string,
|
||||
_allowRetainedEpoch: string,
|
||||
_generationEpochGraceTtl: string,
|
||||
_requireActiveJob: string,
|
||||
sequenceCount = '1',
|
||||
...chunkSuffixes: string[]
|
||||
) => {
|
||||
if (_numKeys === 1) {
|
||||
const frontier = await publisher.get(seqKey);
|
||||
await publisher.publish(jobKey, _generationEpochKey);
|
||||
return frontier ?? '0';
|
||||
}
|
||||
const count = Number(sequenceCount);
|
||||
const val = (await publisher.incr(seqKey)) as number;
|
||||
for (let i = 1; i < count; i++) {
|
||||
await publisher.incr(seqKey);
|
||||
}
|
||||
let ttl = Number(ttlSeconds);
|
||||
const seqTtl = (await publisher.ttl(seqKey)) as number;
|
||||
if (seqTtl < Math.floor(ttl / 2)) {
|
||||
|
|
@ -93,7 +100,13 @@ export function createMockPublisher(): MockPublisher {
|
|||
await publisher.expire(seqKey, ttl);
|
||||
}
|
||||
const seq = val - 1;
|
||||
await publisher.publish(channel, `${prefix}${seq}${suffix}`);
|
||||
if (chunkSuffixes.length > 0) {
|
||||
for (let i = 0; i < count; i++) {
|
||||
await publisher.publish(channel, `${prefix}${seq + i}${chunkSuffixes[i]}`);
|
||||
}
|
||||
} else {
|
||||
await publisher.publish(channel, `${prefix}${seq}${suffix}`);
|
||||
}
|
||||
return seq;
|
||||
},
|
||||
);
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ interface PubSubMessage {
|
|||
|
||||
/**
|
||||
* Producer-side buffer of coalescable chunk publications for one stream.
|
||||
* Payloads are pre-serialized at enqueue so a flush only joins strings, and
|
||||
* Payload suffixes are pre-serialized at enqueue so a flush only passes strings, and
|
||||
* each resolver settles its caller's receipt with `baseSeq + index`.
|
||||
*/
|
||||
interface PendingChunkBatch {
|
||||
|
|
@ -147,12 +147,13 @@ interface PreemptRegistration {
|
|||
* allowRetainedEpoch ("0" | "1"),
|
||||
* generationEpochGraceTtl,
|
||||
* requireActiveJob ("0" | "1"),
|
||||
* sequenceCount
|
||||
* sequenceCount,
|
||||
* ...chunkSuffixes (optional, one per event for a coalesced publication)
|
||||
* ]
|
||||
* RETURNS: the 0-indexed first seq assigned to this frame, or -1 when the generation
|
||||
* guard fails. A single-event frame passes count 1 and splices the seq; a coalesced
|
||||
* frame passes its event count, reserves that many consecutive sequences in one INCRBY,
|
||||
* and splices the base — event i in the frame owns base + i.
|
||||
* guard fails. A single frame splices its seq into prefix/suffix. A coalesced
|
||||
* publication reserves consecutive sequences in one INCRBY and publishes one
|
||||
* legacy chunk frame per suffix, keeping pre-batching subscribers compatible.
|
||||
*
|
||||
* During a rolling deployment, a job created by the previous version can expire without
|
||||
* leaving a generation marker. A tagged terminal event may claim that absent marker only
|
||||
|
|
@ -187,7 +188,13 @@ const PUBLISH_SEQ_LUA =
|
|||
'redis.call("EXPIRE", KEYS[1], ttl) ' +
|
||||
'end ' +
|
||||
'local seq = val - count ' +
|
||||
'if #ARGV > 9 then ' +
|
||||
'for i = 1, count do ' +
|
||||
'redis.call("PUBLISH", ARGV[1], ARGV[2] .. string.format("%d", seq + i - 1) .. ARGV[9 + i]) ' +
|
||||
'end ' +
|
||||
'else ' +
|
||||
'redis.call("PUBLISH", ARGV[1], ARGV[2] .. string.format("%d", seq) .. ARGV[3]) ' +
|
||||
'end ' +
|
||||
'return seq';
|
||||
|
||||
/** A normal generation guard correctly rejects events from an old epoch once
|
||||
|
|
@ -470,6 +477,7 @@ export class RedisEventTransport implements IEventTransport {
|
|||
expectedGenerationId?: number,
|
||||
allowRetainedEpoch = false,
|
||||
requireActiveJob = false,
|
||||
chunkSuffixes: string[] = [],
|
||||
): Promise<number> {
|
||||
const seq = await this.publisher.eval(
|
||||
PUBLISH_SEQ_LUA,
|
||||
|
|
@ -486,6 +494,7 @@ export class RedisEventTransport implements IEventTransport {
|
|||
String(GENERATION_EPOCH_GRACE_TTL_SECONDS),
|
||||
requireActiveJob ? '1' : '0',
|
||||
String(count),
|
||||
...chunkSuffixes,
|
||||
);
|
||||
return seq as number;
|
||||
}
|
||||
|
|
@ -548,9 +557,19 @@ export class RedisEventTransport implements IEventTransport {
|
|||
}
|
||||
|
||||
const batch = pending;
|
||||
const encoded = JSON.stringify(event);
|
||||
batch.events.push(encoded);
|
||||
batch.bytes += encoded.length;
|
||||
const payload = JSON.stringify(event);
|
||||
const [, envelopeSuffix] = RedisEventTransport.buildPayloadParts({
|
||||
type: EventTypes.CHUNK,
|
||||
...(generationId != null && { generationId }),
|
||||
});
|
||||
batch.events.push(
|
||||
payload === undefined ? envelopeSuffix : `,"data":${payload}${envelopeSuffix}`,
|
||||
);
|
||||
/** Match RedisJobStore's raw JSON length, excluding the publication envelope.
|
||||
* Counting its wrapper would flush publications before their durable appends
|
||||
* near the byte cap, letting a remote resume skip not-yet-durable events.
|
||||
* Reuse the serialized payload so size accounting adds no second traversal. */
|
||||
batch.bytes += payload?.length ?? 0;
|
||||
const receipt = new Promise<number | false | undefined>((resolve) => {
|
||||
batch.resolvers.push(resolve);
|
||||
});
|
||||
|
|
@ -566,12 +585,12 @@ export class RedisEventTransport implements IEventTransport {
|
|||
}
|
||||
|
||||
/**
|
||||
* Publish the stream's pending coalesced chunks as one CHUNK_BATCH frame.
|
||||
* Publish the pending window in one EVAL using legacy CHUNK frames.
|
||||
*
|
||||
* One INCRBY reserves a consecutive sequence per buffered event, so subscribers
|
||||
* unpack the frame into individually sequenced chunks and the reorder buffer is
|
||||
* none the wiser. The events array is spliced from pre-serialized payloads for
|
||||
* the same reason single frames are: no server-side re-encoding.
|
||||
* One INCRBY reserves consecutive sequences and Lua publishes each serialized
|
||||
* suffix with its own sequence. This saves round trips and repeated guard/TTL
|
||||
* work without requiring older subscribers to understand CHUNK_BATCH. Payloads
|
||||
* are spliced, not decoded/re-encoded by Lua, preserving JSON semantics.
|
||||
*/
|
||||
private flushCoalescedChunks(streamId: string): Promise<void> {
|
||||
const pending = this.pendingBatches.get(streamId);
|
||||
|
|
@ -585,19 +604,17 @@ export class RedisEventTransport implements IEventTransport {
|
|||
}
|
||||
|
||||
const { generationId, events, resolvers } = pending;
|
||||
const prefix = `{"type":${JSON.stringify(EventTypes.CHUNK_BATCH)},"baseSeq":`;
|
||||
const suffix =
|
||||
(generationId != null ? `,"generationId":${generationId}` : '') +
|
||||
`,"events":[${events.join(',')}]}`;
|
||||
const prefix = `{"type":${JSON.stringify(EventTypes.CHUNK)},"seq":`;
|
||||
|
||||
return this.evalPublishSequenced(
|
||||
streamId,
|
||||
prefix,
|
||||
suffix,
|
||||
'',
|
||||
events.length,
|
||||
generationId,
|
||||
false,
|
||||
generationId != null,
|
||||
events,
|
||||
).then(
|
||||
(baseSeq) => {
|
||||
if (baseSeq === -1) {
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@
|
|||
const MAX_COALESCE_WINDOW_MS = 1000;
|
||||
/** Coalesced-batch safety caps: a full buffer flushes immediately, ahead of the window. */
|
||||
export const MAX_COALESCED_EVENTS = 64;
|
||||
/** Shared raw-event JSON string-length budget, excluding transport envelopes.
|
||||
* Keep both coalescers on the same accounting basis, including for non-ASCII JSON. */
|
||||
export const MAX_COALESCED_BYTES = 128 * 1024;
|
||||
|
||||
/**
|
||||
|
|
@ -14,10 +16,10 @@ export const MAX_COALESCED_BYTES = 128 * 1024;
|
|||
* resume frontier assumes an event is never visible in the durable chunk log
|
||||
* meaningfully earlier than its sequence lands on the shared counter, so
|
||||
* batching one side without the other reopens that race for the full window
|
||||
* instead of a same-tick skew. 0 (the default) disables coalescing.
|
||||
* instead of a same-tick skew. Unset uses 25ms; explicit 0 disables coalescing.
|
||||
*/
|
||||
export function resolveCoalesceWindowMs(): number {
|
||||
const raw = Number(process.env.STREAM_DELTA_COALESCE_MS ?? 0);
|
||||
const raw = Number(process.env.STREAM_DELTA_COALESCE_MS ?? 25);
|
||||
if (!Number.isFinite(raw) || raw <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue