mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-20 06:51:59 +00:00
🪞 fix: Deliver Each Text Delta Once When a Redis Resume Lands on the Hosting Replica (#18418)
* 🪞 fix: Deliver Each Text Delta Once When a Redis Resume Lands on the Hosting Replica On the replica that hosts a run, a Redis resume snapshot is the run's live content array. It was read before the Pub/Sub subscription existed and serialized after it, so every text delta produced in between reached the client twice: once in the snapshot and once from Pub/Sub. The client appends deltas, so the text doubled until the final message replaced it. Publish each chunk with the emitting manager's id and emission count. After subscribing, take the snapshot again as a copy at a known count, and skip that manager's text and reasoning deltas up to it. Other events are still delivered, since the client applies them idempotently and run-step deltas are not always folded into the content. * 🪞 fix: Anchor a Hosted Resume on a Local Content Copy Instead of a Second State Read Anchoring the snapshot by reading the whole resume state again added three Redis read phases to every resume on the hosting replica. The content, the emission count and the run steps the copy needs are all local, so take them synchronously after the fenced job read the resume already makes, and keep the rest of the state from the first read.
This commit is contained in:
parent
89b67d141d
commit
25a5a80d5d
8 changed files with 480 additions and 11 deletions
|
|
@ -86,7 +86,11 @@ import { InMemoryEventTransport } from './implementations/InMemoryEventTransport
|
|||
import { InMemoryJobStore } from './implementations/InMemoryJobStore';
|
||||
import { attachAskUserQuestionAnswers, normalizeResumeRunStepIndices } from '~/agents/hitl/resume';
|
||||
import { ASK_USER_QUESTION_TOOL_NAME } from '~/agents/hitl/askUserQuestionTool';
|
||||
import { emitChunkWithReceipt } from './internal/chunkPublication';
|
||||
import {
|
||||
readChunkEmission,
|
||||
stampChunkEmission,
|
||||
emitChunkWithReceipt,
|
||||
} from './internal/chunkPublication';
|
||||
import { resolveCoalesceWindowMs } from './internal/coalescing';
|
||||
import {
|
||||
REDIS_ABORT_TERMINAL_GRACE_MS,
|
||||
|
|
@ -519,6 +523,15 @@ function isCoalescableDeltaEvent(eventType: string | undefined): boolean {
|
|||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deltas a client appends to text it already holds, and that every emitter folds into the
|
||||
* run's content before emitting. Run-step deltas are left out: OAuth and action prompts emit
|
||||
* them without folding, so a content snapshot cannot be assumed to hold one.
|
||||
*/
|
||||
function isAppendedTextDelta(eventType: string | undefined): boolean {
|
||||
return eventType === 'on_message_delta' || eventType === 'on_reasoning_delta';
|
||||
}
|
||||
|
||||
function getReplayStepId(event: t.ServerSentEvent): unknown {
|
||||
if (!('event' in event) || !event.data || typeof event.data !== 'object') {
|
||||
return undefined;
|
||||
|
|
@ -912,6 +925,14 @@ class GenerationJobManagerClass {
|
|||
/** Jobs actively owned by this process, pinned to their durable creation epoch. */
|
||||
private ownedJobs = new Map<string, number>();
|
||||
|
||||
/**
|
||||
* This manager's identity on published chunks, with a count of everything it has emitted.
|
||||
* A resume on the replica that hosts a run copies the run's content at a known count, and
|
||||
* the pair lets it recognise its own deltas up to that count when Pub/Sub hands them back.
|
||||
*/
|
||||
private readonly emitterId = randomUUID();
|
||||
private emissionOrdinal = 0;
|
||||
|
||||
/** Serializes replay-event read/modify/write updates per stream. */
|
||||
private replayEventWriteQueues = new Map<string, Promise<void>>();
|
||||
|
||||
|
|
@ -6441,6 +6462,8 @@ class GenerationJobManagerClass {
|
|||
sequence: number;
|
||||
}> = [];
|
||||
let snapshotFrontier = 0;
|
||||
/** This manager's emission count when a hosted run's content was copied for a Redis client. */
|
||||
let localFrontier: number | undefined;
|
||||
let snapshotClassified = this._isRedis;
|
||||
const classifyEmission = (event: t.ServerSentEvent, sequence: number): void => {
|
||||
if (sequence <= snapshotFrontier) {
|
||||
|
|
@ -6698,6 +6721,12 @@ class GenerationJobManagerClass {
|
|||
if (capturedEventSet.has(event) || snapshotCoveredEventSet.has(event)) {
|
||||
return;
|
||||
}
|
||||
if (localFrontier != null && isAppendedTextDelta((event as { event?: string }).event)) {
|
||||
const emission = readChunkEmission(event);
|
||||
if (emission?.[0] === this.emitterId && emission[1] <= localFrontier) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
onChunk(event);
|
||||
};
|
||||
subscription = await this.attachSubscription(
|
||||
|
|
@ -6776,6 +6805,34 @@ class GenerationJobManagerClass {
|
|||
return this.subscribeWithResume(streamId, onChunk, onDone, onError, options);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* On the replica that hosts the run, the snapshot's content is the run's live array.
|
||||
* It was read before the subscription existed and is serialized after it, so every
|
||||
* delta in between reached the client twice: once in the snapshot and once from
|
||||
* Pub/Sub. Replace it with a copy taken now, at a known emission count, and let
|
||||
* delivery skip this manager's text and reasoning deltas up to that count. Everything
|
||||
* else is still delivered: the client applies those events idempotently, and only the
|
||||
* content is exact at the count. The copy, the count and the run steps are all local,
|
||||
* so the sync frame waits on no further read.
|
||||
*/
|
||||
if (this._isRedis && resumeState != null && pendingOverflow == null && liveOverflow == null) {
|
||||
const frontier = this.emissionOrdinal;
|
||||
const content = this.jobStore.copyHostContent?.(streamId, runtime.createdAt);
|
||||
if (content != null) {
|
||||
localFrontier = frontier;
|
||||
resumeState.aggregatedContent = attachAskUserQuestionAnswers(
|
||||
content,
|
||||
liveJob.resolvedAskUserQuestions ?? [],
|
||||
);
|
||||
resumeState.runSteps = this.withBufferedRunSteps(
|
||||
streamId,
|
||||
runtime.createdAt,
|
||||
resumeState.runSteps,
|
||||
content,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (!resumeState?.pendingAction) {
|
||||
if (
|
||||
liveJob?.status === 'requires_action' &&
|
||||
|
|
@ -7129,6 +7186,11 @@ class GenerationJobManagerClass {
|
|||
return;
|
||||
}
|
||||
const sequence = ++runtime.emissionSequence;
|
||||
/** Counted in the same synchronous step as the caller's fold into the run's content. */
|
||||
const ordinal = ++this.emissionOrdinal;
|
||||
if (this._isRedis) {
|
||||
stampChunkEmission(event, [this.emitterId, ordinal]);
|
||||
}
|
||||
let signalSnapshotReady!: () => void;
|
||||
const snapshotReady = new Promise<void>((resolve) => {
|
||||
signalSnapshotReady = resolve;
|
||||
|
|
@ -9099,6 +9161,22 @@ class GenerationJobManagerClass {
|
|||
this.clearPreemptIds(runtime, runtime.createdAt, steerIds);
|
||||
}
|
||||
|
||||
/** Stored run steps under the ones this replica has seen since, indexed against `content`. */
|
||||
private withBufferedRunSteps(
|
||||
streamId: string,
|
||||
createdAt: number,
|
||||
runSteps: Agents.RunStep[],
|
||||
content: Agents.MessageContentComplex[],
|
||||
): Agents.RunStep[] {
|
||||
const bufferState = this.runStepBuffers?.get(streamId);
|
||||
const bufferedRunSteps = bufferState?.createdAt === createdAt ? bufferState.steps : [];
|
||||
const runStepsById = new Map(runSteps.map((runStep) => [runStep.id, runStep]));
|
||||
for (const runStep of bufferedRunSteps) {
|
||||
runStepsById.set(runStep.id, runStep);
|
||||
}
|
||||
return normalizeResumeRunStepIndices([...runStepsById.values()], content);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get resume state for reconnecting clients.
|
||||
*/
|
||||
|
|
@ -9225,14 +9303,10 @@ class GenerationJobManagerClass {
|
|||
}
|
||||
}
|
||||
const reconstructedContent = result?.content ?? [];
|
||||
const bufferState = this.runStepBuffers?.get(streamId);
|
||||
const bufferedRunSteps = bufferState?.createdAt === jobData.createdAt ? bufferState.steps : [];
|
||||
const runStepsById = new Map(runSteps.map((runStep) => [runStep.id, runStep]));
|
||||
for (const runStep of bufferedRunSteps) {
|
||||
runStepsById.set(runStep.id, runStep);
|
||||
}
|
||||
const effectiveRunSteps = normalizeResumeRunStepIndices(
|
||||
[...runStepsById.values()],
|
||||
const effectiveRunSteps = this.withBufferedRunSteps(
|
||||
streamId,
|
||||
jobData.createdAt,
|
||||
runSteps,
|
||||
reconstructedContent,
|
||||
);
|
||||
let titleEvent: t.ResumeState['titleEvent'];
|
||||
|
|
|
|||
|
|
@ -1,6 +1,11 @@
|
|||
import { logger } from '@librechat/data-schemas';
|
||||
import type { Redis } from 'ioredis';
|
||||
import { emitChunkWithReceipt, emitObservedChunk } from '~/stream/internal/chunkPublication';
|
||||
import {
|
||||
emitObservedChunk,
|
||||
readChunkEmission,
|
||||
stampChunkEmission,
|
||||
emitChunkWithReceipt,
|
||||
} from '~/stream/internal/chunkPublication';
|
||||
import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';
|
||||
import { RedisEventTransport } from '~/stream/implementations/RedisEventTransport';
|
||||
import { GenerationPublicationFencedError } from '~/stream/interfaces/IJobStore';
|
||||
|
|
@ -2462,4 +2467,64 @@ describe('RedisEventTransport', () => {
|
|||
disposeAbort();
|
||||
transport.destroy();
|
||||
});
|
||||
|
||||
it('hands a subscriber the emission a chunk was published with, per event and batched', async () => {
|
||||
const mockPublisher = createMockPublisher();
|
||||
const mockSubscriber = createMockSubscriber();
|
||||
const transport = new RedisEventTransport(
|
||||
mockPublisher as unknown as Redis,
|
||||
mockSubscriber as unknown as Redis,
|
||||
);
|
||||
const streamId = 'redis-chunk-emission';
|
||||
const received: unknown[] = [];
|
||||
const subscription = transport.subscribe(streamId, {
|
||||
onChunk: (event) => received.push(event),
|
||||
});
|
||||
const messageHandler = getMessageHandler(mockSubscriber);
|
||||
mockPublisher.publish.mockImplementation(async (channel: string, payload: string) => {
|
||||
messageHandler(channel, payload);
|
||||
return 1;
|
||||
});
|
||||
await subscription.ready;
|
||||
|
||||
const single = { delta: 'single' };
|
||||
const batched = { delta: 'batched' };
|
||||
const unstamped = { delta: 'unstamped' };
|
||||
stampChunkEmission(single, ['manager-a', 7]);
|
||||
stampChunkEmission(batched, ['manager-a', 8]);
|
||||
await emitChunkWithReceipt(transport, streamId, single, 101);
|
||||
await emitChunkWithReceipt(transport, streamId, batched, 101, { coalesce: true });
|
||||
await emitChunkWithReceipt(transport, streamId, unstamped, 101);
|
||||
|
||||
expect(received).toEqual([single, batched, unstamped]);
|
||||
expect(received.map(readChunkEmission)).toEqual([
|
||||
['manager-a', 7],
|
||||
['manager-a', 8],
|
||||
undefined,
|
||||
]);
|
||||
|
||||
subscription.unsubscribe();
|
||||
transport.destroy();
|
||||
});
|
||||
|
||||
it('ignores an emission an older or foreign publisher shaped differently', () => {
|
||||
const mockSubscriber = createMockSubscriber();
|
||||
const transport = new RedisEventTransport(
|
||||
createMockPublisher() as unknown as Redis,
|
||||
mockSubscriber as unknown as Redis,
|
||||
);
|
||||
const streamId = 'redis-chunk-emission-malformed';
|
||||
const received: unknown[] = [];
|
||||
transport.subscribe(streamId, { onChunk: (event) => received.push(event) });
|
||||
const messageHandler = getMessageHandler(mockSubscriber);
|
||||
const channel = `stream:{${streamId}}:events`;
|
||||
|
||||
for (const [seq, emission] of [['manager-a'], 'manager-a:3', ['manager-a', '3']].entries()) {
|
||||
messageHandler(channel, JSON.stringify({ type: 'chunk', seq, data: { seq }, emission }));
|
||||
}
|
||||
|
||||
expect(received).toHaveLength(3);
|
||||
expect(received.map(readChunkEmission)).toEqual([undefined, undefined, undefined]);
|
||||
transport.destroy();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -746,6 +746,25 @@ describe('RedisJobStore', () => {
|
|||
);
|
||||
});
|
||||
|
||||
test('copies the content of a hosted run without touching Redis', async () => {
|
||||
const redis = { isCluster: true, eval: jest.fn() } as unknown as Cluster;
|
||||
const store = new RedisJobStore(redis);
|
||||
const live = [{ type: 'text', text: 'one ' }];
|
||||
store.setContentParts('stream-host-copy', live as never, 101);
|
||||
|
||||
const copy = store.copyHostContent('stream-host-copy', 101);
|
||||
/** The run keeps writing; the copy holds what it had at the call. */
|
||||
live[0] = { type: 'text', text: 'one two ' };
|
||||
|
||||
expect(copy).toEqual([{ type: 'text', text: 'one ' }]);
|
||||
await expect(store.getContentParts('stream-host-copy', 101)).resolves.toEqual({
|
||||
content: [{ type: 'text', text: 'one two ' }],
|
||||
});
|
||||
expect(store.copyHostContent('stream-host-copy', 100)).toBeNull();
|
||||
expect(store.copyHostContent('stream-not-hosted')).toBeNull();
|
||||
expect(redis.eval).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('guards local content caches by creation epoch', async () => {
|
||||
const evalRedis = jest.fn().mockResolvedValue(false);
|
||||
const redis = {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,232 @@
|
|||
import { logger } from '@librechat/data-schemas';
|
||||
import { createContentAggregator } from '@librechat/agents';
|
||||
import type { Agents } from 'librechat-data-provider';
|
||||
import type { Redis, Cluster } from 'ioredis';
|
||||
import type { IJobStore } from '~/stream/interfaces/IJobStore';
|
||||
import type * as t from '~/types';
|
||||
import {
|
||||
ioredisClient as staticRedisClient,
|
||||
keyvRedisClient as staticKeyvClient,
|
||||
keyvRedisClientReady,
|
||||
} from '~/cache/redisClients';
|
||||
import { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
|
||||
import { createStreamServices } from '~/stream/createStreamServices';
|
||||
|
||||
logger.silent = true;
|
||||
|
||||
type AggregatedEvent = Parameters<
|
||||
ReturnType<typeof createContentAggregator>['aggregateContent']
|
||||
>[0];
|
||||
|
||||
type HostContentStore = IJobStore & Required<Pick<IJobStore, 'copyHostContent'>>;
|
||||
|
||||
/** Where in the attachment the run produces its next delta. */
|
||||
type Window = 'before-subscribe' | 'after-subscribe' | 'after-snapshot';
|
||||
|
||||
const STEP_ID = 'step-message';
|
||||
|
||||
const delta = (text: string): t.ServerSentEvent => ({
|
||||
event: 'on_message_delta',
|
||||
data: { id: STEP_ID, delta: { content: { type: 'text', text } } },
|
||||
});
|
||||
|
||||
const PROMPT_EVENT = 'on_run_step_delta';
|
||||
|
||||
const PROMPT: t.ServerSentEvent = {
|
||||
event: PROMPT_EVENT,
|
||||
data: { id: 'step-prompt', delta: { type: 'tool_calls', auth: 'https://example.test/sign-in' } },
|
||||
};
|
||||
|
||||
const deltaText = (event: t.ServerSentEvent): string => {
|
||||
const data = (event as { data?: { delta?: { content?: { text?: string } } } }).data;
|
||||
return data?.delta?.content?.text ?? '';
|
||||
};
|
||||
|
||||
/**
|
||||
* A resuming client holds the sync frame's snapshot, then its pending events, then what the
|
||||
* subscription delivers. On the replica that hosts the run the snapshot is the run's own
|
||||
* content, so every delta has to land in exactly one of the three.
|
||||
*
|
||||
* Run with: USE_REDIS=true npx jest resumeFrontier
|
||||
*/
|
||||
describe('Resume snapshot frontier on the hosting replica (real Redis)', () => {
|
||||
let originalEnv: NodeJS.ProcessEnv;
|
||||
let ioredisClient: Redis | Cluster | null = null;
|
||||
let dynamicKeyvClient: unknown = null;
|
||||
let dynamicKeyvReady: Promise<unknown> | null = null;
|
||||
const testPrefix = 'ResumeFrontier-Test';
|
||||
|
||||
beforeAll(async () => {
|
||||
originalEnv = { ...process.env };
|
||||
process.env.USE_REDIS = process.env.USE_REDIS ?? 'true';
|
||||
process.env.REDIS_URI = process.env.REDIS_URI ?? 'redis://127.0.0.1:6379';
|
||||
process.env.REDIS_KEY_PREFIX = testPrefix;
|
||||
jest.resetModules();
|
||||
const redisModule = await import('~/cache/redisClients');
|
||||
ioredisClient = redisModule.ioredisClient;
|
||||
dynamicKeyvClient = redisModule.keyvRedisClient;
|
||||
dynamicKeyvReady = redisModule.keyvRedisClientReady;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
jest.restoreAllMocks();
|
||||
jest.resetModules();
|
||||
if (!ioredisClient) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const keys = await ioredisClient.keys(`${testPrefix}*`);
|
||||
const streamKeys = await ioredisClient.keys('stream:*');
|
||||
await Promise.all([...keys, ...streamKeys].map((key) => ioredisClient!.del(key)));
|
||||
} catch {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
for (const ready of [keyvRedisClientReady, dynamicKeyvReady]) {
|
||||
await ready?.catch(() => {});
|
||||
}
|
||||
for (const client of [ioredisClient, staticRedisClient, staticKeyvClient, dynamicKeyvClient]) {
|
||||
try {
|
||||
await (client as { disconnect: () => void | Promise<void> } | null)?.disconnect();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
process.env = originalEnv;
|
||||
});
|
||||
|
||||
/**
|
||||
* Resumes a run that produces `two ` in the named window and `three` once delivery is live,
|
||||
* and returns the text a client would hold beside the run's own. A reconnect has had a
|
||||
* subscriber before, which is what lets the run batch its deltas on the way to Redis.
|
||||
*/
|
||||
async function resumeWhileStreaming(
|
||||
window: Window,
|
||||
attachment: 'first' | 'reconnect',
|
||||
): Promise<{ client: string; host: string; prompts: number }> {
|
||||
const manager = new GenerationJobManagerClass();
|
||||
const services = createStreamServices({ useRedis: true, redisClient: ioredisClient! });
|
||||
manager.configure(services);
|
||||
manager.initialize();
|
||||
|
||||
const streamId = `resume-frontier-${window}-${attachment}-${Date.now()}`;
|
||||
await manager.createJob(streamId, 'user-1');
|
||||
const { contentParts, aggregateContent } = createContentAggregator();
|
||||
manager.setContentParts(streamId, contentParts as Agents.MessageContentComplex[]);
|
||||
/** The order the run's own handlers use: fold into the content, then emit. */
|
||||
const emit = async (event: t.ServerSentEvent): Promise<void> => {
|
||||
aggregateContent(event as unknown as AggregatedEvent);
|
||||
await manager.emitChunk(streamId, event);
|
||||
};
|
||||
await emit({
|
||||
event: 'on_run_step',
|
||||
data: { id: STEP_ID, runId: 'run-1', index: 0, stepDetails: { type: 'message_creation' } },
|
||||
});
|
||||
if (attachment === 'reconnect') {
|
||||
const earlier = await manager.subscribe(streamId, () => {});
|
||||
await emit(delta('zero '));
|
||||
earlier!.unsubscribe();
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
await emit(delta('one '));
|
||||
|
||||
let fired = false;
|
||||
const produce = async (): Promise<void> => {
|
||||
if (!fired) {
|
||||
fired = true;
|
||||
await emit(delta('two '));
|
||||
/** A sign-in prompt is emitted without being folded into the run's content. */
|
||||
await manager.emitChunk(streamId, PROMPT);
|
||||
}
|
||||
};
|
||||
const getResumeState = manager.getResumeState.bind(manager);
|
||||
const stateReads = jest.spyOn(manager, 'getResumeState').mockImplementation(async (...args) => {
|
||||
const state = await getResumeState(...args);
|
||||
if (window === 'before-subscribe') {
|
||||
await produce();
|
||||
}
|
||||
return state;
|
||||
});
|
||||
const transport = services.eventTransport;
|
||||
const sync = transport.syncReorderBuffer!.bind(transport);
|
||||
jest.spyOn(transport, 'syncReorderBuffer').mockImplementation(async (...args) => {
|
||||
await sync(...args);
|
||||
if (window === 'after-subscribe') {
|
||||
await produce();
|
||||
}
|
||||
});
|
||||
const store = services.jobStore as HostContentStore;
|
||||
const copyHostContent = store.copyHostContent.bind(store);
|
||||
jest.spyOn(store, 'copyHostContent').mockImplementation((...args) => {
|
||||
const content = copyHostContent(...args);
|
||||
if (window === 'after-snapshot') {
|
||||
/** As soon as the manager's synchronous step around the copy has finished. */
|
||||
queueMicrotask(() => void produce());
|
||||
}
|
||||
return content;
|
||||
});
|
||||
|
||||
const live: t.ServerSentEvent[] = [];
|
||||
const { subscription, resumeState, pendingEvents } = await manager.subscribeWithResume(
|
||||
streamId,
|
||||
(event) => live.push(event),
|
||||
);
|
||||
expect(fired).toBe(true);
|
||||
expect(subscription).not.toBeNull();
|
||||
/** The copy is local: a hosted resume reads the resume state from Redis once. */
|
||||
expect(stateReads).toHaveBeenCalledTimes(1);
|
||||
/** The route serializes the frame here, and only then lets live delivery start. */
|
||||
const frame = JSON.parse(JSON.stringify({ resumeState, pendingEvents })) as {
|
||||
resumeState: t.ResumeState;
|
||||
pendingEvents: t.ServerSentEvent[];
|
||||
};
|
||||
subscription!.activate();
|
||||
await emit(delta('three'));
|
||||
|
||||
const deadline = Date.now() + 3000;
|
||||
while (!live.some((event) => deltaText(event) === 'three') && Date.now() < deadline) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
}
|
||||
|
||||
const snapshot = (frame.resumeState.aggregatedContent ?? []) as Array<{ text?: string }>;
|
||||
const client =
|
||||
(snapshot[0]?.text ?? '') +
|
||||
frame.pendingEvents.map(deltaText).join('') +
|
||||
live.map(deltaText).join('');
|
||||
const host = (contentParts[0] as { text?: string }).text ?? '';
|
||||
const prompts = [...frame.pendingEvents, ...live].filter(
|
||||
(event) => (event as { event?: string }).event === PROMPT_EVENT,
|
||||
).length;
|
||||
subscription!.unsubscribe();
|
||||
await manager.destroy();
|
||||
return { client, host, prompts };
|
||||
}
|
||||
|
||||
describe.each(['first', 'reconnect'] as const)('%s attachment', (attachment) => {
|
||||
const expected = attachment === 'reconnect' ? 'zero one two three' : 'one two three';
|
||||
|
||||
test.each(['before-subscribe', 'after-subscribe', 'after-snapshot'] as const)(
|
||||
'a delta produced %s reaches the resuming client exactly once',
|
||||
async (window) => {
|
||||
if (!ioredisClient) {
|
||||
console.warn('Redis not available, skipping test');
|
||||
return;
|
||||
}
|
||||
const { client, host } = await resumeWhileStreaming(window, attachment);
|
||||
expect(host).toBe(expected);
|
||||
expect(client).toBe(expected);
|
||||
},
|
||||
);
|
||||
|
||||
test('a run-step delta the run did not fold into its content is still delivered', async () => {
|
||||
if (!ioredisClient) {
|
||||
console.warn('Redis not available, skipping test');
|
||||
return;
|
||||
}
|
||||
const { prompts } = await resumeWhileStreaming('after-subscribe', attachment);
|
||||
expect(prompts).toBe(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -4,6 +4,12 @@ import type { Redis, Cluster } from 'ioredis';
|
|||
import type { IEventTransport, PreemptMessage } from '~/stream/interfaces/IJobStore';
|
||||
import type { ChunkPublicationOptions } from '~/stream/internal/chunkPublication';
|
||||
import type { ReplayLimits, ReplayPublication } from '../internal/replay';
|
||||
import {
|
||||
readChunkEmission,
|
||||
parseChunkEmission,
|
||||
stampChunkEmission,
|
||||
registerChunkPublicationCapability,
|
||||
} from '~/stream/internal/chunkPublication';
|
||||
import {
|
||||
MAX_COALESCED_BYTES,
|
||||
MAX_COALESCED_EVENTS,
|
||||
|
|
@ -13,7 +19,6 @@ import {
|
|||
REDIS_ABORT_ACK_TIMEOUT_MS,
|
||||
REDIS_EVENT_REORDER_TIMEOUT_MS,
|
||||
} from '~/stream/internal/timing';
|
||||
import { registerChunkPublicationCapability } from '~/stream/internal/chunkPublication';
|
||||
import { GenerationPublicationFencedError } from '~/stream/interfaces/IJobStore';
|
||||
import { instrumentIORedisClient, RedisUseCases } from '~/cache/redisTelemetry';
|
||||
import { PUBLISH_REPLAY_LUA, READ_REPLAY_LUA } from '../internal/replay';
|
||||
|
|
@ -75,6 +80,8 @@ interface PubSubMessage {
|
|||
events?: unknown[];
|
||||
/** Immutable identity of the generation that emitted the event. */
|
||||
generationId?: number;
|
||||
/** The emitting manager and its emission count for a CHUNK; absent from older publishers. */
|
||||
emission?: readonly [string, number];
|
||||
/** Opaque nonce linking a replacement abort to its owner acknowledgement. */
|
||||
abortRequestId?: string;
|
||||
/** Payload for PREEMPT messages; fenced by its own createdAt. */
|
||||
|
|
@ -546,12 +553,14 @@ export class RedisEventTransport implements IEventTransport {
|
|||
if (this.pendingBatches.has(streamId)) {
|
||||
void this.flushCoalescedChunks(streamId);
|
||||
}
|
||||
const emission = readChunkEmission(event);
|
||||
return this.publishWithSequence(
|
||||
streamId,
|
||||
{
|
||||
type: EventTypes.CHUNK,
|
||||
data: event,
|
||||
...(generationId != null && { generationId }),
|
||||
...(emission != null && { emission }),
|
||||
},
|
||||
generationId,
|
||||
false,
|
||||
|
|
@ -589,9 +598,11 @@ export class RedisEventTransport implements IEventTransport {
|
|||
|
||||
const batch = pending;
|
||||
const payload = JSON.stringify(event);
|
||||
const emission = readChunkEmission(event);
|
||||
const [, envelopeSuffix] = RedisEventTransport.buildPayloadParts({
|
||||
type: EventTypes.CHUNK,
|
||||
...(generationId != null && { generationId }),
|
||||
...(emission != null && { emission }),
|
||||
});
|
||||
batch.events.push(
|
||||
payload === undefined ? envelopeSuffix : `,"data":${payload}${envelopeSuffix}`,
|
||||
|
|
@ -1143,6 +1154,12 @@ export class RedisEventTransport implements IEventTransport {
|
|||
streamState: StreamSubscribers,
|
||||
message: PubSubMessage,
|
||||
): void {
|
||||
if (message.type === EventTypes.CHUNK) {
|
||||
const emission = parseChunkEmission(message.emission);
|
||||
if (emission != null) {
|
||||
stampChunkEmission(message.data, emission);
|
||||
}
|
||||
}
|
||||
for (const [, handlers] of streamState.handlers) {
|
||||
if (handlers.onReplayMessage != null) continue;
|
||||
switch (message.type) {
|
||||
|
|
|
|||
|
|
@ -4407,6 +4407,23 @@ export class RedisJobStore implements IJobStoreV2 {
|
|||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* A copy of the content of a run this replica hosts, taken synchronously. The run folds an
|
||||
* event into that array and emits it in one synchronous step, so the copy holds exactly
|
||||
* what the run had emitted at the instant of the call.
|
||||
*/
|
||||
copyHostContent(
|
||||
streamId: string,
|
||||
expectedCreatedAt?: number,
|
||||
): Agents.MessageContentComplex[] | null {
|
||||
const hostParts = this.getLocalEntry(
|
||||
this.localContentParts,
|
||||
streamId,
|
||||
expectedCreatedAt,
|
||||
)?.value.deref();
|
||||
return hostParts != null && hostParts.length > 0 ? structuredClone(hostParts) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store collectedUsage reference in local cache.
|
||||
* This is used for abort handling to spend tokens for all models.
|
||||
|
|
|
|||
|
|
@ -1110,6 +1110,15 @@ export interface IJobStore {
|
|||
contentParts: Agents.MessageContentComplex[],
|
||||
expectedCreatedAt?: number,
|
||||
): void;
|
||||
/**
|
||||
* A synchronous copy of the content array `setContentParts` registered on this replica, or
|
||||
* null when this replica does not host the run. A store that only ever returns copies from
|
||||
* `getContentParts` has no need for it.
|
||||
*/
|
||||
copyHostContent?(
|
||||
streamId: string,
|
||||
expectedCreatedAt?: number,
|
||||
): Agents.MessageContentComplex[] | null;
|
||||
getContentParts(
|
||||
streamId: string,
|
||||
expectedCreatedAt?: number,
|
||||
|
|
|
|||
|
|
@ -19,6 +19,42 @@ export interface ChunkPublicationOptions {
|
|||
coalesce?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Who emitted a chunk and where it sits in that emitter's order: a manager's id and its
|
||||
* emission count. A subscriber on the emitting replica compares it with the count its resume
|
||||
* snapshot was copied at, so it can tell a delta the snapshot already holds from a later one.
|
||||
*/
|
||||
export type ChunkEmission = readonly [emitter: string, ordinal: number];
|
||||
|
||||
/**
|
||||
* Beside the event rather than inside it: the event is the client's payload, and the
|
||||
* publication call shapes are part of the transport contract.
|
||||
*/
|
||||
const chunkEmissions = new WeakMap<object, ChunkEmission>();
|
||||
|
||||
export function stampChunkEmission(event: unknown, emission: ChunkEmission): void {
|
||||
if (event != null && typeof event === 'object') {
|
||||
chunkEmissions.set(event, emission);
|
||||
}
|
||||
}
|
||||
|
||||
export function readChunkEmission(event: unknown): ChunkEmission | undefined {
|
||||
return event != null && typeof event === 'object' ? chunkEmissions.get(event) : undefined;
|
||||
}
|
||||
|
||||
/** A stamp read off the wire, where an older or foreign publisher may have sent anything. */
|
||||
export function parseChunkEmission(value: unknown): ChunkEmission | undefined {
|
||||
if (
|
||||
Array.isArray(value) &&
|
||||
value.length === 2 &&
|
||||
typeof value[0] === 'string' &&
|
||||
Number.isSafeInteger(value[1])
|
||||
) {
|
||||
return [value[0], value[1]];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
type ChunkPublicationCapability = (
|
||||
streamId: string,
|
||||
event: unknown,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue