mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-20 15:02:02 +00:00
🧭 refactor: Give the Resumable Stream a Core Run and Subscription (#17117)
This commit is contained in:
parent
0354562075
commit
5364225d0c
9 changed files with 1073 additions and 737 deletions
|
|
@ -1,17 +1,14 @@
|
|||
import { request } from 'librechat-data-provider';
|
||||
import { GENERATION_PROTOCOL_VERSION } from '@librechat/chat';
|
||||
import { GENERATION_PROTOCOL_VERSION, generationProtocolHeaders } from '@librechat/chat';
|
||||
|
||||
export {
|
||||
GENERATION_PROTOCOL_HEADER,
|
||||
GENERATION_PROTOCOL_VERSION,
|
||||
generationProtocolHeaders,
|
||||
getGenerationProtocolVersion,
|
||||
supportsGenerationProtocolV2,
|
||||
} from '@librechat/chat';
|
||||
export type { GenerationProtocolVersion } from '@librechat/chat';
|
||||
export const GENERATION_PROTOCOL_HEADER = 'X-LibreChat-Generation-Protocol';
|
||||
|
||||
export const generationProtocolHeaders = (): Record<string, string> => ({
|
||||
[GENERATION_PROTOCOL_HEADER]: String(GENERATION_PROTOCOL_VERSION),
|
||||
});
|
||||
|
||||
export const withGenerationProtocolQuery = (url: string): string =>
|
||||
`${url}${url.includes('?') ? '&' : '?'}generationProtocolVersion=${GENERATION_PROTOCOL_VERSION}`;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import { useEffect, useMemo, useState } from 'react';
|
||||
import { useQuery, useQueries, useQueryClient } from '@tanstack/react-query';
|
||||
import { apiBaseUrl, QueryKeys, request, dataService } from 'librechat-data-provider';
|
||||
import type { Agents, TConversation, TPendingSteer } from 'librechat-data-provider';
|
||||
import type { TConversation } from 'librechat-data-provider';
|
||||
import type { StreamStatusResponse } from '@librechat/chat';
|
||||
import {
|
||||
isNotFoundError,
|
||||
updateConvoInAllQueries,
|
||||
|
|
@ -11,32 +12,7 @@ import {
|
|||
import { generationProtocolHeaders, withGenerationProtocolQuery } from './protocol';
|
||||
import { useGetStartupConfig } from '../Endpoints';
|
||||
|
||||
export interface StreamStatusResponse {
|
||||
/** Exact protocol selected by the job. Missing/1 is always legacy. */
|
||||
generationProtocolVersion?: number;
|
||||
active: boolean;
|
||||
/** Client-local cache marker used to force useResumeOnLoad to replace a
|
||||
* stale generation attachment with the newer epoch occupying this stream. */
|
||||
generationHandoff?: boolean;
|
||||
streamId?: string;
|
||||
status?: 'running' | 'complete' | 'error' | 'aborted' | 'requires_action';
|
||||
aggregatedContent?: Array<{ type: string; text?: string }>;
|
||||
createdAt?: number;
|
||||
/** Generation age computed on the server's clock, so a reattaching client
|
||||
* can rebuild a clock-local elapsed baseline free of cross-machine skew. */
|
||||
elapsedMs?: number;
|
||||
resumeState?: Agents.ResumeState;
|
||||
/** The run's temporary state as the server recorded it at admission, so a rebuilt
|
||||
* submission keeps a hidden chat out of history and titling after a reload. */
|
||||
isTemporary?: boolean;
|
||||
/** Live pending approval when `status === 'requires_action'`; mirrors
|
||||
* `resumeState.pendingAction`, surfaced top-level for the resume-on-load path. */
|
||||
pendingAction?: Agents.PendingAction;
|
||||
/** Acknowledged steers a terminal drain parked because no subscriber was
|
||||
* live for the final/abort event. Reads are replayable until an exact
|
||||
* recovery turn persists or the user discards the item. */
|
||||
unrecoveredSteers?: TPendingSteer[];
|
||||
}
|
||||
export type { StreamStatusResponse } from '@librechat/chat';
|
||||
|
||||
export const streamStatusQueryKey = (conversationId: string) => ['streamStatus', conversationId];
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
235
packages/chat/src/core/__tests__/subscription.spec.ts
Normal file
235
packages/chat/src/core/__tests__/subscription.spec.ts
Normal file
|
|
@ -0,0 +1,235 @@
|
|||
import type {
|
||||
TMessage,
|
||||
ChatEvent,
|
||||
TSubmission,
|
||||
ChatTransport,
|
||||
ChatTransportOptions,
|
||||
} from 'librechat-data-provider';
|
||||
import type {
|
||||
PageVisibility,
|
||||
ResumableScheduler,
|
||||
SubscriptionTarget,
|
||||
StreamStatusResponse,
|
||||
} from '../resumable';
|
||||
import { ResumableRun, ResumableSubscription } from '../resumable';
|
||||
|
||||
/** Timers and frames that run only when the test says so. */
|
||||
const createScheduler = () => {
|
||||
let now = 0;
|
||||
const timers = new Map<number, { at: number; callback: () => void }>();
|
||||
let nextId = 1;
|
||||
const scheduler: ResumableScheduler = {
|
||||
setTimeout: (callback, delay) => {
|
||||
const id = nextId++;
|
||||
timers.set(id, { at: now + delay, callback });
|
||||
return id as unknown as ReturnType<typeof setTimeout>;
|
||||
},
|
||||
clearTimeout: (handle) => {
|
||||
timers.delete(handle as unknown as number);
|
||||
},
|
||||
requestFrame: () => nextId++,
|
||||
cancelFrame: () => undefined,
|
||||
now: () => now,
|
||||
};
|
||||
const advance = (ms: number) => {
|
||||
now += ms;
|
||||
for (const [id, timer] of [...timers]) {
|
||||
if (timer.at <= now) {
|
||||
timers.delete(id);
|
||||
timer.callback();
|
||||
}
|
||||
}
|
||||
};
|
||||
return { scheduler, advance, pending: () => timers.size };
|
||||
};
|
||||
|
||||
const createVisibility = () => {
|
||||
let visible = true;
|
||||
const listeners = new Set<() => void>();
|
||||
const visibility: PageVisibility = {
|
||||
isVisible: () => visible,
|
||||
subscribe: (listener) => {
|
||||
listeners.add(listener);
|
||||
return () => listeners.delete(listener);
|
||||
},
|
||||
};
|
||||
const show = () => {
|
||||
visible = true;
|
||||
listeners.forEach((listener) => listener());
|
||||
};
|
||||
return { visibility, show, listeners };
|
||||
};
|
||||
|
||||
const submission = {
|
||||
userMessage: { messageId: 'u-1', conversationId: 'c-1' } as TMessage,
|
||||
initialResponse: { messageId: 'u-1_', conversationId: 'c-1' } as TMessage,
|
||||
conversation: { conversationId: 'c-1' },
|
||||
} as TSubmission;
|
||||
|
||||
const setup = (overrides: Partial<SubscriptionTarget> = {}) => {
|
||||
const clock = createScheduler();
|
||||
const page = createVisibility();
|
||||
const run = new ResumableRun(clock.scheduler);
|
||||
const fetchStatus = jest.fn<Promise<StreamStatusResponse>, [string]>();
|
||||
const subscribe = jest.fn<void, [SubscriptionTarget]>();
|
||||
const logger = { log: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
const target: SubscriptionTarget = {
|
||||
streamId: 'c-1',
|
||||
submission,
|
||||
isResume: false,
|
||||
generationCreatedAt: 2000,
|
||||
generationProtocolVersion: 2,
|
||||
...overrides,
|
||||
};
|
||||
run.submission = target.submission;
|
||||
const sub = new ResumableSubscription(
|
||||
run,
|
||||
{ logger, visibility: page.visibility, fetchStatus, subscribe },
|
||||
target,
|
||||
);
|
||||
return { run, sub, clock, page, fetchStatus, subscribe };
|
||||
};
|
||||
|
||||
describe('ResumableSubscription', () => {
|
||||
it('is current only while it holds the pane, its submission and its lifecycle', () => {
|
||||
const controller = new AbortController();
|
||||
const { run, sub } = setup({ signal: controller.signal });
|
||||
expect(sub.isCurrent()).toBe(false);
|
||||
sub.claim();
|
||||
expect(sub.isCurrent()).toBe(true);
|
||||
|
||||
sub.replaceSubmission({ ...submission, isRegenerate: true });
|
||||
expect(run.submission).toBe(sub.submission);
|
||||
expect(sub.isCurrent()).toBe(true);
|
||||
|
||||
run.submission = submission;
|
||||
expect(sub.isCurrent()).toBe(false);
|
||||
run.submission = sub.submission;
|
||||
controller.abort();
|
||||
expect(sub.isCurrent()).toBe(false);
|
||||
});
|
||||
|
||||
it('builds the stream route with the resume cursor and generation fence', () => {
|
||||
const { sub } = setup({ streamId: 'c 1', isResume: true });
|
||||
const url = new URL(sub.url(), 'http://localhost');
|
||||
expect(url.pathname).toBe('/api/agents/chat/stream/c%201');
|
||||
expect(Object.fromEntries(url.searchParams)).toEqual({
|
||||
resume: 'true',
|
||||
generationCreatedAt: '2000',
|
||||
generationProtocolVersion: '2',
|
||||
});
|
||||
});
|
||||
|
||||
it('routes connection events to their handlers', () => {
|
||||
const { sub } = setup();
|
||||
let options: ChatTransportOptions | undefined;
|
||||
const transport = {
|
||||
send: jest.fn(),
|
||||
reconnectToStream: jest.fn((_request, opts: ChatTransportOptions) => {
|
||||
options = opts;
|
||||
return { closed: false };
|
||||
}),
|
||||
} as ChatTransport;
|
||||
const handlers = {
|
||||
onOpen: jest.fn(),
|
||||
onFrame: jest.fn(),
|
||||
onFailure: jest.fn(),
|
||||
onClose: jest.fn(),
|
||||
};
|
||||
sub.connect(transport, handlers);
|
||||
|
||||
const emit = (event: ChatEvent) => options?.onEvent(event);
|
||||
emit({ type: 'open' } as ChatEvent);
|
||||
emit({ type: 'error', status: 0 });
|
||||
emit({ type: 'abort' });
|
||||
emit({ type: 'title', data: { event: 'title', data: { title: 't' } } } as ChatEvent);
|
||||
expect(handlers.onOpen).toHaveBeenCalledTimes(1);
|
||||
expect(handlers.onFailure).toHaveBeenCalledWith({ type: 'error', status: 0 });
|
||||
expect(handlers.onClose).toHaveBeenCalledTimes(1);
|
||||
expect(handlers.onFrame).toHaveBeenCalledTimes(1);
|
||||
expect(transport.reconnectToStream).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ headers: { 'X-LibreChat-Generation-Protocol': '2' } }),
|
||||
expect.objectContaining({ signal: sub.controller.signal }),
|
||||
);
|
||||
});
|
||||
|
||||
it('re-attaches after a delay only while it is still current', () => {
|
||||
const { run, sub, clock, subscribe } = setup();
|
||||
sub.claim();
|
||||
sub.scheduleReattach(1000);
|
||||
clock.advance(999);
|
||||
expect(subscribe).not.toHaveBeenCalled();
|
||||
clock.advance(1);
|
||||
expect(subscribe).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ streamId: 'c-1', isResume: true, generationCreatedAt: 2000 }),
|
||||
);
|
||||
|
||||
sub.scheduleReattach(1000, 'c-2', 3000);
|
||||
run.stream = new AbortController();
|
||||
clock.advance(1000);
|
||||
expect(subscribe).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('re-attaches on foreground when the connection closed while hidden', () => {
|
||||
const { run, sub, page, subscribe } = setup();
|
||||
sub.claim();
|
||||
sub.connection = { closed: true };
|
||||
sub.watchForeground();
|
||||
run.reconnectAttempt = 0;
|
||||
page.show();
|
||||
expect(sub.controller.signal.aborted).toBe(true);
|
||||
expect(run.reconnectAttempt).toBe(1);
|
||||
expect(subscribe).toHaveBeenCalledWith(expect.objectContaining({ isResume: true }));
|
||||
});
|
||||
|
||||
it('re-attaches on foreground when the status proves an open connection terminal', async () => {
|
||||
const { sub, page, fetchStatus, subscribe } = setup();
|
||||
sub.claim();
|
||||
sub.connection = { closed: false };
|
||||
sub.watchForeground();
|
||||
fetchStatus.mockResolvedValueOnce({ active: true, createdAt: 2000 });
|
||||
page.show();
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(subscribe).not.toHaveBeenCalled();
|
||||
|
||||
fetchStatus.mockResolvedValueOnce({ active: false, createdAt: 2000 });
|
||||
page.show();
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(fetchStatus).toHaveBeenLastCalledWith('c-1');
|
||||
expect(subscribe).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('keeps one foreground listener per pane', () => {
|
||||
const { run, sub, page } = setup();
|
||||
sub.watchForeground();
|
||||
const second = new ResumableSubscription(run, sub.host, {
|
||||
streamId: 'c-1',
|
||||
submission,
|
||||
isResume: true,
|
||||
generationProtocolVersion: 2,
|
||||
});
|
||||
second.watchForeground();
|
||||
expect(page.listeners.size).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('ResumableRun', () => {
|
||||
it('drops the cleared edit prefix only when a new generation binds', () => {
|
||||
const run = new ResumableRun(createScheduler().scheduler);
|
||||
run.bindPrefixState('gen-1');
|
||||
run.editPrefixCleared = true;
|
||||
run.bindPrefixState('gen-1');
|
||||
expect(run.editPrefixCleared).toBe(true);
|
||||
run.bindPrefixState('gen-2');
|
||||
expect(run.editPrefixCleared).toBe(false);
|
||||
});
|
||||
|
||||
it('clears a pending reattach', () => {
|
||||
const clock = createScheduler();
|
||||
const run = new ResumableRun(clock.scheduler);
|
||||
run.reconnectTimeout = clock.scheduler.setTimeout(() => undefined, 1000);
|
||||
run.clearReconnectTimeout();
|
||||
expect(run.reconnectTimeout).toBeNull();
|
||||
expect(clock.pending()).toBe(0);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
export const GENERATION_PROTOCOL_VERSION = 2 as const;
|
||||
export const GENERATION_PROTOCOL_HEADER = 'X-LibreChat-Generation-Protocol';
|
||||
|
||||
export type GenerationProtocolVersion = 1 | typeof GENERATION_PROTOCOL_VERSION;
|
||||
|
||||
|
|
@ -22,3 +23,7 @@ export const getGenerationProtocolVersion = (value: unknown): GenerationProtocol
|
|||
|
||||
export const supportsGenerationProtocolV2 = (value: unknown): boolean =>
|
||||
getGenerationProtocolVersion(value) === GENERATION_PROTOCOL_VERSION;
|
||||
|
||||
export const generationProtocolHeaders = (): Record<string, string> => ({
|
||||
[GENERATION_PROTOCOL_HEADER]: String(GENERATION_PROTOCOL_VERSION),
|
||||
});
|
||||
|
|
|
|||
55
packages/chat/src/core/resumable/host.ts
Normal file
55
packages/chat/src/core/resumable/host.ts
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
import type { Agents, TPendingSteer } from 'librechat-data-provider';
|
||||
|
||||
/** Where a resumable run reports what it is doing. The app passes its own logger. */
|
||||
export interface ResumableLogger {
|
||||
log: (...args: unknown[]) => void;
|
||||
warn: (...args: unknown[]) => void;
|
||||
error: (...args: unknown[]) => void;
|
||||
}
|
||||
|
||||
/** Whether the page is in the foreground, and a way to hear when that changes. */
|
||||
export interface PageVisibility {
|
||||
isVisible: () => boolean;
|
||||
/** Calls `listener` on every visibility change; returns the unsubscribe. */
|
||||
subscribe: (listener: () => void) => () => void;
|
||||
}
|
||||
|
||||
/** The document's own visibility. */
|
||||
export const documentVisibility: PageVisibility = {
|
||||
isVisible: () => document.visibilityState === 'visible',
|
||||
subscribe: (listener) => {
|
||||
document.addEventListener('visibilitychange', listener);
|
||||
return () => document.removeEventListener('visibilitychange', listener);
|
||||
},
|
||||
};
|
||||
|
||||
/** The status route's answer for a conversation's generation. */
|
||||
export interface StreamStatusResponse {
|
||||
/** Exact protocol selected by the job. Missing/1 is always legacy. */
|
||||
generationProtocolVersion?: number;
|
||||
active: boolean;
|
||||
/** Client-local cache marker used to force useResumeOnLoad to replace a
|
||||
* stale generation attachment with the newer epoch occupying this stream. */
|
||||
generationHandoff?: boolean;
|
||||
streamId?: string;
|
||||
status?: 'running' | 'complete' | 'error' | 'aborted' | 'requires_action';
|
||||
aggregatedContent?: Array<{ type: string; text?: string }>;
|
||||
createdAt?: number;
|
||||
/** Generation age computed on the server's clock, so a reattaching client
|
||||
* can rebuild a clock-local elapsed baseline free of cross-machine skew. */
|
||||
elapsedMs?: number;
|
||||
resumeState?: Agents.ResumeState;
|
||||
/** The run's temporary state as the server recorded it at admission, so a rebuilt
|
||||
* submission keeps a hidden chat out of history and titling after a reload. */
|
||||
isTemporary?: boolean;
|
||||
/** Live pending approval when `status === 'requires_action'`; mirrors
|
||||
* `resumeState.pendingAction`, surfaced top-level for the resume-on-load path. */
|
||||
pendingAction?: Agents.PendingAction;
|
||||
/** Acknowledged steers a terminal drain parked because no subscriber was
|
||||
* live for the final/abort event. Reads are replayable until an exact
|
||||
* recovery turn persists or the user discards the item. */
|
||||
unrecoveredSteers?: TPendingSteer[];
|
||||
}
|
||||
|
||||
/** Reads a conversation's generation status. */
|
||||
export type FetchStreamStatus = (conversationId: string) => Promise<StreamStatusResponse>;
|
||||
|
|
@ -3,3 +3,6 @@ export * from './errors';
|
|||
export * from './events';
|
||||
export * from './resume';
|
||||
export * from './messages';
|
||||
export * from './run';
|
||||
export * from './host';
|
||||
export * from './subscription';
|
||||
|
|
|
|||
132
packages/chat/src/core/resumable/run.ts
Normal file
132
packages/chat/src/core/resumable/run.ts
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
import type { TSubmission } from 'librechat-data-provider';
|
||||
|
||||
/** A pending timer, as the scheduler that set it returns it. */
|
||||
export type TimerHandle = ReturnType<typeof setTimeout>;
|
||||
|
||||
/**
|
||||
* The clock and frame source a resumable run waits on. The browser supplies its globals; a test
|
||||
* or another runtime supplies its own.
|
||||
*/
|
||||
export interface ResumableScheduler {
|
||||
setTimeout: (callback: () => void, delay: number) => TimerHandle;
|
||||
clearTimeout: (handle: TimerHandle) => void;
|
||||
requestFrame: (callback: () => void) => number;
|
||||
cancelFrame: (handle: number) => void;
|
||||
now: () => number;
|
||||
}
|
||||
|
||||
/** The browser's timers and animation frames. */
|
||||
export const browserScheduler: ResumableScheduler = {
|
||||
setTimeout: (callback, delay) => setTimeout(callback, delay),
|
||||
clearTimeout: (handle) => clearTimeout(handle),
|
||||
requestFrame: (callback) => requestAnimationFrame(callback),
|
||||
cancelFrame: (handle) => cancelAnimationFrame(handle),
|
||||
now: () => Date.now(),
|
||||
};
|
||||
|
||||
/**
|
||||
* What one chat pane knows about the generation it is attached to, across every submission and
|
||||
* reconnect: the live connection and its retry ladder, the submission it serves, the stream ids
|
||||
* it created optimistically, and the frame retries still waiting on a response placeholder.
|
||||
* The pane owns one for its lifetime; every subscription reads and fences against it.
|
||||
*/
|
||||
export class ResumableRun {
|
||||
/** Closes the live connection; replaced by every subscription. */
|
||||
stream: AbortController | null = null;
|
||||
/** Removes the foreground re-attach listener owned by the newest subscription; exactly one
|
||||
* is registered at a time. */
|
||||
stopForegroundReattach: (() => void) | null = null;
|
||||
reconnectAttempt = 0;
|
||||
reconnectTimeout: TimerHandle | null = null;
|
||||
/** The submission this pane serves; a subscription whose submission is no longer this one is
|
||||
* stale. */
|
||||
submission: TSubmission | null = null;
|
||||
/** Suppresses the normal close/cleanup idle transition while a stale generation is being
|
||||
* atomically handed off to its live replacement. */
|
||||
replacementHandoff = false;
|
||||
readonly optimisticStreamIds: Set<string> = new Set();
|
||||
readonly createdStreamIds: Set<string> = new Set();
|
||||
/** Pending action whose tool-call content part hasn't rendered yet, retried on the next frame
|
||||
* so a fast pause-before-render race still attaches. */
|
||||
pendingActionRetry: number | null = null;
|
||||
/** EVERY steer retry frame, not only the newest one. Several steers can be applied before
|
||||
* their shared response placeholder renders; retaining one handle let the older chains
|
||||
* outlive navigation/terminal cleanup and mutate a later generation that reused the same
|
||||
* response id. */
|
||||
readonly steerRetryFrames: Set<number> = new Set();
|
||||
/** EVERY outstanding label-retry frame, not a single handle: labels fire two events per slot
|
||||
* (reservation, then fill), so concurrent retry chains are the norm: a single handle would
|
||||
* let cleanup cancel only the newest chain while the others kept running for up to 120
|
||||
* frames and could apply a stale label to a replacement generation that reuses the same
|
||||
* response id (edits do). */
|
||||
readonly activityLabelRetryFrames: Set<number> = new Set();
|
||||
/**
|
||||
* Set once a SYNC has replaced the response with the server's completion-local snapshot,
|
||||
* which discards the prefix an edited resubmission retained. From that point incoming indices
|
||||
* are absolute and `editPrefixLength` must no longer be applied, by run steps or labels.
|
||||
*/
|
||||
editPrefixCleared = false;
|
||||
editPrefixFirstPartFolded = false;
|
||||
/** Generation the cleared-prefix state above belongs to, so it is dropped when a new
|
||||
* generation starts rather than when a subscribe happens to be live. */
|
||||
prefixStateGenerationId: string | null = null;
|
||||
/** The generation epoch the pane's stream is attached to, kept so a run left mid-stream can
|
||||
* carry it. A server that reports no epoch leaves it unset. */
|
||||
subscribedEpoch: number | null = null;
|
||||
/** Whether the current submission's run end already reached the queue drain. A run that
|
||||
* ended attached must not also be resolved as detached when the user later leaves. */
|
||||
runEndPublished = false;
|
||||
/** Whether the server named the current submission's user message (`created`, or a resume's
|
||||
* sync). Only a run whose identity it confirmed is remembered when the user leaves. */
|
||||
identityConfirmed = false;
|
||||
|
||||
constructor(readonly scheduler: ResumableScheduler = browserScheduler) {}
|
||||
|
||||
/** Drops the backoff still waiting to re-attach, if any. */
|
||||
clearReconnectTimeout(): void {
|
||||
if (this.reconnectTimeout) {
|
||||
this.scheduler.clearTimeout(this.reconnectTimeout);
|
||||
this.reconnectTimeout = null;
|
||||
}
|
||||
}
|
||||
|
||||
cancelSteerRetryFrames(): void {
|
||||
for (const frameId of this.steerRetryFrames) {
|
||||
this.scheduler.cancelFrame(frameId);
|
||||
}
|
||||
this.steerRetryFrames.clear();
|
||||
}
|
||||
|
||||
/** Cancels every frame retry still waiting on a response placeholder. */
|
||||
cancelRetryFrames(): void {
|
||||
if (this.pendingActionRetry != null) {
|
||||
this.scheduler.cancelFrame(this.pendingActionRetry);
|
||||
this.pendingActionRetry = null;
|
||||
}
|
||||
for (const frameId of this.activityLabelRetryFrames) {
|
||||
this.scheduler.cancelFrame(frameId);
|
||||
}
|
||||
this.activityLabelRetryFrames.clear();
|
||||
this.cancelSteerRetryFrames();
|
||||
}
|
||||
|
||||
/**
|
||||
* A NEW generation starts with its retained prefix intact, so the cleared-prefix state from a
|
||||
* previous one must not carry over. Keyed on the submission's `clientRequestId`, which is new
|
||||
* per edit attempt and stable across reconnects.
|
||||
*/
|
||||
bindPrefixState(generationId: string): void {
|
||||
if (this.prefixStateGenerationId === generationId) {
|
||||
return;
|
||||
}
|
||||
this.prefixStateGenerationId = generationId;
|
||||
this.editPrefixCleared = false;
|
||||
this.editPrefixFirstPartFolded = false;
|
||||
}
|
||||
|
||||
/** Forgets a stream once its run has ended. */
|
||||
releaseStream(streamId: string): void {
|
||||
this.optimisticStreamIds.delete(streamId);
|
||||
this.createdStreamIds.delete(streamId);
|
||||
}
|
||||
}
|
||||
296
packages/chat/src/core/resumable/subscription.ts
Normal file
296
packages/chat/src/core/resumable/subscription.ts
Normal file
|
|
@ -0,0 +1,296 @@
|
|||
import { apiBaseUrl } from 'librechat-data-provider';
|
||||
import type {
|
||||
TMessage,
|
||||
ChatEvent,
|
||||
TSubmission,
|
||||
ChatTransport,
|
||||
ChatStreamConnection,
|
||||
} from 'librechat-data-provider';
|
||||
import type { ResumableLogger, PageVisibility, FetchStreamStatus } from './host';
|
||||
import type { GenerationProtocolVersion } from '../protocol';
|
||||
import type { StepEvent } from '../reducer';
|
||||
import type { ResumableRun } from './run';
|
||||
import { GENERATION_PROTOCOL_VERSION, generationProtocolHeaders } from '../protocol';
|
||||
|
||||
/** The generation one subscription attaches to, and how. */
|
||||
export interface SubscriptionTarget {
|
||||
streamId: string;
|
||||
submission: TSubmission;
|
||||
/** Asks the server to replay the run's state (`sync`) before live events. */
|
||||
isResume: boolean;
|
||||
generationCreatedAt?: number;
|
||||
generationProtocolVersion: GenerationProtocolVersion;
|
||||
/** The pane's lifecycle: once aborted, nothing this subscription does lands. */
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
/** How a subscription reaches the stream and decides when to re-attach. */
|
||||
export interface SubscriptionHost {
|
||||
logger: ResumableLogger;
|
||||
visibility: PageVisibility;
|
||||
fetchStatus: FetchStreamStatus;
|
||||
/** Attaches a fresh subscription to `target`; how a subscription re-attaches itself. */
|
||||
subscribe: (target: SubscriptionTarget) => void;
|
||||
}
|
||||
|
||||
/** What a subscription does with the events its connection reports. */
|
||||
export interface SubscriptionHandlers {
|
||||
onOpen: () => void;
|
||||
onFrame: (event: ChatEvent) => void;
|
||||
onFailure: (event: Extract<ChatEvent, { type: 'error' }>) => void;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* One attachment of a pane to a running generation: the connection, the submission as the
|
||||
* stream has revised it so far, and the fences that make every late callback a no-op once a
|
||||
* newer attachment, submission or pane lifecycle has taken over.
|
||||
*/
|
||||
export class ResumableSubscription {
|
||||
readonly streamId: string;
|
||||
readonly isResume: boolean;
|
||||
readonly generationCreatedAt?: number;
|
||||
readonly generationProtocolVersion: GenerationProtocolVersion;
|
||||
readonly signal?: AbortSignal;
|
||||
/** Whether the pane created this stream's conversation optimistically, from `/c/new`. */
|
||||
readonly startedAsNewConversation: boolean;
|
||||
/** Closing is per connection: the transport tells this subscription's own close (`abort`)
|
||||
* apart from a cancel by the user agent (`error` with status 0). */
|
||||
readonly controller: AbortController = new AbortController();
|
||||
submission: TSubmission;
|
||||
userMessage: TMessage;
|
||||
textIndex: number | null = null;
|
||||
finalReceived = false;
|
||||
/**
|
||||
* This subscription must never be revived. Set by the terminal recoveries that do not ride a
|
||||
* FINAL or served-error frame (the 404 and retry-ceiling reconciles both leave the attachment
|
||||
* pointing at a stream the server no longer has) and by the dev-only navigation simulator.
|
||||
* `finalReceived` covers the frame-carried terminals.
|
||||
*/
|
||||
retired = false;
|
||||
/** Step events that arrived before `created` named the turn, replayed once it does. */
|
||||
readonly preCreatedStepEvents: StepEvent[] = [];
|
||||
connection: ChatStreamConnection | null = null;
|
||||
private foregroundStatusCheckInFlight = false;
|
||||
|
||||
constructor(
|
||||
readonly run: ResumableRun,
|
||||
readonly host: SubscriptionHost,
|
||||
target: SubscriptionTarget,
|
||||
) {
|
||||
this.streamId = target.streamId;
|
||||
this.isResume = target.isResume;
|
||||
this.generationCreatedAt = target.generationCreatedAt;
|
||||
this.generationProtocolVersion = target.generationProtocolVersion;
|
||||
this.signal = target.signal;
|
||||
this.submission = target.submission;
|
||||
this.userMessage = target.submission.userMessage;
|
||||
this.startedAsNewConversation = run.optimisticStreamIds.has(target.streamId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Every pane lifecycle owns one AbortSignal, while every reconnect owns one connection. Both
|
||||
* identities matter: cleanup can close an attachment after its async listener has already
|
||||
* crossed an `await`, and a reconnect can supersede an older attachment without changing the
|
||||
* submission object. The submission check is the final fence against a later conversation
|
||||
* installing its own submission in this pane.
|
||||
*/
|
||||
isCurrent(): boolean {
|
||||
return (
|
||||
this.signal?.aborted !== true &&
|
||||
this.run.stream === this.controller &&
|
||||
this.run.submission === this.submission
|
||||
);
|
||||
}
|
||||
|
||||
/** The submission with the user message as the stream has named it so far. */
|
||||
liveSubmission(): TSubmission {
|
||||
return { ...this.submission, userMessage: this.userMessage };
|
||||
}
|
||||
|
||||
/** Adopts a revised submission as the one this subscription and its pane serve. */
|
||||
replaceSubmission(next: TSubmission): void {
|
||||
this.submission = next;
|
||||
this.run.submission = next;
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.controller.abort();
|
||||
}
|
||||
|
||||
/** The conversation this subscription's run belongs to, as far as it is known. */
|
||||
conversationId(): string {
|
||||
return this.submission.conversation?.conversationId ?? this.streamId;
|
||||
}
|
||||
|
||||
/** Re-attaches now, with the run's state replayed. */
|
||||
reattach(): void {
|
||||
this.host.subscribe(this.resumeTarget(this.submission));
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-attaches after `delay` with the run's state replayed, unless a newer attachment or
|
||||
* submission takes over first. A different stream or epoch is passed when the status read
|
||||
* that scheduled it named one.
|
||||
*/
|
||||
scheduleReattach(
|
||||
delay: number,
|
||||
streamId: string = this.streamId,
|
||||
generationCreatedAt: number | undefined = this.generationCreatedAt,
|
||||
): void {
|
||||
this.run.reconnectTimeout = this.run.scheduler.setTimeout(() => {
|
||||
if (this.isCurrent() && this.run.submission) {
|
||||
this.host.subscribe({
|
||||
...this.resumeTarget(this.run.submission),
|
||||
streamId,
|
||||
generationCreatedAt,
|
||||
});
|
||||
}
|
||||
}, delay);
|
||||
}
|
||||
|
||||
private resumeTarget(submission: TSubmission): SubscriptionTarget {
|
||||
return {
|
||||
streamId: this.streamId,
|
||||
submission,
|
||||
isResume: true,
|
||||
generationCreatedAt: this.generationCreatedAt,
|
||||
generationProtocolVersion: this.generationProtocolVersion,
|
||||
signal: this.signal,
|
||||
};
|
||||
}
|
||||
|
||||
/** The stream route, with the resume cursor and generation fence in its query. */
|
||||
url(): string {
|
||||
const baseUrl = `${apiBaseUrl()}/api/agents/chat/stream/${encodeURIComponent(this.streamId)}`;
|
||||
const query = new URLSearchParams();
|
||||
if (this.isResume) {
|
||||
query.set('resume', 'true');
|
||||
}
|
||||
if (this.generationCreatedAt != null) {
|
||||
query.set('generationCreatedAt', String(this.generationCreatedAt));
|
||||
}
|
||||
query.set('generationProtocolVersion', String(GENERATION_PROTOCOL_VERSION));
|
||||
const queryString = query.toString();
|
||||
return queryString ? `${baseUrl}?${queryString}` : baseUrl;
|
||||
}
|
||||
|
||||
/** Makes this the pane's live attachment, superseding any earlier one. */
|
||||
claim(): void {
|
||||
this.run.stream = this.controller;
|
||||
}
|
||||
|
||||
/** Opens the connection and routes its events to `handlers`. */
|
||||
connect(transport: ChatTransport, handlers: SubscriptionHandlers): void {
|
||||
this.connection = transport.reconnectToStream(
|
||||
{ url: this.url(), headers: generationProtocolHeaders() },
|
||||
{
|
||||
signal: this.controller.signal,
|
||||
onEvent: (event) => {
|
||||
switch (event.type) {
|
||||
case 'open':
|
||||
handlers.onOpen();
|
||||
return;
|
||||
case 'error':
|
||||
handlers.onFailure(event);
|
||||
return;
|
||||
case 'abort':
|
||||
handlers.onClose();
|
||||
return;
|
||||
default:
|
||||
handlers.onFrame(event);
|
||||
}
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private reattachOnForeground(): void {
|
||||
this.host.logger.log('ResumableSSE', 'Re-attaching stream on foreground');
|
||||
/** Any backoff still pending targets this same attachment, so returning to the app
|
||||
* supersedes it rather than waiting the delay out; its callback finds a superseded
|
||||
* subscription and does nothing. */
|
||||
this.run.clearReconnectTimeout();
|
||||
this.run.reconnectAttempt = Math.max(this.run.reconnectAttempt, 1);
|
||||
this.close();
|
||||
this.reattach();
|
||||
}
|
||||
|
||||
private isSettled(): boolean {
|
||||
return this.finalReceived || this.retired || this.run.replacementHandoff;
|
||||
}
|
||||
|
||||
/**
|
||||
* A suspended page can lose its stream without the transport ever reporting it. When a
|
||||
* mobile browser freezes the tab, an intermediary closes the response body underneath it,
|
||||
* and XHR surfaces that as an ordinary load: sse.js dispatches nothing for a body that simply
|
||||
* ends, so neither the error nor the abort recovery ever runs. Nothing else re-reads the
|
||||
* conversation while the pane stays mounted, which is why the response the run finished
|
||||
* writing only appears after a reload.
|
||||
*
|
||||
* Some mobile WebViews leave the XHR marked OPEN even after the suspended request stopped
|
||||
* delivering events. In that case the durable run status is the only authority that
|
||||
* distinguishes a healthy live attachment from a terminal one holding stale partial content.
|
||||
* A terminal status forces the same resume path as a closed transport; it returns the
|
||||
* synthesized terminal frame or 404 that reconciles persisted messages.
|
||||
*/
|
||||
private handleForegroundReattach(): void {
|
||||
const { visibility, fetchStatus, logger } = this.host;
|
||||
if (!visibility.isVisible() || this.isSettled() || !this.isCurrent() || !this.run.submission) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.connection?.closed === true) {
|
||||
this.reattachOnForeground();
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.foregroundStatusCheckInFlight) {
|
||||
return;
|
||||
}
|
||||
this.foregroundStatusCheckInFlight = true;
|
||||
const foregroundConvoId = this.conversationId();
|
||||
const { generationCreatedAt } = this;
|
||||
void fetchStatus(foregroundConvoId)
|
||||
.then((status) => {
|
||||
if (
|
||||
status.active !== false ||
|
||||
(status.createdAt != null &&
|
||||
generationCreatedAt != null &&
|
||||
status.createdAt !== generationCreatedAt) ||
|
||||
!this.isCurrent() ||
|
||||
this.isSettled() ||
|
||||
!visibility.isVisible()
|
||||
) {
|
||||
return;
|
||||
}
|
||||
logger.log(
|
||||
'ResumableSSE',
|
||||
'Apparently open stream is terminal - reconciling on foreground',
|
||||
{ conversationId: foregroundConvoId, generationCreatedAt },
|
||||
);
|
||||
this.reattachOnForeground();
|
||||
})
|
||||
.catch((error) => {
|
||||
if (!this.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
logger.warn('ResumableSSE', 'Could not verify stream on foreground', {
|
||||
conversationId: foregroundConvoId,
|
||||
generationCreatedAt,
|
||||
error,
|
||||
});
|
||||
})
|
||||
.finally(() => {
|
||||
this.foregroundStatusCheckInFlight = false;
|
||||
});
|
||||
}
|
||||
|
||||
/** Replaces the pane's foreground listener with this subscription's. */
|
||||
watchForeground(): void {
|
||||
this.run.stopForegroundReattach?.();
|
||||
this.run.stopForegroundReattach = this.host.visibility.subscribe(() =>
|
||||
this.handleForegroundReattach(),
|
||||
);
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue