🚀 refactor: Move the Resumable Generation Start Into the Chat Core (#17131)

* refactor: Move the Resumable Generation Start Into the Chat Core

* fix: Call a Host Transport's Start on the Transport Itself

Forwarding transport.start as a bare function rebound this to the resumable host, breaking any host transport whose start is an instance method.
This commit is contained in:
Marco Beretta 2026-10-20 22:24:15 +02:00 • committed by GitHub
parent 3f7fc0b5fb
commit 2f724fccff
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 549 additions and 388 deletions

View file

@ -1,3 +1,4 @@
import { createElement } from 'react';
import { getDefaultStore } from 'jotai';
import { QueryClient } from '@tanstack/react-query';
import { renderHook, act, waitFor } from '@testing-library/react';
@ -12,6 +13,7 @@ import {
} from 'librechat-data-provider';
import type { TMessage, TSubmission } from 'librechat-data-provider';
import type { Query, QueryKey } from '@tanstack/react-query';
import type { ReactNode } from 'react';
import {
activeUsageResponseIdFamily,
liveTokensFamily,
@ -315,6 +317,8 @@ import useResumableSSE, {
} from '~/hooks/SSE/useResumableSSE';
import useSSE from '~/hooks/SSE/useSSE';
import type { PendingSteer } from '@librechat/chat';
import type { Transport } from '~/hooks/Chat/contract';
import { ChatTransportContext, defaultChatTransport } from '~/Providers/ChatTransportContext';
import {
recoveryDispositionsFamily,
queuedMessagesByConvoId,
@ -627,6 +631,25 @@ describe('useResumableSSE', () => {
});
});
it('calls a host transport start on the transport itself', async () => {
const receivers: unknown[] = [];
const hostTransport: Transport = {
...defaultChatTransport,
start(this: Transport, startRequest, options) {
receivers.push(this);
return defaultChatTransport.start(startRequest, options);
},
};
const wrapper = ({ children }: { children: ReactNode }) =>
createElement(ChatTransportContext.Provider, { value: hostTransport }, children);
const { unmount } = renderHook(() => useResumableSSE(buildSubmission(), buildChatHelpers()), {
wrapper,
});
await waitFor(() => expect(receivers).toHaveLength(1));
expect(receivers[0]).toBe(hostTransport);
unmount();
});
it.each([useSSE, useResumableSSE])(
'%p retains authoritative legacy text response ownership for subsequent usage events',
async (useTransport) => {

View file

@ -3,6 +3,7 @@ import { v4 } from 'uuid';
import { useSetRecoilState } from 'recoil';
import { useStore, useSetAtom } from 'jotai';
import { useQueryClient } from '@tanstack/react-query';
import { Constants, QueryKeys, dataService } from 'librechat-data-provider';
import { ABORT_SWEEP_STATUSES, selectLocalSteersForQueue } from '@librechat/chat';
import {
resolveDetachedRunEnd,
@ -10,13 +11,6 @@ import {
canRestoreRecovery,
blockRecovery,
} from '@librechat/chat';
import {
Constants,
QueryKeys,
dataService,
createPayload,
removeNullishValues,
} from 'librechat-data-provider';
import {
submissionByIndex,
liveAppliedSteerIdsAtom,
@ -34,44 +28,27 @@ import {
} from '@librechat/chat/react';
import {
toStartGenerationError,
waitForRetryDelay,
getRecoverySteerId,
getRetryAfterDelay,
isServerNotReadyError,
getOptimisticMessages,
isResumedStartResponse,
isSettledStartResponse,
isRetryableNetworkError,
isReplacedStartResponse,
isInitialNewConversation,
getStreamStartFailureText,
keepLocalCodeApprovalMode,
hydrateSubmissionMessages,
getStartGenerationStreamId,
buildOptimisticConversation,
getGenerationProtocolVersion,
GENERATION_PROTOCOL_VERSION,
getStartGenerationCreatedAt,
isUnnegotiatedV2StartControl,
supportsGenerationProtocolV2,
START_GENERATION_NETWORK_RETRIES,
getStartGenerationConversationId,
getPredecessorMismatchStartResponse,
START_GENERATION_READINESS_TIMEOUT_MS,
ResumableRun,
documentVisibility,
attachResumableStream,
startGeneration,
} from '@librechat/chat';
import type {
ResumableHost,
ResumableCache,
SubscriptionTarget,
StartGenerationResult,
GenerationProtocolVersion,
} from '@librechat/chat';
import type {
TMessage,
TPayload,
TSubmission,
TConversation,
TPendingSteer,
@ -435,117 +412,137 @@ export default function useResumableSSE(
* Follows same auth pattern as useSSE
* @param isResume - If true, adds ?resume=true to trigger sync event from server
*/
const subscribeToStream = useCallback(
(target: SubscriptionTarget) => {
const host: ResumableHost = {
logger,
visibility: documentVisibility,
fetchStatus: fetchStreamStatus,
subscribe: (next) => subscribeToStream(next),
cache: resumableCache,
pane: {
setIsSubmitting,
setShowStopButton,
setSubmission,
setStreamId,
setGenerationEpoch: updateActiveGenerationCreatedAt,
setPendingAction: (conversationId, action) =>
jotaiStore.set(pendingApprovalActionFamily(conversationId), action),
clearComposerDrafts: (conversationId, options) =>
clearComposerDrafts(runIndex, conversationId, options),
publishRunEnd: setRunEnd,
setAbortScroll,
renewActiveRun: () => setActiveRunId(v4()),
runEndTarget: resolveRunEndTarget,
clearDrainAfterAbort,
refreshBalance: () => {
(startupConfig?.balance?.enabled ?? false) && balanceQuery.refetch();
},
updateConversation: isAddedRequest ? undefined : setConversation,
resolveDetachedEnd: (conversationId, messages) => {
const detachedRun = jotaiStore.get(detachedRunByConvoId(conversationId));
const resolvable =
detachedRun != null &&
!isDetachedRunManual(jotaiStore.get, conversationId, detachedRun);
return {
resolvable,
end:
resolvable && messages != null
? resolveDetachedRunEnd(conversationId, detachedRun, messages)
: null,
const buildHost = useCallback(
(): ResumableHost => ({
logger,
visibility: documentVisibility,
fetchStatus: fetchStreamStatus,
subscribe: (next) => attachResumableStream(run, buildHost(), next),
cache: resumableCache,
pane: {
setIsSubmitting,
setShowStopButton,
setSubmission,
setStreamId,
setGenerationEpoch: updateActiveGenerationCreatedAt,
setPendingAction: (conversationId, action) =>
jotaiStore.set(pendingApprovalActionFamily(conversationId), action),
clearComposerDrafts: (conversationId, options) =>
clearComposerDrafts(runIndex, conversationId, options),
publishRunEnd: setRunEnd,
setAbortScroll,
renewActiveRun: () => setActiveRunId(v4()),
runEndTarget: resolveRunEndTarget,
clearDrainAfterAbort,
refreshBalance: () => {
(startupConfig?.balance?.enabled ?? false) && balanceQuery.refetch();
},
updateConversation: isAddedRequest ? undefined : setConversation,
resolveDetachedEnd: (conversationId, messages) => {
const detachedRun = jotaiStore.get(detachedRunByConvoId(conversationId));
const resolvable =
detachedRun != null &&
!isDetachedRunManual(jotaiStore.get, conversationId, detachedRun);
return {
resolvable,
end:
resolvable && messages != null
? resolveDetachedRunEnd(conversationId, detachedRun, messages)
: null,
};
},
},
steers: {
resolve: resolveSteerChip,
isRecoveryHeld: (conversationId, recoverySteerId) =>
jotaiStore.get(recoveryDispositionsFamily(conversationId))[recoverySteerId] != null,
blockRecovery: (conversationId, recoverySteerId) =>
jotaiStore.set(recoveryDispositionsFamily(conversationId), (previous) =>
blockRecovery(previous, recoverySteerId),
),
restoreQueued: restoreQueuedSubmission,
update: updateSteerChips,
markLiveApplied: (steerId) =>
setLiveAppliedSteerIds((prev) => appendAppliedSteerIds(prev, [steerId])),
seed: seedSteerChips,
appliedIds: collectAppliedSteerIds,
settleApplied: settleAppliedSteerParts,
convertToQueued: convertSteersToQueued,
convertLocalToQueued: convertLocalSteersToQueued,
},
events: {
stepHandler,
showStartFailure: (failedSubmission, errorData, reason) => {
let text: string | undefined;
if (reason === 'recovery_held') {
text = localize('com_ui_steer_recovery_held');
} else if (reopenTermsOnRefusal(queryClient, errorData)) {
text = localize('com_error_terms_not_accepted');
}
errorHandler({
data: getStreamStartFailureData(errorData, text),
submission: failedSubmission as EventSubmission,
});
},
finalHandler: (data, finalSubmission) =>
finalHandler(data as TFinalResData, finalSubmission),
createdHandler,
attachmentHandler,
titleHandler,
contentHandler,
messageHandler,
prunePtcTraces,
resetContentHandler,
syncStepMessage,
clearStepMaps,
flushPendingDeltas,
resetLive,
showError: (text, errorSubmission) =>
errorHandler({
data: { text } as unknown as Parameters<typeof errorHandler>[0]['data'],
submission: errorSubmission as EventSubmission,
}),
},
messages: { get: getMessages, set: setMessages },
usage: {
bindResponse,
contextHandler,
usageHandler,
tapStream,
tapContent,
finalizeUsage,
backfillUsage,
seedLiveFromContent: (content, usageSubmission) =>
seedLive(countTrailingOutputChars(content), usageSubmission),
},
stream: transport.stream({ token }),
start: (request, options) => transport.start(request, options),
exposeDebugControls: import.meta.env.DEV
? ({ killNetwork, closeClean }) => {
// Debug hooks for testing reconnection vs clean close behavior (dev only)
const debugWindow = window as Window & {
__killNetwork?: () => void;
__closeClean?: () => void;
};
},
},
steers: {
resolve: resolveSteerChip,
update: updateSteerChips,
markLiveApplied: (steerId) =>
setLiveAppliedSteerIds((prev) => appendAppliedSteerIds(prev, [steerId])),
seed: seedSteerChips,
appliedIds: collectAppliedSteerIds,
settleApplied: settleAppliedSteerParts,
convertToQueued: convertSteersToQueued,
convertLocalToQueued: convertLocalSteersToQueued,
},
events: {
stepHandler,
finalHandler: (data, finalSubmission) =>
finalHandler(data as TFinalResData, finalSubmission),
createdHandler,
attachmentHandler,
titleHandler,
contentHandler,
messageHandler,
prunePtcTraces,
resetContentHandler,
syncStepMessage,
clearStepMaps,
flushPendingDeltas,
resetLive,
showError: (text, errorSubmission) =>
errorHandler({
data: { text } as unknown as Parameters<typeof errorHandler>[0]['data'],
submission: errorSubmission as EventSubmission,
}),
},
messages: { get: getMessages, set: setMessages },
usage: {
bindResponse,
contextHandler,
usageHandler,
tapStream,
tapContent,
finalizeUsage,
backfillUsage,
seedLiveFromContent: (content, usageSubmission) =>
seedLive(countTrailingOutputChars(content), usageSubmission),
},
stream: transport.stream({ token }),
exposeDebugControls: import.meta.env.DEV
? ({ killNetwork, closeClean }) => {
// Debug hooks for testing reconnection vs clean close behavior (dev only)
const debugWindow = window as Window & {
__killNetwork?: () => void;
__closeClean?: () => void;
};
debugWindow.__killNetwork = killNetwork;
debugWindow.__closeClean = closeClean;
}
: undefined,
parts: {
applySteer: applySteerPart,
findSteerTarget: findSteerMessageIndex,
applyActivityLabel: applyActivityLabelPart,
findActivityLabelTarget: findActivityLabelMessageIndex,
offsetActivityPhaseBoundary,
applyReasoningLabel,
findReasoningLabelTarget: findReasoningLabelMessageIndex,
},
};
attachResumableStream(run, host, target);
},
debugWindow.__killNetwork = killNetwork;
debugWindow.__closeClean = closeClean;
}
: undefined,
parts: {
applySteer: applySteerPart,
findSteerTarget: findSteerMessageIndex,
applyActivityLabel: applyActivityLabelPart,
findActivityLabelTarget: findActivityLabelMessageIndex,
offsetActivityPhaseBoundary,
applyReasoningLabel,
findReasoningLabelTarget: findReasoningLabelMessageIndex,
},
}),
[
run,
localize,
queryClient,
restoreQueuedSubmission,
resumableCache,
isAddedRequest,
setConversation,
@ -597,260 +594,9 @@ export default function useResumableSSE(
],
);
/**
* Start generation (POST request that returns streamId)
* Posts through the host transport, which owns auth refresh.
* Retries transient network failures and startup readiness responses.
* Readiness retries honor Retry-After until cleanup or the readiness window expires.
*/
const startGeneration = useCallback(
async (
currentSubmission: TSubmission,
signal?: AbortSignal,
): Promise<StartGenerationResult | null> => {
const payloadData = createPayload(currentSubmission);
let { payload } = payloadData;
payload = removeNullishValues(payload) as TPayload;
clearStepMaps();
const url = payloadData.server;
let lastError: unknown = null;
let requestAttempts = 0;
let networkAttempts = 0;
let readinessAttempts = 0;
const readinessDeadline = run.scheduler.now() + START_GENERATION_READINESS_TIMEOUT_MS;
while (!signal?.aborted) {
const recoverySteerId = getRecoverySteerId(currentSubmission);
const conversationId = currentSubmission.conversation?.conversationId;
if (
recoverySteerId != null &&
conversationId &&
jotaiStore.get(recoveryDispositionsFamily(conversationId))[recoverySteerId] != null
) {
restoreQueuedSubmission(currentSubmission);
errorHandler({
data: getStreamStartFailureData(localize('com_ui_steer_recovery_held')),
submission: currentSubmission as EventSubmission,
});
setShowStopButton(false);
setIsSubmitting(false);
setSubmission(null);
return null;
}
requestAttempts += 1;
try {
const data = await transport.start({ server: url, payload }, { signal });
if (signal?.aborted) {
return null;
}
if (isSettledStartResponse(data)) {
const conversationId = getStartGenerationConversationId(data);
if (conversationId) {
logger.log('ResumableSSE', 'Generation already settled:', { conversationId });
return {
status: 'settled',
conversationId,
generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
};
}
lastError = { response: { data } };
break;
}
if (isReplacedStartResponse(data)) {
const streamId = getStartGenerationStreamId(data);
const conversationId = getStartGenerationConversationId(data);
const generationCreatedAt = getStartGenerationCreatedAt(data);
if (streamId && conversationId && generationCreatedAt != null) {
return {
status: 'replaced',
streamId,
conversationId,
generationCreatedAt,
generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
};
}
lastError = { response: { data } };
break;
}
if (isUnnegotiatedV2StartControl(data)) {
/** Never reinterpret a v2-only control word as a legacy stream
* response merely because it also carries a stream id. */
lastError = { response: { data } };
break;
}
const streamId = getStartGenerationStreamId(data);
if (streamId) {
const resumed = isResumedStartResponse(data);
const generationCreatedAt = getStartGenerationCreatedAt(data);
const generationProtocolVersion = getGenerationProtocolVersion(data);
logger.log('ResumableSSE', 'Generation started:', {
streamId,
resumed,
generationCreatedAt,
generationProtocolVersion,
});
return {
status: 'stream',
streamId,
resumed,
generationProtocolVersion,
...(generationCreatedAt != null && { generationCreatedAt }),
};
}
lastError = { response: { data } };
break;
} catch (error) {
if (signal?.aborted) {
return null;
}
const predecessorMismatch = getPredecessorMismatchStartResponse(error);
if (predecessorMismatch != null) {
/** Admission was atomically rejected before this queued turn
* created anything. Put the exact row back immediately and let
* the caller reconcile or hand off to the generation that won. */
restoreQueuedSubmission(currentSubmission);
logger.log('ResumableSSE', 'Queued generation lost predecessor fence', {
expectedPredecessorCreatedAt: currentSubmission.expectedPredecessorCreatedAt,
...predecessorMismatch,
});
return {
status: 'predecessor_mismatch',
...predecessorMismatch,
generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
};
}
lastError = error;
const isNetworkError = isRetryableNetworkError(error);
const isServerNotReady = isServerNotReadyError(error);
const remainingReadinessMs = readinessDeadline - run.scheduler.now();
const shouldRetryNetwork =
isNetworkError && networkAttempts < START_GENERATION_NETWORK_RETRIES - 1;
const shouldRetryServerNotReady = isServerNotReady && remainingReadinessMs > 0;
if (shouldRetryNetwork || shouldRetryServerNotReady) {
networkAttempts += isNetworkError ? 1 : 0;
readinessAttempts += isServerNotReady ? 1 : 0;
const fallbackDelay = Math.min(1000 * Math.pow(2, requestAttempts - 1), 8000);
const retryDelay = isServerNotReady
? Math.min(getRetryAfterDelay(error, fallbackDelay), remainingReadinessMs)
: fallbackDelay;
const reason = isServerNotReady ? 'Server not ready' : 'Network error';
const attempt = isServerNotReady ? readinessAttempts : networkAttempts;
const limit = isServerNotReady
? `${Math.ceil(START_GENERATION_READINESS_TIMEOUT_MS / 1000)}s readiness window`
: `${START_GENERATION_NETWORK_RETRIES}`;
logger.log(
'ResumableSSE',
`${reason} starting generation, retrying in ${retryDelay}ms (attempt ${attempt}/${limit})`,
);
const shouldContinue = await waitForRetryDelay(retryDelay, signal);
if (!shouldContinue) {
return null;
}
continue;
}
// Don't retry: either not a network error or max retries reached
break;
}
}
if (signal?.aborted) {
return null;
}
logger.error('ResumableSSE', 'Error starting generation:', lastError);
const startError = toStartGenerationError(lastError);
const errorData = startError?.response?.data;
const responseStatus = startError?.response?.status;
if (responseStatus != null && responseStatus >= 400 && responseStatus < 500) {
const recoverySteerId = getRecoverySteerId(currentSubmission);
const conversationId = currentSubmission.conversation?.conversationId;
const recoveryRejected =
errorData != null &&
typeof errorData === 'object' &&
'code' in errorData &&
(errorData.code === 'RECOVERY_PAYLOAD_MISMATCH' ||
errorData.code === 'INVALID_RECOVERY_REQUEST');
if (recoveryRejected && recoverySteerId != null && conversationId) {
jotaiStore.set(recoveryDispositionsFamily(conversationId), (previous) =>
blockRecovery(previous, recoverySteerId),
);
}
// The server rejected admission before exposing a generation. Restore
// the exact queue row/position; ambiguous transport/5xx outcomes must
// first reconcile durable state instead of risking a duplicate start.
restoreQueuedSubmission(currentSubmission);
} else {
const recoverySteerId = getRecoverySteerId(currentSubmission);
const recoveryConvoId = currentSubmission.conversation?.conversationId;
if (recoverySteerId != null && recoveryConvoId) {
try {
const status = await fetchStreamStatus(recoveryConvoId);
if (signal?.aborted) {
return null;
}
const releasedSource = status.unrecoveredSteers?.find(
(steer) => steer.steerId === recoverySteerId,
);
if (
status.active === false &&
supportsGenerationProtocolV2(status) &&
releasedSource != null
) {
convertSteersToQueued(recoveryConvoId, [releasedSource], {
generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
allowPreviouslyConvertedIds: [recoverySteerId],
});
}
} catch (error) {
if (signal?.aborted) {
return null;
}
logger.warn('ResumableSSE', 'Could not recover source after start failure', {
conversationId: recoveryConvoId,
error,
});
}
}
}
if (signal?.aborted) {
return null;
}
const termsRefused = reopenTermsOnRefusal(queryClient, errorData);
errorHandler({
data: getStreamStartFailureData(
errorData,
termsRefused ? localize('com_error_terms_not_accepted') : undefined,
),
submission: currentSubmission as EventSubmission,
});
setShowStopButton(false);
setIsSubmitting(false);
setSubmission(null);
return null;
},
[
run,
clearStepMaps,
convertSteersToQueued,
errorHandler,
jotaiStore,
localize,
queryClient,
restoreQueuedSubmission,
setIsSubmitting,
setShowStopButton,
setSubmission,
transport,
],
const subscribeToStream = useCallback(
(target: SubscriptionTarget) => attachResumableStream(run, buildHost(), target),
[buildHost, run],
);
useEffect(() => {
@ -952,7 +698,7 @@ export default function useResumableSSE(
} else {
// New generation: start and then subscribe
logger.log('ResumableSSE', 'Starting NEW generation');
const startResult = await startGeneration(submission, signal);
const startResult = await startGeneration(run, buildHost(), submission, signal);
if (!isCurrentEffect()) {
return;
}

View file

@ -110,6 +110,9 @@ export const createHost = (
resolve: jest.fn(() => []),
markLiveApplied: jest.fn(),
update: jest.fn(),
isRecoveryHeld: jest.fn(() => false),
blockRecovery: jest.fn(),
restoreQueued: jest.fn(),
seed: jest.fn(),
appliedIds: jest.fn(() => []),
settleApplied: jest.fn(),
@ -131,6 +134,7 @@ export const createHost = (
flushPendingDeltas: jest.fn(),
resetLive: jest.fn(),
showError: jest.fn(),
showStartFailure: jest.fn(),
},
messages: {
get: jest.fn(() => messages),
@ -158,6 +162,7 @@ export const createHost = (
seedLiveFromContent: jest.fn(),
},
stream: { send: jest.fn(), reconnectToStream: jest.fn(() => ({ closed: false })) },
start: jest.fn(),
};
return host;
};

View file

@ -0,0 +1,111 @@
import type { TMessage, TSubmission } from 'librechat-data-provider';
import { createHost, createScheduler } from '../__fixtures__/resumable';
import { ResumableRun, startGeneration } from '../resumable';
const submission = {
userMessage: { messageId: 'u-1', conversationId: 'c-1', isCreatedByUser: true, text: 'Hi' },
initialResponse: { messageId: 'u-1_', conversationId: 'c-1' } as TMessage,
conversation: { conversationId: 'c-1', endpoint: 'agents' },
endpointOption: { endpoint: 'agents' },
} as unknown as TSubmission;
const httpError = (status: number, data: unknown) =>
Object.assign(new Error('failed'), { response: { status, data, headers: {} } });
const setup = () => {
const host = createHost();
const run = new ResumableRun(createScheduler().scheduler);
return { host, run };
};
describe('startGeneration', () => {
it('resolves to the stream a start names', async () => {
const { host, run } = setup();
(host.start as jest.Mock).mockResolvedValue({
streamId: 's-1',
generationCreatedAt: 1000,
generationProtocolVersion: 2,
});
await expect(startGeneration(run, host, submission)).resolves.toEqual({
status: 'stream',
streamId: 's-1',
resumed: false,
generationCreatedAt: 1000,
generationProtocolVersion: 2,
});
expect(host.events.clearStepMaps).toHaveBeenCalled();
});
it('reports a settled v2 start', async () => {
const { host, run } = setup();
(host.start as jest.Mock).mockResolvedValue({
status: 'settled',
conversationId: 'c-1',
generationProtocolVersion: 2,
});
await expect(startGeneration(run, host, submission)).resolves.toEqual({
status: 'settled',
conversationId: 'c-1',
generationProtocolVersion: 2,
});
});
it('restores the queued turn when the predecessor fence was lost', async () => {
const { host, run } = setup();
(host.start as jest.Mock).mockRejectedValue(
httpError(409, {
status: 'predecessor_mismatch',
code: 'GENERATION_PREDECESSOR_MISMATCH',
streamId: 'c-1',
conversationId: 'c-1',
generationCreatedAt: 3000,
active: true,
}),
);
await expect(startGeneration(run, host, submission)).resolves.toMatchObject({
status: 'predecessor_mismatch',
generationCreatedAt: 3000,
});
expect(host.steers.restoreQueued).toHaveBeenCalledWith(submission);
});
it('writes the error card and idles the pane on a rejected start', async () => {
const { host, run } = setup();
const body = { message: 'forbidden' };
(host.start as jest.Mock).mockRejectedValue(httpError(403, body));
await expect(startGeneration(run, host, submission)).resolves.toBeNull();
expect(host.steers.restoreQueued).toHaveBeenCalledWith(submission);
expect(host.events.showStartFailure).toHaveBeenCalledWith(submission, body);
expect(host.pane.setIsSubmitting).toHaveBeenCalledWith(false);
expect(host.pane.setSubmission).toHaveBeenCalledWith(null);
});
it('blocks a recovery the server refused', async () => {
const { host, run } = setup();
(host.start as jest.Mock).mockRejectedValue(
httpError(400, { code: 'RECOVERY_PAYLOAD_MISMATCH' }),
);
await startGeneration(run, host, { ...submission, recoverySteerId: 'steer-1' } as TSubmission);
expect(host.steers.blockRecovery).toHaveBeenCalledWith('c-1', 'steer-1');
});
it('refuses a recovery the user already dealt with', async () => {
const { host, run } = setup();
(host.steers.isRecoveryHeld as jest.Mock).mockReturnValue(true);
const recovery = { ...submission, recoverySteerId: 'steer-1' } as TSubmission;
await expect(startGeneration(run, host, recovery)).resolves.toBeNull();
expect(host.start).not.toHaveBeenCalled();
expect(host.events.showStartFailure).toHaveBeenCalledWith(recovery, undefined, 'recovery_held');
});
it('stops quietly once the pane abandons the start', async () => {
const { host, run } = setup();
const controller = new AbortController();
(host.start as jest.Mock).mockImplementation(async () => {
controller.abort();
throw Object.assign(new Error('offline'), { code: 'ERR_NETWORK' });
});
await expect(startGeneration(run, host, submission, controller.signal)).resolves.toBeNull();
expect(host.events.showStartFailure).not.toHaveBeenCalled();
});
});

View file

@ -6,6 +6,7 @@ import type {
TPendingSteer,
TAttachment,
ChatTransport,
ChatTransportRequest,
ChatTitleFrame,
ChatFinalFrame,
EventSubmission,
@ -127,6 +128,12 @@ export interface ResumableSteers {
markLiveApplied: (steerId: string) => void;
/** Patches the chips the server re-labelled (preempt capability); never removes one. */
update: (event: TSteerUpdatedEvent) => void;
/** Whether the user has already dealt with the parked steer a recovery start replays. */
isRecoveryHeld: (conversationId: string, recoverySteerId: string) => boolean;
/** Refuses further recovery of a parked steer the server rejected. */
blockRecovery: (conversationId: string, recoverySteerId: string) => void;
/** Puts a queued turn whose start was refused back in its queue position. */
restoreQueued: (submission: TSubmission, expectedPredecessorCreatedAt?: number) => void;
/** Re-seeds the chips from the server's still-queued steers. */
seed: (
conversationId: string,
@ -247,6 +254,13 @@ export interface ResumableEvents {
flushPendingDeltas: () => void;
/** Discards the live usage estimate for a run that stopped streaming. */
resetLive: (submission: TSubmission) => void;
/** Writes the error card for a start that failed, from the response body it failed with, or
* for a recovery start the user already dealt with. */
showStartFailure: (
submission: TSubmission,
errorData?: unknown,
reason?: 'recovery_held',
) => void;
/** Writes the failed turn's error card from the text (or raw error value) the server sent. */
showError: (text: string | boolean | { message?: string }, submission: TSubmission) => void;
}
@ -308,6 +322,8 @@ export interface ResumableHost extends SubscriptionHost {
usage: ResumableUsage;
/** The stream side of the app's transport, bound to the signed-in user. */
stream: ChatTransport;
/** POSTs a turn to the resumable generation route; resolves to the start response. */
start: (request: ChatTransportRequest, options?: { signal?: AbortSignal }) => Promise<unknown>;
/** Receives the dev-only controls that simulate a dropped or a clean close. */
exposeDebugControls?: (controls: { killNetwork: () => void; closeClean: () => void }) => void;
}

View file

@ -15,3 +15,4 @@ export * from './patches';
export * from './frames';
export * from './sync';
export * from './attach';
export * from './launch';

View file

@ -0,0 +1,259 @@
import { createPayload, removeNullishValues } from 'librechat-data-provider';
import type { TPayload, TSubmission } from 'librechat-data-provider';
import type { StartGenerationResult } from './start';
import type { ResumableHost } from './host';
import type { ResumableRun } from './run';
import {
getRecoverySteerId,
getRetryAfterDelay,
waitForRetryDelay,
isServerNotReadyError,
isResumedStartResponse,
isSettledStartResponse,
toStartGenerationError,
isRetryableNetworkError,
isReplacedStartResponse,
getStartGenerationStreamId,
getStartGenerationCreatedAt,
isUnnegotiatedV2StartControl,
START_GENERATION_NETWORK_RETRIES,
getStartGenerationConversationId,
getPredecessorMismatchStartResponse,
START_GENERATION_READINESS_TIMEOUT_MS,
} from './start';
import {
GENERATION_PROTOCOL_VERSION,
getGenerationProtocolVersion,
supportsGenerationProtocolV2,
} from '../protocol';
/**
* Starts a generation (the POST that returns the stream to attach to) through the host
* transport, which owns auth refresh. Retries transient network failures and startup
* readiness responses; readiness retries honor Retry-After until `signal` aborts or the
* readiness window expires. Resolves to null when the start failed (its error card written)
* or was abandoned.
*/
export async function startGeneration(
run: ResumableRun,
host: ResumableHost,
currentSubmission: TSubmission,
signal?: AbortSignal,
): Promise<StartGenerationResult | null> {
const payloadData = createPayload(currentSubmission);
let { payload } = payloadData;
payload = removeNullishValues(payload) as TPayload;
host.events.clearStepMaps();
const url = payloadData.server;
let lastError: unknown = null;
let requestAttempts = 0;
let networkAttempts = 0;
let readinessAttempts = 0;
const readinessDeadline = run.scheduler.now() + START_GENERATION_READINESS_TIMEOUT_MS;
while (!signal?.aborted) {
const recoverySteerId = getRecoverySteerId(currentSubmission);
const conversationId = currentSubmission.conversation?.conversationId;
if (
recoverySteerId != null &&
conversationId &&
host.steers.isRecoveryHeld(conversationId, recoverySteerId)
) {
host.steers.restoreQueued(currentSubmission);
host.events.showStartFailure(currentSubmission, undefined, 'recovery_held');
host.pane.setShowStopButton(false);
host.pane.setIsSubmitting(false);
host.pane.setSubmission(null);
return null;
}
requestAttempts += 1;
try {
const data = await host.start({ server: url, payload }, { signal });
if (signal?.aborted) {
return null;
}
if (isSettledStartResponse(data)) {
const conversationId = getStartGenerationConversationId(data);
if (conversationId) {
host.logger.log('ResumableSSE', 'Generation already settled:', { conversationId });
return {
status: 'settled',
conversationId,
generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
};
}
lastError = { response: { data } };
break;
}
if (isReplacedStartResponse(data)) {
const streamId = getStartGenerationStreamId(data);
const conversationId = getStartGenerationConversationId(data);
const generationCreatedAt = getStartGenerationCreatedAt(data);
if (streamId && conversationId && generationCreatedAt != null) {
return {
status: 'replaced',
streamId,
conversationId,
generationCreatedAt,
generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
};
}
lastError = { response: { data } };
break;
}
if (isUnnegotiatedV2StartControl(data)) {
/** Never reinterpret a v2-only control word as a legacy stream
* response merely because it also carries a stream id. */
lastError = { response: { data } };
break;
}
const streamId = getStartGenerationStreamId(data);
if (streamId) {
const resumed = isResumedStartResponse(data);
const generationCreatedAt = getStartGenerationCreatedAt(data);
const generationProtocolVersion = getGenerationProtocolVersion(data);
host.logger.log('ResumableSSE', 'Generation started:', {
streamId,
resumed,
generationCreatedAt,
generationProtocolVersion,
});
return {
status: 'stream',
streamId,
resumed,
generationProtocolVersion,
...(generationCreatedAt != null && { generationCreatedAt }),
};
}
lastError = { response: { data } };
break;
} catch (error) {
if (signal?.aborted) {
return null;
}
const predecessorMismatch = getPredecessorMismatchStartResponse(error);
if (predecessorMismatch != null) {
/** Admission was atomically rejected before this queued turn
* created anything. Put the exact row back immediately and let
* the caller reconcile or hand off to the generation that won. */
host.steers.restoreQueued(currentSubmission);
host.logger.log('ResumableSSE', 'Queued generation lost predecessor fence', {
expectedPredecessorCreatedAt: currentSubmission.expectedPredecessorCreatedAt,
...predecessorMismatch,
});
return {
status: 'predecessor_mismatch',
...predecessorMismatch,
generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
};
}
lastError = error;
const isNetworkError = isRetryableNetworkError(error);
const isServerNotReady = isServerNotReadyError(error);
const remainingReadinessMs = readinessDeadline - run.scheduler.now();
const shouldRetryNetwork =
isNetworkError && networkAttempts < START_GENERATION_NETWORK_RETRIES - 1;
const shouldRetryServerNotReady = isServerNotReady && remainingReadinessMs > 0;
if (shouldRetryNetwork || shouldRetryServerNotReady) {
networkAttempts += isNetworkError ? 1 : 0;
readinessAttempts += isServerNotReady ? 1 : 0;
const fallbackDelay = Math.min(1000 * Math.pow(2, requestAttempts - 1), 8000);
const retryDelay = isServerNotReady
? Math.min(getRetryAfterDelay(error, fallbackDelay), remainingReadinessMs)
: fallbackDelay;
const reason = isServerNotReady ? 'Server not ready' : 'Network error';
const attempt = isServerNotReady ? readinessAttempts : networkAttempts;
const limit = isServerNotReady
? `${Math.ceil(START_GENERATION_READINESS_TIMEOUT_MS / 1000)}s readiness window`
: `${START_GENERATION_NETWORK_RETRIES}`;
host.logger.log(
'ResumableSSE',
`${reason} starting generation, retrying in ${retryDelay}ms (attempt ${attempt}/${limit})`,
);
const shouldContinue = await waitForRetryDelay(retryDelay, signal);
if (!shouldContinue) {
return null;
}
continue;
}
// Don't retry: either not a network error or max retries reached
break;
}
}
if (signal?.aborted) {
return null;
}
host.logger.error('ResumableSSE', 'Error starting generation:', lastError);
const startError = toStartGenerationError(lastError);
const errorData = startError?.response?.data;
const responseStatus = startError?.response?.status;
if (responseStatus != null && responseStatus >= 400 && responseStatus < 500) {
const recoverySteerId = getRecoverySteerId(currentSubmission);
const conversationId = currentSubmission.conversation?.conversationId;
const recoveryRejected =
errorData != null &&
typeof errorData === 'object' &&
'code' in errorData &&
(errorData.code === 'RECOVERY_PAYLOAD_MISMATCH' ||
errorData.code === 'INVALID_RECOVERY_REQUEST');
if (recoveryRejected && recoverySteerId != null && conversationId) {
host.steers.blockRecovery(conversationId, recoverySteerId);
}
// The server rejected admission before exposing a generation. Restore
// the exact queue row/position; ambiguous transport/5xx outcomes must
// first reconcile durable state instead of risking a duplicate start.
host.steers.restoreQueued(currentSubmission);
} else {
const recoverySteerId = getRecoverySteerId(currentSubmission);
const recoveryConvoId = currentSubmission.conversation?.conversationId;
if (recoverySteerId != null && recoveryConvoId) {
try {
const status = await host.fetchStatus(recoveryConvoId);
if (signal?.aborted) {
return null;
}
const releasedSource = status.unrecoveredSteers?.find(
(steer) => steer.steerId === recoverySteerId,
);
if (
status.active === false &&
supportsGenerationProtocolV2(status) &&
releasedSource != null
) {
host.steers.convertToQueued(recoveryConvoId, [releasedSource], {
generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
allowPreviouslyConvertedIds: [recoverySteerId],
});
}
} catch (error) {
if (signal?.aborted) {
return null;
}
host.logger.warn('ResumableSSE', 'Could not recover source after start failure', {
conversationId: recoveryConvoId,
error,
});
}
}
}
if (signal?.aborted) {
return null;
}
host.events.showStartFailure(currentSubmission, errorData);
host.pane.setShowStopButton(false);
host.pane.setIsSubmitting(false);
host.pane.setSubmission(null);
return null;
}