🛟 refactor: Move Resumable Transport Failure Handling Into the Chat Core (#17120)
Some checks are pending
Backend Unit Tests / Build packages (push) Waiting to run
Backend Unit Tests / Codegraph select (push) Waiting to run
Backend Unit Tests / TypeScript type checks (push) Blocked by required conditions
Backend Unit Tests / Circular dependency checks (push) Waiting to run
Backend Unit Tests / Tests: api (shard 1/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 2/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 3/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: data-provider (push) Blocked by required conditions
Backend Unit Tests / Tests: data-schemas (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 1/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 2/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 3/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 4/4) (push) Blocked by required conditions
Codegraph E2E Votes / vote (full suite) (push) Waiting to run
Docker Dev Branch Images Build / publish (push) Waiting to run
Frontend Unit Tests / Client build recovery regression (push) Waiting to run
Frontend Unit Tests / Codegraph select (push) Waiting to run
Frontend Unit Tests / Build packages (push) Waiting to run
Frontend Unit Tests / TypeScript type checks (client) (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/chat (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/client (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 1/2) (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 2/2) (push) Blocked by required conditions
Frontend Unit Tests / Vite build verification (push) Blocked by required conditions
Sync Locize Translations & Create Translation PR / Sync Translation Keys with Locize (push) Waiting to run
Sync Locize Translations & Create Translation PR / Create Translation PR on Version Published (push) Blocked by required conditions

This commit is contained in:
Marco Beretta 2026-10-15 19:05:12 +02:00 • committed by GitHub
parent f43e7fa041
commit d4e52acdfb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 952 additions and 651 deletions

View file

@ -8,7 +8,6 @@ import { ABORT_SWEEP_STATUSES, selectLocalSteersForQueue } from '@librechat/chat
import {
Constants,
QueryKeys,
ErrorTypes,
StepEvents,
StepTypes,
SteerEvents,
@ -20,15 +19,12 @@ import {
UsageEvents,
createPayload,
ApprovalEvents,
ViolationTypes,
removeNullishValues,
} from 'librechat-data-provider';
import {
MAX_RETRIES,
getStepEventId,
toStartGenerationError,
isOAuthStepEvent,
getReconnectDelay,
waitForRetryDelay,
applyPendingAction,
getRecoverySteerId,
@ -48,7 +44,6 @@ import {
keepLocalCodeApprovalMode,
hydrateSubmissionMessages,
buildResumeEventSubmission,
completedResponseMessageId,
getStartGenerationStreamId,
buildOptimisticConversation,
buildCreatedInitialResponse,
@ -65,9 +60,9 @@ import {
ResumableRun,
documentVisibility,
ResumableSubscription,
handoffToReplacement,
authorizeTerminalTeardown,
reconcileGenerationLifecycle,
handleTransportFailure,
clearPendingApprovalForTerminal,
clearAttachedGenerationCreatedAt,
} from '@librechat/chat';
@ -288,6 +283,16 @@ export default function useResumableSSE(
queryClient.invalidateQueries({ queryKey: streamStatusQueryKey(conversationId) }),
addActiveJob,
removeActiveJob,
fetchConversation: (conversationId) =>
fetchConvoSnapshot(queryClient, conversationId, () =>
dataService.getConversationById(conversationId),
),
setConversation: (conversationId, conversation) =>
queryClient.setQueryData([QueryKeys.conversation, conversationId], conversation),
upsertConversation: (conversation) =>
upsertConvoInAllQueries(queryClient, conversation, true, 'snapshot'),
removeConversation: (conversationId) =>
removeConvoFromAllQueries(queryClient, conversationId),
}),
[addActiveJob, queryClient, removeActiveJob],
);
@ -487,6 +492,23 @@ export default function useResumableSSE(
clearComposerDrafts(runIndex, conversationId, options),
publishRunEnd: setRunEnd,
clearDrainAfterAbort,
refreshBalance: () => {
(startupConfig?.balance?.enabled ?? false) && balanceQuery.refetch();
},
updateConversation: isAddedRequest ? undefined : setConversation,
resolveDetachedEnd: (conversationId, messages) => {
const detachedRun = jotaiStore.get(detachedRunByConvoId(conversationId));
const resolvable =
detachedRun != null &&
!isDetachedRunManual(jotaiStore.get, conversationId, detachedRun);
return {
resolvable,
end:
resolvable && messages != null
? resolveDetachedRunEnd(conversationId, detachedRun, messages)
: null,
};
},
},
steers: {
seed: seedSteerChips,
@ -495,7 +517,17 @@ export default function useResumableSSE(
convertToQueued: convertSteersToQueued,
convertLocalToQueued: convertLocalSteersToQueued,
},
events: { clearStepMaps, flushPendingDeltas, resetLive },
events: {
clearStepMaps,
flushPendingDeltas,
resetLive,
showError: (text, errorSubmission) =>
errorHandler({
data: { text } as unknown as Parameters<typeof errorHandler>[0]['data'],
submission: errorSubmission as EventSubmission,
}),
},
messages: { get: getMessages, set: setMessages },
};
const sub = new ResumableSubscription(run, host, target);
const {
@ -1587,644 +1619,6 @@ export default function useResumableSSE(
}
};
/**
* Error event handler - handles BOTH:
* 1. HTTP-level errors (status present) - 404, 409, network failures (0)
* 2. Server-sent error events (event: error with data) - known errors like ViolationTypes/ErrorTypes
*
* Order matters: check the status first since HTTP errors may also include data.
* The transport has already retried a 401 with a refreshed token.
*/
const handleTransportFailure = async ({
status: responseCode,
data,
}: Pick<Extract<ChatEvent, { type: 'error' }>, 'status' | 'data'>) => {
if (!sub.isCurrent()) {
return;
}
if (sub.finalReceived) {
logger.log('ResumableSSE', 'Ignoring error after FINAL event', {
responseCode,
hasData: data != null,
});
return;
}
(startupConfig?.balance?.enabled ?? false) && balanceQuery.refetch();
if (responseCode === 409) {
const replacementConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;
const handedOff = await handoffToReplacement(terminal, replacementConvoId);
if (!sub.isCurrent()) {
return;
}
if (handedOff) {
return;
}
}
// 404 → job completed & was cleaned up; messages are persisted in DB.
// Refetch and reconcile them before recovering local steers: the final
// response may contain an applied steer whose live event was missed.
if (responseCode === 404) {
const convoId = sub.submission.conversation?.conversationId;
logger.log('ResumableSSE', 'Stream 404, invalidating messages for:', convoId);
// Keep the UI non-idle while the status read decides whether this is
// terminal cleanup or a handoff to a newer generation epoch.
run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);
run.cancelSteerRetryFrames();
sub.close();
/** Terminal: drop any in-flight live estimate so the gauge doesn't
* keep counting stale streamed output after the stream ends */
resetLive(sub.liveSubmission());
clearComposerDrafts(runIndex, convoId, {
includeNewChatDraft:
!convoId ||
convoId === Constants.NEW_CONVO ||
run.optimisticStreamIds.has(currentStreamId) ||
isInitialNewConversation(sub.submission),
});
clearStepMaps();
let persistedMessages: TMessage[] | undefined;
if (convoId) {
try {
const messageQueryKey = [QueryKeys.messages, convoId] as const;
await queryClient.invalidateQueries({
queryKey: messageQueryKey,
refetchType: 'none',
});
if (!sub.isCurrent()) {
return;
}
const fetched = await queryClient.fetchQuery<TMessage[]>({
queryKey: messageQueryKey,
// The first stream can finish before CREATED mounts the saved-chat query.
queryFn: () => fetchConversationMessages(convoId),
});
if (!sub.isCurrent()) {
return;
}
if (Array.isArray(fetched)) {
persistedMessages = fetched;
}
} catch (error) {
if (!sub.isCurrent()) {
return;
}
logger.warn('ResumableSSE', 'Could not reconcile persisted messages after 404', {
conversationId: convoId,
error,
});
}
queryClient.removeQueries({ queryKey: streamStatusQueryKey(convoId) });
}
const recoveryConvoId = convoId ?? currentStreamId;
const reconciledIds = new Set(
persistedMessages ? collectAppliedSteerIds(persistedMessages) : [],
);
if (persistedMessages) {
settleAppliedSteerParts(recoveryConvoId, persistedMessages);
}
// The status replay is owner-gated and non-destructive; creating a
// recovered follow-up consumes its exact source. Prefer these
// authoritative leftovers over guessing from local chips.
let confirmedV2Terminal = false;
try {
const status = await fetchStreamStatus(recoveryConvoId);
if (!sub.isCurrent()) {
return;
}
if (
generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
!supportsGenerationProtocolV2(status)
) {
sub.scheduleReattach(1_000);
return;
}
if (
status.active === true &&
status.createdAt != null &&
(generationCreatedAt == null || status.createdAt !== generationCreatedAt)
) {
const handedOff = await handoffToReplacement(terminal, recoveryConvoId, status);
if (!sub.isCurrent()) {
return;
}
if (handedOff) {
return;
}
/** Legacy generations have no replacement-handoff contract.
* Never attach this submission to a different epoch. */
sub.scheduleReattach(1_000);
return;
}
if (status.active === true) {
/** A stream lookup can briefly lose its routing record while the
* authoritative job status still proves the same generation is
* alive. Preserve ownership and retry the attachment instead of
* publishing a terminal run-end (which could auto-drain queued
* text into a still-running generation). */
const activeStreamId = status.streamId ?? currentStreamId;
queryClient.setQueryData(streamStatusQueryKey(recoveryConvoId), status);
if (status.createdAt != null) {
updateActiveGenerationCreatedAt(
recoveryConvoId,
status.createdAt,
undefined,
generationProtocolVersion,
);
}
addActiveJob(activeStreamId);
seedSteerChips(
recoveryConvoId,
(status.resumeState?.pendingSteers ?? []) as TPendingSteer[],
status.createdAt,
generationProtocolVersion,
);
settleAppliedSteerParts(
recoveryConvoId,
status.resumeState?.aggregatedContent ?? status.aggregatedContent,
);
if (status.pendingAction ?? status.resumeState?.pendingAction) {
applyPendingActionToMessages(
(status.pendingAction ??
status.resumeState?.pendingAction) as Agents.PendingAction,
);
}
setIsSubmitting(true);
setShowStopButton(status.createdAt != null);
run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);
sub.scheduleReattach(1_000, activeStreamId, status.createdAt);
return;
}
confirmedV2Terminal =
generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
supportsGenerationProtocolV2(status);
const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal;
const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : [];
if (unrecovered.length > 0) {
const recoverySteerId = getRecoverySteerId(sub.submission);
convertSteersToQueued(recoveryConvoId, unrecovered, {
generationProtocolVersion,
...(recoverySteerId != null && {
allowPreviouslyConvertedIds: [recoverySteerId],
}),
});
for (const steer of unrecovered) {
reconciledIds.add(steer.steerId);
if (steer.clientSteerId) {
reconciledIds.add(steer.clientSteerId);
}
}
}
} catch (error) {
if (!sub.isCurrent()) {
return;
}
logger.warn('ResumableSSE', 'Could not recover parked steers after 404', {
conversationId: recoveryConvoId,
error,
});
}
removeActiveJob(currentStreamId);
clearAttachedGenerationCreatedAt(terminal);
clearPendingApprovalForTerminal(terminal, recoveryConvoId);
if (
!run.createdStreamIds.has(currentStreamId) &&
run.optimisticStreamIds.has(currentStreamId)
) {
// Both fresh and resumed subscriptions can miss every event of a fast turn.
// A missing job proves neither that the conversation was saved nor that it failed.
try {
const persisted = await fetchConvoSnapshot(queryClient, recoveryConvoId, () =>
dataService.getConversationById(recoveryConvoId),
);
if (!sub.isCurrent()) return;
if (persisted?.conversationId === recoveryConvoId) {
queryClient.setQueryData([QueryKeys.conversation, recoveryConvoId], persisted);
upsertConvoInAllQueries(queryClient, persisted, true, 'snapshot');
if (!isAddedRequest) {
setConversation?.((current) => {
if (
current?.conversationId != null &&
current.conversationId !== Constants.NEW_CONVO &&
current.conversationId !== recoveryConvoId
)
return current;
return keepLocalCodeApprovalMode(
{ ...current, ...persisted },
current,
current?.conversationId,
);
});
}
}
} catch (error) {
if (!sub.isCurrent()) return;
if (toStartGenerationError(error)?.response?.status === 404) {
removeConvoFromAllQueries(queryClient, currentStreamId);
}
}
// An inconclusive read must not turn a saved chat into a phantom.
invalidateConversationLists(queryClient);
queryClient.invalidateQueries({ queryKey: [QueryKeys.pinnedConversations] });
}
if (persistedMessages) {
setMessages(persistedMessages);
}
sub.retired = true;
setIsSubmitting(false);
setShowStopButton(false);
// Only infer recovery from local accepted chips after the persisted
// final was successfully inspected. If that fetch failed, their
// applied-vs-leftover outcome is unknown and converting could send
// already-applied words twice.
if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) {
convertLocalSteersToQueued(recoveryConvoId, {
excludeSteerIds: reconciledIds,
generationProtocolVersion,
});
}
// The true outcome is unknown here (job record already cleaned up):
// a non-'completed' outcome releases parked interrupt flags without
// auto-sending queued messages the user may not want fired. A run this
// pane left and came back to is the exception: the history just read
// names its response by epoch, which is the outcome it would have had.
// History read before the job finished may not hold that response yet, or
// may hold only the unfinished snapshot saved when the pane left, so a
// marker that read leaves unresolved or stopped outlives this end for the
// next visit, which reads history after the job is known to be done.
const detachedRun = jotaiStore.get(detachedRunByConvoId(recoveryConvoId));
const detachedResolvable =
detachedRun != null &&
!isDetachedRunManual(jotaiStore.get, recoveryConvoId, detachedRun);
const detachedEnd =
detachedResolvable && persistedMessages != null
? resolveDetachedRunEnd(recoveryConvoId, detachedRun, persistedMessages)
: null;
setRunEnd(
{
conversationId: recoveryConvoId,
outcome: 'aborted',
endedAt: run.scheduler.now(),
...detachedEnd,
startedAsNewConvo: run.optimisticStreamIds.has(currentStreamId),
},
{
retainDetached:
detachedResolvable && (detachedEnd == null || detachedEnd.outcome === 'aborted'),
},
);
setStreamId(null);
run.releaseStream(currentStreamId);
run.reconnectAttempt = 0;
return;
}
/**
* Server-sent error event (event: error with data) - no responseCode.
* These are known errors (ErrorTypes, ViolationTypes) that should be displayed to user.
* Only check the data if there's no HTTP status, since HTTP errors may also have body data.
* Note: responseCode === 0 means transport failure (connection dropped) - treat as network error,
* not a server-sent error payload. Use `== null` to only match undefined/null (no HTTP status).
*/
if (responseCode == null && data != null) {
sub.finalReceived = true;
const recoveryConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;
if (
!(await authorizeTerminalTeardown(terminal, recoveryConvoId, 'error')) ||
!sub.isCurrent()
) {
return;
}
logger.log('ResumableSSE', 'Server-sent error event received:', data);
run.cancelSteerRetryFrames();
sub.close();
/** FLUSH (not cancel): the error card below is built from the cache
* tail, so queued tokens must land first, and a stale trailing
* frame must never overwrite the error write. */
flushPendingDeltas();
removeActiveJob(currentStreamId);
clearAttachedGenerationCreatedAt(terminal);
clearPendingApprovalForTerminal(terminal, recoveryConvoId);
resetLive(sub.liveSubmission());
if (
!run.createdStreamIds.has(currentStreamId) &&
run.optimisticStreamIds.has(currentStreamId)
) {
removeConvoFromAllQueries(queryClient, currentStreamId);
}
const errorSupportsV2 = supportsGenerationProtocolV2(data);
const errorString =
typeof data === 'string' ? data : (data.error ?? data.message ?? JSON.stringify(data));
// Check if it's a known error type (ViolationTypes or ErrorTypes)
let isKnownError = false;
try {
const parsed = typeof errorString === 'string' ? JSON.parse(errorString) : errorString;
const errorType = parsed?.type ?? parsed?.code;
if (errorType) {
const violationValues = Object.values(ViolationTypes) as string[];
const errorTypeValues = Object.values(ErrorTypes) as string[];
isKnownError =
violationValues.includes(errorType) || errorTypeValues.includes(errorType);
}
} catch {
// Not JSON or parsing failed - treat as generic error
}
logger.log('ResumableSSE', 'Error type check:', { isKnownError, errorString });
// Display the error to user via errorHandler
errorHandler({
data: { text: errorString } as unknown as Parameters<typeof errorHandler>[0]['data'],
submission: sub.submission as EventSubmission,
});
setIsSubmitting(false);
setShowStopButton(false);
const recoverySteerId = getRecoverySteerId(sub.submission);
if (
recoverySteerId != null &&
generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
errorSupportsV2
) {
try {
/** The error was received on the exact generation-fenced stream.
* Read status only now, after terminal was observed: a claim
* started before this recovery could return a stale parked copy
* after a successful commit and must never authorize redelivery. */
const terminalStatus = await fetchStreamStatus(recoveryConvoId);
if (!sub.isCurrent()) {
return;
}
const releasedSource = terminalStatus.unrecoveredSteers?.find(
(steer) => steer.steerId === recoverySteerId,
);
if (
terminalStatus.active === false &&
supportsGenerationProtocolV2(terminalStatus) &&
releasedSource != null
) {
convertSteersToQueued(recoveryConvoId, [releasedSource], {
generationProtocolVersion,
allowPreviouslyConvertedIds: [recoverySteerId],
});
}
} catch (error) {
if (!sub.isCurrent()) {
return;
}
logger.warn('ResumableSSE', 'Could not recover source after generation error', {
conversationId: recoveryConvoId,
error,
});
}
}
// The error terminal's backstop parks acked leftovers server-side;
// claim it now so a reload can't resurrect the chips converted here.
if (generationProtocolVersion === 1 || errorSupportsV2) {
convertLocalSteersToQueued(recoveryConvoId, {
claimParked: true,
generationProtocolVersion,
});
}
setRunEnd({
conversationId: recoveryConvoId,
outcome: 'error',
endedAt: run.scheduler.now(),
generationCreatedAt,
});
setStreamId(null);
run.releaseStream(currentStreamId);
run.reconnectAttempt = 0;
return;
}
// Network failure or unknown HTTP error - attempt reconnection with backoff
logger.log('ResumableSSE', 'Stream error (network failure) - will attempt reconnect', {
responseCode,
hasData: data != null,
});
if (run.reconnectAttempt < MAX_RETRIES) {
// Increment counter BEFORE close() so abort handler knows we're reconnecting
run.reconnectAttempt++;
const delay = getReconnectDelay(run.reconnectAttempt);
logger.log(
'ResumableSSE',
`Reconnecting in ${delay}ms (attempt ${run.reconnectAttempt}/${MAX_RETRIES})`,
);
sub.close();
sub.scheduleReattach(delay);
// Keep UI in "submitting" state during reconnection attempts
// so user knows we're still trying (abort handler may have reset these)
setIsSubmitting(true);
setShowStopButton(generationCreatedAt != null);
} else {
logger.error('ResumableSSE', 'Max reconnect attempts reached');
sub.close();
flushPendingDeltas();
const recoveryConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;
let status: Awaited<ReturnType<typeof fetchStreamStatus>> | undefined;
try {
status = await fetchStreamStatus(recoveryConvoId);
if (!sub.isCurrent()) {
return;
}
} catch (error) {
if (!sub.isCurrent()) {
return;
}
logger.warn('ResumableSSE', 'Could not determine job state after reconnect limit', {
conversationId: recoveryConvoId,
error,
});
}
if (
status != null &&
generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
!supportsGenerationProtocolV2(status)
) {
// A legacy responder cannot adjudicate a v2 job during rollout.
status = undefined;
}
if (
status?.active === true &&
status.createdAt != null &&
(generationCreatedAt == null || status.createdAt !== generationCreatedAt)
) {
const handedOff = await handoffToReplacement(terminal, recoveryConvoId, status);
if (!sub.isCurrent()) {
return;
}
if (handedOff) {
return;
}
run.reconnectAttempt = 0;
sub.scheduleReattach(30_000);
return;
}
if (status?.active === true) {
// The transport gave up, not the generation. Keep the active-job
// marker and accepted chips intact, then release this stale
// submission so useResumeOnLoad can establish a fresh attachment.
queryClient.setQueryData(streamStatusQueryKey(recoveryConvoId), status);
seedSteerChips(
recoveryConvoId,
(status.resumeState?.pendingSteers ?? []) as TPendingSteer[],
status.createdAt,
generationProtocolVersion,
);
settleAppliedSteerParts(
recoveryConvoId,
status.resumeState?.aggregatedContent ?? status.aggregatedContent,
);
if (status.pendingAction ?? status.resumeState?.pendingAction) {
applyPendingActionToMessages(
(status.pendingAction ?? status.resumeState?.pendingAction) as Agents.PendingAction,
);
}
setIsSubmitting(true);
setShowStopButton(generationCreatedAt != null);
run.reconnectAttempt = 0;
sub.scheduleReattach(30_000);
return;
}
if (status == null) {
// An inconclusive status read is still not proof of termination.
// Preserve server/client ownership and let the normal resume-on-load
// status query retry rather than duplicating accepted words.
queryClient.invalidateQueries({ queryKey: streamStatusQueryKey(recoveryConvoId) });
setIsSubmitting(true);
setShowStopButton(generationCreatedAt != null);
run.reconnectAttempt = 0;
sub.scheduleReattach(30_000);
return;
}
// The status endpoint proved the job terminal. Reconcile persisted
// content before recovering leftovers, just like the 404 path.
run.cancelSteerRetryFrames();
let persistedMessages: TMessage[] | undefined;
try {
const messageQueryKey = [QueryKeys.messages, recoveryConvoId] as const;
await queryClient.invalidateQueries({
queryKey: messageQueryKey,
refetchType: 'none',
});
if (!sub.isCurrent()) {
return;
}
const fetched = await queryClient.fetchQuery<TMessage[]>({
queryKey: messageQueryKey,
});
if (!sub.isCurrent()) {
return;
}
if (Array.isArray(fetched)) {
persistedMessages = fetched;
}
} catch (error) {
if (!sub.isCurrent()) {
return;
}
logger.warn('ResumableSSE', 'Could not reconcile terminal messages after disconnect', {
conversationId: recoveryConvoId,
error,
});
}
const authoritativeValues = [
...(persistedMessages ?? []),
...(status.resumeState?.aggregatedContent ?? status.aggregatedContent ?? []),
];
const confirmedV2Terminal =
generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
supportsGenerationProtocolV2(status);
const reconciledIds = new Set(collectAppliedSteerIds(authoritativeValues));
settleAppliedSteerParts(recoveryConvoId, authoritativeValues);
const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal;
const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : [];
if (unrecovered.length > 0) {
const recoverySteerId = getRecoverySteerId(sub.submission);
convertSteersToQueued(recoveryConvoId, unrecovered, {
generationProtocolVersion,
...(recoverySteerId != null && {
allowPreviouslyConvertedIds: [recoverySteerId],
}),
});
for (const steer of unrecovered) {
reconciledIds.add(steer.steerId);
if (steer.clientSteerId) {
reconciledIds.add(steer.clientSteerId);
}
}
}
if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) {
convertLocalSteersToQueued(recoveryConvoId, {
excludeSteerIds: reconciledIds,
generationProtocolVersion,
});
}
resetLive(sub.liveSubmission());
removeActiveJob(currentStreamId);
clearAttachedGenerationCreatedAt(terminal);
clearPendingApprovalForTerminal(terminal, recoveryConvoId);
if (
!run.createdStreamIds.has(currentStreamId) &&
run.optimisticStreamIds.has(currentStreamId)
) {
removeConvoFromAllQueries(queryClient, currentStreamId);
}
sub.retired = true;
setIsSubmitting(false);
setShowStopButton(false);
let recoveryOutcome: 'completed' | 'aborted' | 'error' = 'error';
if (status.status === 'complete' && persistedMessages != null) {
recoveryOutcome = 'completed';
} else if (status.status === 'aborted') {
recoveryOutcome = 'aborted';
}
const recoveredResponseMessageId =
recoveryOutcome === 'completed'
? completedResponseMessageId(
persistedMessages,
sub.userMessage?.messageId,
status?.resumeState?.responseMessageId ??
sub.submission.initialResponse?.messageId,
)
: undefined;
setRunEnd({
conversationId: recoveryConvoId,
outcome: recoveryOutcome,
endedAt: run.scheduler.now(),
generationCreatedAt: status.createdAt ?? generationCreatedAt,
...(recoveredResponseMessageId != null && {
responseMessageId: recoveredResponseMessageId,
}),
});
setSubmission(null);
setStreamId(null);
run.releaseStream(currentStreamId);
run.reconnectAttempt = 0;
}
};
/**
* The transport emits `abort` only for this hook's own closes. A cancel
* the user agent issued (a backgrounded or frozen mobile tab, while the
@ -2277,7 +1671,7 @@ export default function useResumableSSE(
sub.connect(transport.stream({ token }), {
onOpen: handleOpen,
onFrame: (event) => void handleFrame(event),
onFailure: (event) => void handleTransportFailure(event),
onFailure: (event) => void handleTransportFailure(terminal, event),
onClose: handleClose,
});
@ -2291,7 +1685,7 @@ export default function useResumableSSE(
/** Simulate network drop - triggers error event → reconnection */
debugWindow.__killNetwork = () => {
logger.log('Debug', 'Simulating network drop...');
void handleTransportFailure({});
void handleTransportFailure(terminal, {});
};
/** Simulate clean close (navigation away) - triggers abort event → no reconnection */
@ -2332,7 +1726,6 @@ export default function useResumableSSE(
startupConfig?.balance?.enabled,
balanceQuery,
removeActiveJob,
queryClient,
bindResponse,
contextHandler,
usageHandler,
@ -2351,7 +1744,6 @@ export default function useResumableSSE(
settleAppliedSteerParts,
convertSteersToQueued,
convertLocalSteersToQueued,
addActiveJob,
setSubmission,
updateActiveGenerationCreatedAt,
jotaiStore,

View file

@ -0,0 +1,218 @@
import type { TMessage, TSubmission } from 'librechat-data-provider';
import type {
ResumableHost,
TerminalContext,
ResumableScheduler,
StreamStatusResponse,
} from '../resumable';
import { ResumableRun, ResumableSubscription, handleTransportFailure } from '../resumable';
/** Timers that run only when the test advances the clock. */
const createScheduler = () => {
let now = 0;
const timers = new Map<number, { at: number; callback: () => void }>();
let nextId = 1;
const scheduler: ResumableScheduler = {
setTimeout: (callback, delay) => {
const id = nextId++;
timers.set(id, { at: now + delay, callback });
return id as unknown as ReturnType<typeof setTimeout>;
},
clearTimeout: (handle) => {
timers.delete(handle as unknown as number);
},
requestFrame: () => nextId++,
cancelFrame: () => undefined,
now: () => now,
};
const advance = (ms: number) => {
now += ms;
for (const [id, timer] of [...timers]) {
if (timer.at <= now) {
timers.delete(id);
timer.callback();
}
}
};
return { scheduler, advance };
};
const createHost = (fetchStatus: jest.Mock<Promise<StreamStatusResponse>, [string]>) => {
const host: ResumableHost = {
logger: { log: jest.fn(), warn: jest.fn(), error: jest.fn() },
visibility: { isVisible: () => true, subscribe: () => () => undefined },
fetchStatus,
subscribe: jest.fn(),
cache: {
invalidateMessages: jest.fn(async () => undefined),
fetchMessages: jest.fn(async () => [] as TMessage[]),
refreshConversation: jest.fn(async () => undefined),
refreshConversationLists: jest.fn(async () => undefined),
refreshPinnedConversations: jest.fn(async () => undefined),
setStreamStatus: jest.fn(),
removeStreamStatus: jest.fn(),
invalidateStreamStatus: jest.fn(),
addActiveJob: jest.fn(),
removeActiveJob: jest.fn(),
fetchConversation: jest.fn(async () => undefined),
setConversation: jest.fn(),
upsertConversation: jest.fn(),
removeConversation: jest.fn(),
},
pane: {
setIsSubmitting: jest.fn(),
setShowStopButton: jest.fn(),
setSubmission: jest.fn(),
setStreamId: jest.fn(),
setGenerationEpoch: jest.fn(() => true),
setPendingAction: jest.fn(),
clearComposerDrafts: jest.fn(),
publishRunEnd: jest.fn(),
clearDrainAfterAbort: jest.fn(),
refreshBalance: jest.fn(),
updateConversation: jest.fn(),
resolveDetachedEnd: jest.fn(() => ({ resolvable: false, end: null })),
},
steers: {
seed: jest.fn(),
appliedIds: jest.fn(() => []),
settleApplied: jest.fn(),
convertToQueued: jest.fn(),
convertLocalToQueued: jest.fn(),
},
events: {
clearStepMaps: jest.fn(),
flushPendingDeltas: jest.fn(),
resetLive: jest.fn(),
showError: jest.fn(),
},
messages: { get: jest.fn(() => []), set: jest.fn() },
};
return host;
};
const submission = {
userMessage: { messageId: 'u-1', conversationId: 'c-1', isCreatedByUser: true } as TMessage,
initialResponse: { messageId: 'u-1_', conversationId: 'c-1' } as TMessage,
conversation: { conversationId: 'c-1' },
} as TSubmission;
const setup = () => {
const clock = createScheduler();
const fetchStatus = jest.fn<Promise<StreamStatusResponse>, [string]>();
const host = createHost(fetchStatus);
const run = new ResumableRun(clock.scheduler);
run.submission = submission;
const sub = new ResumableSubscription(run, host, {
streamId: 'c-1',
submission,
isResume: false,
generationCreatedAt: 2000,
generationProtocolVersion: 2,
});
sub.claim();
const ctx: TerminalContext = { sub, host, applyPendingAction: jest.fn() };
return { clock, fetchStatus, host, run, sub, ctx };
};
describe('handleTransportFailure', () => {
it('ignores a failure after the final frame', async () => {
const { ctx, host } = setup();
ctx.sub.finalReceived = true;
await handleTransportFailure(ctx, { status: 0 });
expect(host.pane.refreshBalance).not.toHaveBeenCalled();
});
it('backs off and re-attaches a dropped connection', async () => {
const { ctx, host, run, clock } = setup();
await handleTransportFailure(ctx, { status: 0 });
expect(run.reconnectAttempt).toBe(1);
expect(host.pane.setIsSubmitting).toHaveBeenCalledWith(true);
expect(host.pane.setShowStopButton).toHaveBeenCalledWith(true);
clock.advance(999);
expect(host.subscribe).not.toHaveBeenCalled();
clock.advance(1);
expect(host.subscribe).toHaveBeenCalledWith(expect.objectContaining({ isResume: true }));
});
it('writes a served error once the status proves the run over', async () => {
const { ctx, host, fetchStatus } = setup();
fetchStatus.mockResolvedValue({ active: false, generationProtocolVersion: 2 });
await handleTransportFailure(ctx, {
data: { message: 'quota exceeded', generationProtocolVersion: 2 },
});
expect(ctx.sub.finalReceived).toBe(true);
expect(host.events.flushPendingDeltas).toHaveBeenCalled();
expect(host.events.showError).toHaveBeenCalledWith('quota exceeded', ctx.sub.submission);
expect(host.steers.convertLocalToQueued).toHaveBeenCalledWith('c-1', {
claimParked: true,
generationProtocolVersion: 2,
});
expect(host.pane.publishRunEnd).toHaveBeenCalledWith({
conversationId: 'c-1',
outcome: 'error',
endedAt: 0,
generationCreatedAt: 2000,
});
});
it('settles a missing stream from persisted history', async () => {
const { ctx, host, fetchStatus } = setup();
const persisted = [submission.userMessage];
(host.cache.fetchMessages as jest.Mock).mockResolvedValue(persisted);
fetchStatus.mockResolvedValue({ active: false, generationProtocolVersion: 2 });
await handleTransportFailure(ctx, { status: 404 });
expect(host.cache.fetchMessages).toHaveBeenCalledWith('c-1', true);
expect(host.cache.removeStreamStatus).toHaveBeenCalledWith('c-1');
expect(host.messages.set).toHaveBeenCalledWith(persisted);
expect(ctx.sub.retired).toBe(true);
expect(host.pane.publishRunEnd).toHaveBeenCalledWith(
expect.objectContaining({ conversationId: 'c-1', outcome: 'aborted' }),
{ retainDetached: false },
);
});
it('keeps a missing stream attached while its generation is still live', async () => {
const { ctx, host, fetchStatus, clock } = setup();
fetchStatus.mockResolvedValue({
active: true,
streamId: 'c-1',
createdAt: 2000,
generationProtocolVersion: 2,
});
await handleTransportFailure(ctx, { status: 404 });
expect(host.pane.publishRunEnd).not.toHaveBeenCalled();
expect(host.steers.seed).toHaveBeenCalledWith('c-1', [], 2000, 2);
clock.advance(1000);
expect(host.subscribe).toHaveBeenCalledWith(
expect.objectContaining({ streamId: 'c-1', generationCreatedAt: 2000 }),
);
});
it('waits on a live generation once the retry ceiling is reached', async () => {
const { ctx, host, run, fetchStatus } = setup();
run.reconnectAttempt = 5;
fetchStatus.mockResolvedValue({ active: true, createdAt: 2000, generationProtocolVersion: 2 });
await handleTransportFailure(ctx, { status: 0 });
expect(host.cache.setStreamStatus).toHaveBeenCalled();
expect(run.reconnectAttempt).toBe(0);
expect(host.pane.publishRunEnd).not.toHaveBeenCalled();
});
it('settles a run the status proves terminal past the retry ceiling', async () => {
const { ctx, host, run, fetchStatus } = setup();
run.reconnectAttempt = 5;
fetchStatus.mockResolvedValue({
active: false,
status: 'aborted',
createdAt: 2000,
generationProtocolVersion: 2,
});
await handleTransportFailure(ctx, { status: 0 });
expect(ctx.sub.retired).toBe(true);
expect(host.pane.setSubmission).toHaveBeenCalledWith(null);
expect(host.pane.publishRunEnd).toHaveBeenCalledWith(
expect.objectContaining({ outcome: 'aborted', generationCreatedAt: 2000 }),
);
});
});

View file

@ -60,6 +60,10 @@ const createHost = (fetchStatus: jest.Mock<Promise<StreamStatusResponse>, [strin
invalidateStreamStatus: jest.fn(),
addActiveJob: jest.fn(),
removeActiveJob: jest.fn(),
fetchConversation: jest.fn(async () => undefined),
setConversation: jest.fn(),
upsertConversation: jest.fn(),
removeConversation: jest.fn(),
},
pane: {
setIsSubmitting: jest.fn(),
@ -71,6 +75,9 @@ const createHost = (fetchStatus: jest.Mock<Promise<StreamStatusResponse>, [strin
clearComposerDrafts: jest.fn(),
publishRunEnd: jest.fn(),
clearDrainAfterAbort: jest.fn(),
refreshBalance: jest.fn(),
updateConversation: jest.fn(),
resolveDetachedEnd: jest.fn(() => ({ resolvable: false, end: null })),
},
steers: {
seed: jest.fn(),
@ -79,7 +86,13 @@ const createHost = (fetchStatus: jest.Mock<Promise<StreamStatusResponse>, [strin
convertToQueued: jest.fn(),
convertLocalToQueued: jest.fn(),
},
events: { clearStepMaps: jest.fn(), flushPendingDeltas: jest.fn(), resetLive: jest.fn() },
events: {
clearStepMaps: jest.fn(),
flushPendingDeltas: jest.fn(),
resetLive: jest.fn(),
showError: jest.fn(),
},
messages: { get: jest.fn(() => []), set: jest.fn() },
};
return host;
};

View file

@ -0,0 +1,190 @@
import type { TMessage } from 'librechat-data-provider';
import type { StreamStatusResponse } from './host';
import type { TerminalContext } from './terminal';
import {
restoreLiveStatus,
handoffToReplacement,
clearPendingApprovalForTerminal,
clearAttachedGenerationCreatedAt,
} from './terminal';
import { GENERATION_PROTOCOL_VERSION, supportsGenerationProtocolV2 } from '../protocol';
import { completedResponseMessageId } from './messages';
import { getRecoverySteerId } from './start';
/**
* The connection failed past the last retry. The transport gave up, not necessarily the
* generation: a status read decides whether to keep waiting on it (or hand it to a newer
* epoch), and only a status that proves it terminal settles the run from durable history.
*/
export async function settleAfterRetryCeiling(ctx: TerminalContext): Promise<void> {
const { sub, host } = ctx;
const { run, generationCreatedAt, generationProtocolVersion } = sub;
const currentStreamId = sub.streamId;
host.logger.error('ResumableSSE', 'Max reconnect attempts reached');
sub.close();
host.events.flushPendingDeltas();
const recoveryConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;
let status: StreamStatusResponse | undefined;
try {
status = await host.fetchStatus(recoveryConvoId);
if (!sub.isCurrent()) {
return;
}
} catch (error) {
if (!sub.isCurrent()) {
return;
}
host.logger.warn('ResumableSSE', 'Could not determine job state after reconnect limit', {
conversationId: recoveryConvoId,
error,
});
}
if (
status != null &&
generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
!supportsGenerationProtocolV2(status)
) {
// A legacy responder cannot adjudicate a v2 job during rollout.
status = undefined;
}
if (
status?.active === true &&
status.createdAt != null &&
(generationCreatedAt == null || status.createdAt !== generationCreatedAt)
) {
const handedOff = await handoffToReplacement(ctx, recoveryConvoId, status);
if (!sub.isCurrent()) {
return;
}
if (handedOff) {
return;
}
run.reconnectAttempt = 0;
sub.scheduleReattach(30_000);
return;
}
if (status?.active === true) {
// The transport gave up, not the generation. Keep the active-job
// marker and accepted chips intact, then release this stale
// submission so useResumeOnLoad can establish a fresh attachment.
host.cache.setStreamStatus(recoveryConvoId, status);
restoreLiveStatus(ctx, recoveryConvoId, status, status.createdAt, generationProtocolVersion);
host.pane.setIsSubmitting(true);
host.pane.setShowStopButton(generationCreatedAt != null);
run.reconnectAttempt = 0;
sub.scheduleReattach(30_000);
return;
}
if (status == null) {
// An inconclusive status read is still not proof of termination.
// Preserve server/client ownership and let the normal resume-on-load
// status query retry rather than duplicating accepted words.
host.cache.invalidateStreamStatus(recoveryConvoId);
host.pane.setIsSubmitting(true);
host.pane.setShowStopButton(generationCreatedAt != null);
run.reconnectAttempt = 0;
sub.scheduleReattach(30_000);
return;
}
// The status endpoint proved the job terminal. Reconcile persisted
// content before recovering leftovers, just like the 404 path.
run.cancelSteerRetryFrames();
let persistedMessages: TMessage[] | undefined;
try {
await host.cache.invalidateMessages(recoveryConvoId, 'none');
if (!sub.isCurrent()) {
return;
}
const fetched = await host.cache.fetchMessages(recoveryConvoId);
if (!sub.isCurrent()) {
return;
}
if (Array.isArray(fetched)) {
persistedMessages = fetched;
}
} catch (error) {
if (!sub.isCurrent()) {
return;
}
host.logger.warn('ResumableSSE', 'Could not reconcile terminal messages after disconnect', {
conversationId: recoveryConvoId,
error,
});
}
const authoritativeValues = [
...(persistedMessages ?? []),
...(status.resumeState?.aggregatedContent ?? status.aggregatedContent ?? []),
];
const confirmedV2Terminal =
generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
supportsGenerationProtocolV2(status);
const reconciledIds = new Set(host.steers.appliedIds(authoritativeValues));
host.steers.settleApplied(recoveryConvoId, authoritativeValues);
const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal;
const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : [];
if (unrecovered.length > 0) {
const recoverySteerId = getRecoverySteerId(sub.submission);
host.steers.convertToQueued(recoveryConvoId, unrecovered, {
generationProtocolVersion,
...(recoverySteerId != null && {
allowPreviouslyConvertedIds: [recoverySteerId],
}),
});
for (const steer of unrecovered) {
reconciledIds.add(steer.steerId);
if (steer.clientSteerId) {
reconciledIds.add(steer.clientSteerId);
}
}
}
if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) {
host.steers.convertLocalToQueued(recoveryConvoId, {
excludeSteerIds: reconciledIds,
generationProtocolVersion,
});
}
host.events.resetLive(sub.liveSubmission());
host.cache.removeActiveJob(currentStreamId);
clearAttachedGenerationCreatedAt(ctx);
clearPendingApprovalForTerminal(ctx, recoveryConvoId);
if (!run.createdStreamIds.has(currentStreamId) && run.optimisticStreamIds.has(currentStreamId)) {
host.cache.removeConversation(currentStreamId);
}
sub.retired = true;
host.pane.setIsSubmitting(false);
host.pane.setShowStopButton(false);
let recoveryOutcome: 'completed' | 'aborted' | 'error' = 'error';
if (status.status === 'complete' && persistedMessages != null) {
recoveryOutcome = 'completed';
} else if (status.status === 'aborted') {
recoveryOutcome = 'aborted';
}
const recoveredResponseMessageId =
recoveryOutcome === 'completed'
? completedResponseMessageId(
persistedMessages,
sub.userMessage?.messageId,
status?.resumeState?.responseMessageId ?? sub.submission.initialResponse?.messageId,
)
: undefined;
host.pane.publishRunEnd({
conversationId: recoveryConvoId,
outcome: recoveryOutcome,
endedAt: run.scheduler.now(),
generationCreatedAt: status.createdAt ?? generationCreatedAt,
...(recoveredResponseMessageId != null && {
responseMessageId: recoveredResponseMessageId,
}),
});
host.pane.setSubmission(null);
host.pane.setStreamId(null);
run.releaseStream(currentStreamId);
run.reconnectAttempt = 0;
}

View file

@ -0,0 +1,209 @@
import { ErrorTypes, ViolationTypes } from 'librechat-data-provider';
import type { ChatEvent, ChatErrorData } from 'librechat-data-provider';
import type { TerminalContext } from './terminal';
import {
handoffToReplacement,
authorizeTerminalTeardown,
clearPendingApprovalForTerminal,
clearAttachedGenerationCreatedAt,
} from './terminal';
import { GENERATION_PROTOCOL_VERSION, supportsGenerationProtocolV2 } from '../protocol';
import { MAX_RETRIES, getReconnectDelay, getRecoverySteerId } from './start';
import { settleAfterRetryCeiling } from './ceiling';
import { recoverMissingStream } from './missing';
/**
* Server-sent error event (event: error with data) - no responseCode.
* These are known errors (ErrorTypes, ViolationTypes) that should be displayed to user.
*/
async function handleServedError(
ctx: TerminalContext,
data: ChatErrorData | string,
): Promise<void> {
const { sub, host } = ctx;
const { run, generationCreatedAt, generationProtocolVersion } = sub;
const currentStreamId = sub.streamId;
sub.finalReceived = true;
const recoveryConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;
if (!(await authorizeTerminalTeardown(ctx, recoveryConvoId, 'error')) || !sub.isCurrent()) {
return;
}
host.logger.log('ResumableSSE', 'Server-sent error event received:', data);
run.cancelSteerRetryFrames();
sub.close();
/** FLUSH (not cancel): the error card below is built from the cache
* tail, so queued tokens must land first, and a stale trailing
* frame must never overwrite the error write. */
host.events.flushPendingDeltas();
host.cache.removeActiveJob(currentStreamId);
clearAttachedGenerationCreatedAt(ctx);
clearPendingApprovalForTerminal(ctx, recoveryConvoId);
host.events.resetLive(sub.liveSubmission());
if (!run.createdStreamIds.has(currentStreamId) && run.optimisticStreamIds.has(currentStreamId)) {
host.cache.removeConversation(currentStreamId);
}
const errorSupportsV2 = supportsGenerationProtocolV2(data);
const errorString =
typeof data === 'string' ? data : (data.error ?? data.message ?? JSON.stringify(data));
// Check if it's a known error type (ViolationTypes or ErrorTypes)
let isKnownError = false;
try {
const parsed = typeof errorString === 'string' ? JSON.parse(errorString) : errorString;
const errorType = parsed?.type ?? parsed?.code;
if (errorType) {
const violationValues = Object.values(ViolationTypes) as string[];
const errorTypeValues = Object.values(ErrorTypes) as string[];
isKnownError = violationValues.includes(errorType) || errorTypeValues.includes(errorType);
}
} catch {
// Not JSON or parsing failed - treat as generic error
}
host.logger.log('ResumableSSE', 'Error type check:', { isKnownError, errorString });
// Display the error to user via errorHandler
host.events.showError(errorString, sub.submission);
host.pane.setIsSubmitting(false);
host.pane.setShowStopButton(false);
const recoverySteerId = getRecoverySteerId(sub.submission);
if (
recoverySteerId != null &&
generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
errorSupportsV2
) {
try {
/** The error was received on the exact generation-fenced stream.
* Read status only now, after terminal was observed: a claim
* started before this recovery could return a stale parked copy
* after a successful commit and must never authorize redelivery. */
const terminalStatus = await host.fetchStatus(recoveryConvoId);
if (!sub.isCurrent()) {
return;
}
const releasedSource = terminalStatus.unrecoveredSteers?.find(
(steer) => steer.steerId === recoverySteerId,
);
if (
terminalStatus.active === false &&
supportsGenerationProtocolV2(terminalStatus) &&
releasedSource != null
) {
host.steers.convertToQueued(recoveryConvoId, [releasedSource], {
generationProtocolVersion,
allowPreviouslyConvertedIds: [recoverySteerId],
});
}
} catch (error) {
if (!sub.isCurrent()) {
return;
}
host.logger.warn('ResumableSSE', 'Could not recover source after generation error', {
conversationId: recoveryConvoId,
error,
});
}
}
// The error terminal's backstop parks acked leftovers server-side;
// claim it now so a reload can't resurrect the chips converted here.
if (generationProtocolVersion === 1 || errorSupportsV2) {
host.steers.convertLocalToQueued(recoveryConvoId, {
claimParked: true,
generationProtocolVersion,
});
}
host.pane.publishRunEnd({
conversationId: recoveryConvoId,
outcome: 'error',
endedAt: run.scheduler.now(),
generationCreatedAt,
});
host.pane.setStreamId(null);
run.releaseStream(currentStreamId);
run.reconnectAttempt = 0;
}
/**
* Error event handler - handles BOTH:
* 1. HTTP-level errors (status present) - 404, 409, network failures (0)
* 2. Server-sent error events (event: error with data) - known errors like ViolationTypes/ErrorTypes
*
* Order matters: check the status first since HTTP errors may also include data.
* The transport has already retried a 401 with a refreshed token.
*/
export async function handleTransportFailure(
ctx: TerminalContext,
{ status: responseCode, data }: Pick<Extract<ChatEvent, { type: 'error' }>, 'status' | 'data'>,
): Promise<void> {
const { sub, host } = ctx;
const { run, generationCreatedAt } = sub;
if (!sub.isCurrent()) {
return;
}
if (sub.finalReceived) {
host.logger.log('ResumableSSE', 'Ignoring error after FINAL event', {
responseCode,
hasData: data != null,
});
return;
}
host.pane.refreshBalance();
if (responseCode === 409) {
const handedOff = await handoffToReplacement(ctx, sub.conversationId());
if (!sub.isCurrent()) {
return;
}
if (handedOff) {
return;
}
}
// 404 → job completed & was cleaned up; messages are persisted in DB.
if (responseCode === 404) {
await recoverMissingStream(ctx);
return;
}
/**
* Only check the data if there's no HTTP status, since HTTP errors may also have body data.
* Note: responseCode === 0 means transport failure (connection dropped) - treat as network error,
* not a server-sent error payload. Use `== null` to only match undefined/null (no HTTP status).
*/
if (responseCode == null && data != null) {
await handleServedError(ctx, data);
return;
}
// Network failure or unknown HTTP error - attempt reconnection with backoff
host.logger.log('ResumableSSE', 'Stream error (network failure) - will attempt reconnect', {
responseCode,
hasData: data != null,
});
if (run.reconnectAttempt < MAX_RETRIES) {
// Increment counter BEFORE close() so abort handler knows we're reconnecting
run.reconnectAttempt++;
const delay = getReconnectDelay(run.reconnectAttempt);
host.logger.log(
'ResumableSSE',
`Reconnecting in ${delay}ms (attempt ${run.reconnectAttempt}/${MAX_RETRIES})`,
);
sub.close();
sub.scheduleReattach(delay);
// Keep UI in "submitting" state during reconnection attempts
// so user knows we're still trying (abort handler may have reset these)
host.pane.setIsSubmitting(true);
host.pane.setShowStopButton(generationCreatedAt != null);
} else {
await settleAfterRetryCeiling(ctx);
}
}

View file

@ -1,4 +1,10 @@
import type { Agents, TSubmission, TPendingSteer } from 'librechat-data-provider';
import type {
Agents,
TMessage,
TSubmission,
TConversation,
TPendingSteer,
} from 'librechat-data-provider';
import type { GenerationProtocolVersion } from '../protocol';
import type { RunEnd } from '../steer/types';
@ -139,6 +145,13 @@ export interface ResumableCache {
invalidateStreamStatus: (conversationId: string) => void;
addActiveJob: (streamId: string) => void;
removeActiveJob: (streamId: string) => void;
/** Reads the conversation's saved record; rejects with the HTTP failure, a 404 included. */
fetchConversation: (conversationId: string) => Promise<TConversation | undefined>;
setConversation: (conversationId: string, conversation: TConversation) => void;
/** Places the conversation's snapshot into every conversation list. */
upsertConversation: (conversation: TConversation) => void;
/** Drops the conversation from every conversation list. */
removeConversation: (conversationId: string) => void;
}
/** The pane's controls and run bookkeeping a resumable run drives. */
@ -164,6 +177,18 @@ export interface ResumablePane {
publishRunEnd: (end: RunEnd, options?: { retainDetached?: boolean }) => void;
/** Consumes an armed "interrupt & send" for exactly this generation. */
clearDrainAfterAbort: (conversationId: string, generationCreatedAt?: number) => void;
/** Refreshes the token balance, when the deployment tracks one. */
refreshBalance: () => void;
/** Updates the pane's conversation; absent for a pane that only adds a comparison response. */
updateConversation?: (update: (current: TConversation | null) => TConversation | null) => void;
/**
* The end of a run this pane left mid-stream and is now reattached to, as `messages` record
* it. `resolvable` is false when no such run is remembered or the user stopped it.
*/
resolveDetachedEnd: (
conversationId: string,
messages: TMessage[] | undefined,
) => { resolvable: boolean; end: RunEnd | null };
}
/** What the event handlers that render the stream expose to the run. */
@ -172,6 +197,14 @@ export interface ResumableEvents {
flushPendingDeltas: () => void;
/** Discards the live usage estimate for a run that stopped streaming. */
resetLive: (submission: TSubmission) => void;
/** Writes the failed turn's error card from the text (or raw error value) the server sent. */
showError: (text: string | boolean | { message?: string }, submission: TSubmission) => void;
}
/** The pane's message list. */
export interface ResumableMessages {
get: () => TMessage[] | undefined;
set: (messages: TMessage[]) => void;
}
/** Everything a resumable run needs from the app it runs in. */
@ -180,4 +213,5 @@ export interface ResumableHost extends SubscriptionHost {
pane: ResumablePane;
steers: ResumableSteers;
events: ResumableEvents;
messages: ResumableMessages;
}

View file

@ -8,3 +8,6 @@ export * from './host';
export * from './subscription';
export * from './terminal';
export * from './reconcile';
export * from './failure';
export * from './missing';
export * from './ceiling';

View file

@ -0,0 +1,242 @@
import { Constants } from 'librechat-data-provider';
import type { TMessage } from 'librechat-data-provider';
import type { TerminalContext } from './terminal';
import {
restoreLiveStatus,
handoffToReplacement,
clearPendingApprovalForTerminal,
clearAttachedGenerationCreatedAt,
} from './terminal';
import { GENERATION_PROTOCOL_VERSION, supportsGenerationProtocolV2 } from '../protocol';
import { isInitialNewConversation, keepLocalCodeApprovalMode } from './messages';
import { getRecoverySteerId, toStartGenerationError } from './start';
/**
* A 404 from the stream route: the job completed and was cleaned up, and its messages are
* persisted. Refetches and reconciles them before recovering local steers (the final response
* may contain an applied steer whose live event was missed), unless a status read shows the
* generation, or a newer one, still running.
*/
export async function recoverMissingStream(ctx: TerminalContext): Promise<void> {
const { sub, host } = ctx;
const { run, generationCreatedAt, generationProtocolVersion } = sub;
const currentStreamId = sub.streamId;
const convoId = sub.submission.conversation?.conversationId;
host.logger.log('ResumableSSE', 'Stream 404, invalidating messages for:', convoId);
// Keep the UI non-idle while the status read decides whether this is
// terminal cleanup or a handoff to a newer generation epoch.
run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);
run.cancelSteerRetryFrames();
sub.close();
/** Terminal: drop any in-flight live estimate so the gauge doesn't
* keep counting stale streamed output after the stream ends */
host.events.resetLive(sub.liveSubmission());
host.pane.clearComposerDrafts(convoId, {
includeNewChatDraft:
!convoId ||
convoId === Constants.NEW_CONVO ||
run.optimisticStreamIds.has(currentStreamId) ||
isInitialNewConversation(sub.submission),
});
host.events.clearStepMaps();
let persistedMessages: TMessage[] | undefined;
if (convoId) {
try {
await host.cache.invalidateMessages(convoId, 'none');
if (!sub.isCurrent()) {
return;
}
// The first stream can finish before CREATED mounts the saved-chat query.
const fetched = await host.cache.fetchMessages(convoId, true);
if (!sub.isCurrent()) {
return;
}
if (Array.isArray(fetched)) {
persistedMessages = fetched;
}
} catch (error) {
if (!sub.isCurrent()) {
return;
}
host.logger.warn('ResumableSSE', 'Could not reconcile persisted messages after 404', {
conversationId: convoId,
error,
});
}
host.cache.removeStreamStatus(convoId);
}
const recoveryConvoId = convoId ?? currentStreamId;
const reconciledIds = new Set(persistedMessages ? host.steers.appliedIds(persistedMessages) : []);
if (persistedMessages) {
host.steers.settleApplied(recoveryConvoId, persistedMessages);
}
// The status replay is owner-gated and non-destructive; creating a
// recovered follow-up consumes its exact source. Prefer these
// authoritative leftovers over guessing from local chips.
let confirmedV2Terminal = false;
try {
const status = await host.fetchStatus(recoveryConvoId);
if (!sub.isCurrent()) {
return;
}
if (
generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
!supportsGenerationProtocolV2(status)
) {
sub.scheduleReattach(1_000);
return;
}
if (
status.active === true &&
status.createdAt != null &&
(generationCreatedAt == null || status.createdAt !== generationCreatedAt)
) {
const handedOff = await handoffToReplacement(ctx, recoveryConvoId, status);
if (!sub.isCurrent()) {
return;
}
if (handedOff) {
return;
}
/** Legacy generations have no replacement-handoff contract.
* Never attach this submission to a different epoch. */
sub.scheduleReattach(1_000);
return;
}
if (status.active === true) {
/** A stream lookup can briefly lose its routing record while the
* authoritative job status still proves the same generation is
* alive. Preserve ownership and retry the attachment instead of
* publishing a terminal run-end (which could auto-drain queued
* text into a still-running generation). */
const activeStreamId = status.streamId ?? currentStreamId;
host.cache.setStreamStatus(recoveryConvoId, status);
if (status.createdAt != null) {
host.pane.setGenerationEpoch(
recoveryConvoId,
status.createdAt,
undefined,
generationProtocolVersion,
);
}
host.cache.addActiveJob(activeStreamId);
restoreLiveStatus(ctx, recoveryConvoId, status, status.createdAt, generationProtocolVersion);
host.pane.setIsSubmitting(true);
host.pane.setShowStopButton(status.createdAt != null);
run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);
sub.scheduleReattach(1_000, activeStreamId, status.createdAt);
return;
}
confirmedV2Terminal =
generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
supportsGenerationProtocolV2(status);
const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal;
const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : [];
if (unrecovered.length > 0) {
const recoverySteerId = getRecoverySteerId(sub.submission);
host.steers.convertToQueued(recoveryConvoId, unrecovered, {
generationProtocolVersion,
...(recoverySteerId != null && {
allowPreviouslyConvertedIds: [recoverySteerId],
}),
});
for (const steer of unrecovered) {
reconciledIds.add(steer.steerId);
if (steer.clientSteerId) {
reconciledIds.add(steer.clientSteerId);
}
}
}
} catch (error) {
if (!sub.isCurrent()) {
return;
}
host.logger.warn('ResumableSSE', 'Could not recover parked steers after 404', {
conversationId: recoveryConvoId,
error,
});
}
host.cache.removeActiveJob(currentStreamId);
clearAttachedGenerationCreatedAt(ctx);
clearPendingApprovalForTerminal(ctx, recoveryConvoId);
if (!run.createdStreamIds.has(currentStreamId) && run.optimisticStreamIds.has(currentStreamId)) {
// Both fresh and resumed subscriptions can miss every event of a fast turn.
// A missing job proves neither that the conversation was saved nor that it failed.
try {
const persisted = await host.cache.fetchConversation(recoveryConvoId);
if (!sub.isCurrent()) return;
if (persisted?.conversationId === recoveryConvoId) {
host.cache.setConversation(recoveryConvoId, persisted);
host.cache.upsertConversation(persisted);
host.pane.updateConversation?.((current) => {
if (
current?.conversationId != null &&
current.conversationId !== Constants.NEW_CONVO &&
current.conversationId !== recoveryConvoId
)
return current;
return keepLocalCodeApprovalMode(
{ ...current, ...persisted },
current,
current?.conversationId,
);
});
}
} catch (error) {
if (!sub.isCurrent()) return;
if (toStartGenerationError(error)?.response?.status === 404) {
host.cache.removeConversation(currentStreamId);
}
}
// An inconclusive read must not turn a saved chat into a phantom.
void host.cache.refreshConversationLists();
void host.cache.refreshPinnedConversations();
}
if (persistedMessages) {
host.messages.set(persistedMessages);
}
sub.retired = true;
host.pane.setIsSubmitting(false);
host.pane.setShowStopButton(false);
// Only infer recovery from local accepted chips after the persisted
// final was successfully inspected. If that fetch failed, their
// applied-vs-leftover outcome is unknown and converting could send
// already-applied words twice.
if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) {
host.steers.convertLocalToQueued(recoveryConvoId, {
excludeSteerIds: reconciledIds,
generationProtocolVersion,
});
}
// The true outcome is unknown here (job record already cleaned up):
// a non-'completed' outcome releases parked interrupt flags without
// auto-sending queued messages the user may not want fired. A run this
// pane left and came back to is the exception: the history just read
// names its response by epoch, which is the outcome it would have had.
// History read before the job finished may not hold that response yet, or
// may hold only the unfinished snapshot saved when the pane left, so a
// marker that read leaves unresolved or stopped outlives this end for the
// next visit, which reads history after the job is known to be done.
const detached = host.pane.resolveDetachedEnd(recoveryConvoId, persistedMessages);
const detachedResolvable = detached.resolvable;
const detachedEnd = detached.end;
host.pane.publishRunEnd(
{
conversationId: recoveryConvoId,
outcome: 'aborted',
endedAt: run.scheduler.now(),
...detachedEnd,
startedAsNewConvo: run.optimisticStreamIds.has(currentStreamId),
},
{
retainDetached:
detachedResolvable && (detachedEnd == null || detachedEnd.outcome === 'aborted'),
},
);
host.pane.setStreamId(null);
run.releaseStream(currentStreamId);
run.reconnectAttempt = 0;
}