mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-17 21:43:06 +00:00
🛟 refactor: Move Resumable Transport Failure Handling Into the Chat Core (#17120)
Some checks are pending
Backend Unit Tests / Build packages (push) Waiting to run
Backend Unit Tests / Codegraph select (push) Waiting to run
Backend Unit Tests / TypeScript type checks (push) Blocked by required conditions
Backend Unit Tests / Circular dependency checks (push) Waiting to run
Backend Unit Tests / Tests: api (shard 1/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 2/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 3/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: data-provider (push) Blocked by required conditions
Backend Unit Tests / Tests: data-schemas (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 1/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 2/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 3/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 4/4) (push) Blocked by required conditions
Codegraph E2E Votes / vote (full suite) (push) Waiting to run
Docker Dev Branch Images Build / publish (push) Waiting to run
Frontend Unit Tests / Client build recovery regression (push) Waiting to run
Frontend Unit Tests / Codegraph select (push) Waiting to run
Frontend Unit Tests / Build packages (push) Waiting to run
Frontend Unit Tests / TypeScript type checks (client) (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/chat (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/client (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 1/2) (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 2/2) (push) Blocked by required conditions
Frontend Unit Tests / Vite build verification (push) Blocked by required conditions
Sync Locize Translations & Create Translation PR / Sync Translation Keys with Locize (push) Waiting to run
Sync Locize Translations & Create Translation PR / Create Translation PR on Version Published (push) Blocked by required conditions
Some checks are pending
Backend Unit Tests / Build packages (push) Waiting to run
Backend Unit Tests / Codegraph select (push) Waiting to run
Backend Unit Tests / TypeScript type checks (push) Blocked by required conditions
Backend Unit Tests / Circular dependency checks (push) Waiting to run
Backend Unit Tests / Tests: api (shard 1/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 2/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 3/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: data-provider (push) Blocked by required conditions
Backend Unit Tests / Tests: data-schemas (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 1/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 2/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 3/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 4/4) (push) Blocked by required conditions
Codegraph E2E Votes / vote (full suite) (push) Waiting to run
Docker Dev Branch Images Build / publish (push) Waiting to run
Frontend Unit Tests / Client build recovery regression (push) Waiting to run
Frontend Unit Tests / Codegraph select (push) Waiting to run
Frontend Unit Tests / Build packages (push) Waiting to run
Frontend Unit Tests / TypeScript type checks (client) (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/chat (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/client (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 1/2) (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 2/2) (push) Blocked by required conditions
Frontend Unit Tests / Vite build verification (push) Blocked by required conditions
Sync Locize Translations & Create Translation PR / Sync Translation Keys with Locize (push) Waiting to run
Sync Locize Translations & Create Translation PR / Create Translation PR on Version Published (push) Blocked by required conditions
This commit is contained in:
parent
f43e7fa041
commit
d4e52acdfb
8 changed files with 952 additions and 651 deletions
|
|
@ -8,7 +8,6 @@ import { ABORT_SWEEP_STATUSES, selectLocalSteersForQueue } from '@librechat/chat
|
|||
import {
|
||||
Constants,
|
||||
QueryKeys,
|
||||
ErrorTypes,
|
||||
StepEvents,
|
||||
StepTypes,
|
||||
SteerEvents,
|
||||
|
|
@ -20,15 +19,12 @@ import {
|
|||
UsageEvents,
|
||||
createPayload,
|
||||
ApprovalEvents,
|
||||
ViolationTypes,
|
||||
removeNullishValues,
|
||||
} from 'librechat-data-provider';
|
||||
import {
|
||||
MAX_RETRIES,
|
||||
getStepEventId,
|
||||
toStartGenerationError,
|
||||
isOAuthStepEvent,
|
||||
getReconnectDelay,
|
||||
waitForRetryDelay,
|
||||
applyPendingAction,
|
||||
getRecoverySteerId,
|
||||
|
|
@ -48,7 +44,6 @@ import {
|
|||
keepLocalCodeApprovalMode,
|
||||
hydrateSubmissionMessages,
|
||||
buildResumeEventSubmission,
|
||||
completedResponseMessageId,
|
||||
getStartGenerationStreamId,
|
||||
buildOptimisticConversation,
|
||||
buildCreatedInitialResponse,
|
||||
|
|
@ -65,9 +60,9 @@ import {
|
|||
ResumableRun,
|
||||
documentVisibility,
|
||||
ResumableSubscription,
|
||||
handoffToReplacement,
|
||||
authorizeTerminalTeardown,
|
||||
reconcileGenerationLifecycle,
|
||||
handleTransportFailure,
|
||||
clearPendingApprovalForTerminal,
|
||||
clearAttachedGenerationCreatedAt,
|
||||
} from '@librechat/chat';
|
||||
|
|
@ -288,6 +283,16 @@ export default function useResumableSSE(
|
|||
queryClient.invalidateQueries({ queryKey: streamStatusQueryKey(conversationId) }),
|
||||
addActiveJob,
|
||||
removeActiveJob,
|
||||
fetchConversation: (conversationId) =>
|
||||
fetchConvoSnapshot(queryClient, conversationId, () =>
|
||||
dataService.getConversationById(conversationId),
|
||||
),
|
||||
setConversation: (conversationId, conversation) =>
|
||||
queryClient.setQueryData([QueryKeys.conversation, conversationId], conversation),
|
||||
upsertConversation: (conversation) =>
|
||||
upsertConvoInAllQueries(queryClient, conversation, true, 'snapshot'),
|
||||
removeConversation: (conversationId) =>
|
||||
removeConvoFromAllQueries(queryClient, conversationId),
|
||||
}),
|
||||
[addActiveJob, queryClient, removeActiveJob],
|
||||
);
|
||||
|
|
@ -487,6 +492,23 @@ export default function useResumableSSE(
|
|||
clearComposerDrafts(runIndex, conversationId, options),
|
||||
publishRunEnd: setRunEnd,
|
||||
clearDrainAfterAbort,
|
||||
refreshBalance: () => {
|
||||
(startupConfig?.balance?.enabled ?? false) && balanceQuery.refetch();
|
||||
},
|
||||
updateConversation: isAddedRequest ? undefined : setConversation,
|
||||
resolveDetachedEnd: (conversationId, messages) => {
|
||||
const detachedRun = jotaiStore.get(detachedRunByConvoId(conversationId));
|
||||
const resolvable =
|
||||
detachedRun != null &&
|
||||
!isDetachedRunManual(jotaiStore.get, conversationId, detachedRun);
|
||||
return {
|
||||
resolvable,
|
||||
end:
|
||||
resolvable && messages != null
|
||||
? resolveDetachedRunEnd(conversationId, detachedRun, messages)
|
||||
: null,
|
||||
};
|
||||
},
|
||||
},
|
||||
steers: {
|
||||
seed: seedSteerChips,
|
||||
|
|
@ -495,7 +517,17 @@ export default function useResumableSSE(
|
|||
convertToQueued: convertSteersToQueued,
|
||||
convertLocalToQueued: convertLocalSteersToQueued,
|
||||
},
|
||||
events: { clearStepMaps, flushPendingDeltas, resetLive },
|
||||
events: {
|
||||
clearStepMaps,
|
||||
flushPendingDeltas,
|
||||
resetLive,
|
||||
showError: (text, errorSubmission) =>
|
||||
errorHandler({
|
||||
data: { text } as unknown as Parameters<typeof errorHandler>[0]['data'],
|
||||
submission: errorSubmission as EventSubmission,
|
||||
}),
|
||||
},
|
||||
messages: { get: getMessages, set: setMessages },
|
||||
};
|
||||
const sub = new ResumableSubscription(run, host, target);
|
||||
const {
|
||||
|
|
@ -1587,644 +1619,6 @@ export default function useResumableSSE(
|
|||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Error event handler - handles BOTH:
|
||||
* 1. HTTP-level errors (status present) - 404, 409, network failures (0)
|
||||
* 2. Server-sent error events (event: error with data) - known errors like ViolationTypes/ErrorTypes
|
||||
*
|
||||
* Order matters: check the status first since HTTP errors may also include data.
|
||||
* The transport has already retried a 401 with a refreshed token.
|
||||
*/
|
||||
const handleTransportFailure = async ({
|
||||
status: responseCode,
|
||||
data,
|
||||
}: Pick<Extract<ChatEvent, { type: 'error' }>, 'status' | 'data'>) => {
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (sub.finalReceived) {
|
||||
logger.log('ResumableSSE', 'Ignoring error after FINAL event', {
|
||||
responseCode,
|
||||
hasData: data != null,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
(startupConfig?.balance?.enabled ?? false) && balanceQuery.refetch();
|
||||
|
||||
if (responseCode === 409) {
|
||||
const replacementConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;
|
||||
const handedOff = await handoffToReplacement(terminal, replacementConvoId);
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
if (handedOff) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// 404 → job completed & was cleaned up; messages are persisted in DB.
|
||||
// Refetch and reconcile them before recovering local steers: the final
|
||||
// response may contain an applied steer whose live event was missed.
|
||||
if (responseCode === 404) {
|
||||
const convoId = sub.submission.conversation?.conversationId;
|
||||
logger.log('ResumableSSE', 'Stream 404, invalidating messages for:', convoId);
|
||||
// Keep the UI non-idle while the status read decides whether this is
|
||||
// terminal cleanup or a handoff to a newer generation epoch.
|
||||
run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);
|
||||
run.cancelSteerRetryFrames();
|
||||
sub.close();
|
||||
/** Terminal: drop any in-flight live estimate so the gauge doesn't
|
||||
* keep counting stale streamed output after the stream ends */
|
||||
resetLive(sub.liveSubmission());
|
||||
clearComposerDrafts(runIndex, convoId, {
|
||||
includeNewChatDraft:
|
||||
!convoId ||
|
||||
convoId === Constants.NEW_CONVO ||
|
||||
run.optimisticStreamIds.has(currentStreamId) ||
|
||||
isInitialNewConversation(sub.submission),
|
||||
});
|
||||
clearStepMaps();
|
||||
let persistedMessages: TMessage[] | undefined;
|
||||
if (convoId) {
|
||||
try {
|
||||
const messageQueryKey = [QueryKeys.messages, convoId] as const;
|
||||
await queryClient.invalidateQueries({
|
||||
queryKey: messageQueryKey,
|
||||
refetchType: 'none',
|
||||
});
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
const fetched = await queryClient.fetchQuery<TMessage[]>({
|
||||
queryKey: messageQueryKey,
|
||||
// The first stream can finish before CREATED mounts the saved-chat query.
|
||||
queryFn: () => fetchConversationMessages(convoId),
|
||||
});
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
if (Array.isArray(fetched)) {
|
||||
persistedMessages = fetched;
|
||||
}
|
||||
} catch (error) {
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
logger.warn('ResumableSSE', 'Could not reconcile persisted messages after 404', {
|
||||
conversationId: convoId,
|
||||
error,
|
||||
});
|
||||
}
|
||||
queryClient.removeQueries({ queryKey: streamStatusQueryKey(convoId) });
|
||||
}
|
||||
const recoveryConvoId = convoId ?? currentStreamId;
|
||||
const reconciledIds = new Set(
|
||||
persistedMessages ? collectAppliedSteerIds(persistedMessages) : [],
|
||||
);
|
||||
if (persistedMessages) {
|
||||
settleAppliedSteerParts(recoveryConvoId, persistedMessages);
|
||||
}
|
||||
|
||||
// The status replay is owner-gated and non-destructive; creating a
|
||||
// recovered follow-up consumes its exact source. Prefer these
|
||||
// authoritative leftovers over guessing from local chips.
|
||||
let confirmedV2Terminal = false;
|
||||
try {
|
||||
const status = await fetchStreamStatus(recoveryConvoId);
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
|
||||
!supportsGenerationProtocolV2(status)
|
||||
) {
|
||||
sub.scheduleReattach(1_000);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
status.active === true &&
|
||||
status.createdAt != null &&
|
||||
(generationCreatedAt == null || status.createdAt !== generationCreatedAt)
|
||||
) {
|
||||
const handedOff = await handoffToReplacement(terminal, recoveryConvoId, status);
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
if (handedOff) {
|
||||
return;
|
||||
}
|
||||
/** Legacy generations have no replacement-handoff contract.
|
||||
* Never attach this submission to a different epoch. */
|
||||
sub.scheduleReattach(1_000);
|
||||
return;
|
||||
}
|
||||
if (status.active === true) {
|
||||
/** A stream lookup can briefly lose its routing record while the
|
||||
* authoritative job status still proves the same generation is
|
||||
* alive. Preserve ownership and retry the attachment instead of
|
||||
* publishing a terminal run-end (which could auto-drain queued
|
||||
* text into a still-running generation). */
|
||||
const activeStreamId = status.streamId ?? currentStreamId;
|
||||
queryClient.setQueryData(streamStatusQueryKey(recoveryConvoId), status);
|
||||
if (status.createdAt != null) {
|
||||
updateActiveGenerationCreatedAt(
|
||||
recoveryConvoId,
|
||||
status.createdAt,
|
||||
undefined,
|
||||
generationProtocolVersion,
|
||||
);
|
||||
}
|
||||
addActiveJob(activeStreamId);
|
||||
seedSteerChips(
|
||||
recoveryConvoId,
|
||||
(status.resumeState?.pendingSteers ?? []) as TPendingSteer[],
|
||||
status.createdAt,
|
||||
generationProtocolVersion,
|
||||
);
|
||||
settleAppliedSteerParts(
|
||||
recoveryConvoId,
|
||||
status.resumeState?.aggregatedContent ?? status.aggregatedContent,
|
||||
);
|
||||
if (status.pendingAction ?? status.resumeState?.pendingAction) {
|
||||
applyPendingActionToMessages(
|
||||
(status.pendingAction ??
|
||||
status.resumeState?.pendingAction) as Agents.PendingAction,
|
||||
);
|
||||
}
|
||||
setIsSubmitting(true);
|
||||
setShowStopButton(status.createdAt != null);
|
||||
run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);
|
||||
sub.scheduleReattach(1_000, activeStreamId, status.createdAt);
|
||||
return;
|
||||
}
|
||||
confirmedV2Terminal =
|
||||
generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
|
||||
supportsGenerationProtocolV2(status);
|
||||
const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal;
|
||||
const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : [];
|
||||
if (unrecovered.length > 0) {
|
||||
const recoverySteerId = getRecoverySteerId(sub.submission);
|
||||
convertSteersToQueued(recoveryConvoId, unrecovered, {
|
||||
generationProtocolVersion,
|
||||
...(recoverySteerId != null && {
|
||||
allowPreviouslyConvertedIds: [recoverySteerId],
|
||||
}),
|
||||
});
|
||||
for (const steer of unrecovered) {
|
||||
reconciledIds.add(steer.steerId);
|
||||
if (steer.clientSteerId) {
|
||||
reconciledIds.add(steer.clientSteerId);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
logger.warn('ResumableSSE', 'Could not recover parked steers after 404', {
|
||||
conversationId: recoveryConvoId,
|
||||
error,
|
||||
});
|
||||
}
|
||||
|
||||
removeActiveJob(currentStreamId);
|
||||
clearAttachedGenerationCreatedAt(terminal);
|
||||
clearPendingApprovalForTerminal(terminal, recoveryConvoId);
|
||||
if (
|
||||
!run.createdStreamIds.has(currentStreamId) &&
|
||||
run.optimisticStreamIds.has(currentStreamId)
|
||||
) {
|
||||
// Both fresh and resumed subscriptions can miss every event of a fast turn.
|
||||
// A missing job proves neither that the conversation was saved nor that it failed.
|
||||
try {
|
||||
const persisted = await fetchConvoSnapshot(queryClient, recoveryConvoId, () =>
|
||||
dataService.getConversationById(recoveryConvoId),
|
||||
);
|
||||
if (!sub.isCurrent()) return;
|
||||
if (persisted?.conversationId === recoveryConvoId) {
|
||||
queryClient.setQueryData([QueryKeys.conversation, recoveryConvoId], persisted);
|
||||
upsertConvoInAllQueries(queryClient, persisted, true, 'snapshot');
|
||||
if (!isAddedRequest) {
|
||||
setConversation?.((current) => {
|
||||
if (
|
||||
current?.conversationId != null &&
|
||||
current.conversationId !== Constants.NEW_CONVO &&
|
||||
current.conversationId !== recoveryConvoId
|
||||
)
|
||||
return current;
|
||||
return keepLocalCodeApprovalMode(
|
||||
{ ...current, ...persisted },
|
||||
current,
|
||||
current?.conversationId,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (!sub.isCurrent()) return;
|
||||
if (toStartGenerationError(error)?.response?.status === 404) {
|
||||
removeConvoFromAllQueries(queryClient, currentStreamId);
|
||||
}
|
||||
}
|
||||
// An inconclusive read must not turn a saved chat into a phantom.
|
||||
invalidateConversationLists(queryClient);
|
||||
queryClient.invalidateQueries({ queryKey: [QueryKeys.pinnedConversations] });
|
||||
}
|
||||
if (persistedMessages) {
|
||||
setMessages(persistedMessages);
|
||||
}
|
||||
sub.retired = true;
|
||||
setIsSubmitting(false);
|
||||
setShowStopButton(false);
|
||||
|
||||
// Only infer recovery from local accepted chips after the persisted
|
||||
// final was successfully inspected. If that fetch failed, their
|
||||
// applied-vs-leftover outcome is unknown and converting could send
|
||||
// already-applied words twice.
|
||||
if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) {
|
||||
convertLocalSteersToQueued(recoveryConvoId, {
|
||||
excludeSteerIds: reconciledIds,
|
||||
generationProtocolVersion,
|
||||
});
|
||||
}
|
||||
// The true outcome is unknown here (job record already cleaned up):
|
||||
// a non-'completed' outcome releases parked interrupt flags without
|
||||
// auto-sending queued messages the user may not want fired. A run this
|
||||
// pane left and came back to is the exception: the history just read
|
||||
// names its response by epoch, which is the outcome it would have had.
|
||||
// History read before the job finished may not hold that response yet, or
|
||||
// may hold only the unfinished snapshot saved when the pane left, so a
|
||||
// marker that read leaves unresolved or stopped outlives this end for the
|
||||
// next visit, which reads history after the job is known to be done.
|
||||
const detachedRun = jotaiStore.get(detachedRunByConvoId(recoveryConvoId));
|
||||
const detachedResolvable =
|
||||
detachedRun != null &&
|
||||
!isDetachedRunManual(jotaiStore.get, recoveryConvoId, detachedRun);
|
||||
const detachedEnd =
|
||||
detachedResolvable && persistedMessages != null
|
||||
? resolveDetachedRunEnd(recoveryConvoId, detachedRun, persistedMessages)
|
||||
: null;
|
||||
setRunEnd(
|
||||
{
|
||||
conversationId: recoveryConvoId,
|
||||
outcome: 'aborted',
|
||||
endedAt: run.scheduler.now(),
|
||||
...detachedEnd,
|
||||
startedAsNewConvo: run.optimisticStreamIds.has(currentStreamId),
|
||||
},
|
||||
{
|
||||
retainDetached:
|
||||
detachedResolvable && (detachedEnd == null || detachedEnd.outcome === 'aborted'),
|
||||
},
|
||||
);
|
||||
setStreamId(null);
|
||||
run.releaseStream(currentStreamId);
|
||||
run.reconnectAttempt = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
/**
|
||||
* Server-sent error event (event: error with data) - no responseCode.
|
||||
* These are known errors (ErrorTypes, ViolationTypes) that should be displayed to user.
|
||||
* Only check the data if there's no HTTP status, since HTTP errors may also have body data.
|
||||
* Note: responseCode === 0 means transport failure (connection dropped) - treat as network error,
|
||||
* not a server-sent error payload. Use `== null` to only match undefined/null (no HTTP status).
|
||||
*/
|
||||
if (responseCode == null && data != null) {
|
||||
sub.finalReceived = true;
|
||||
const recoveryConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;
|
||||
if (
|
||||
!(await authorizeTerminalTeardown(terminal, recoveryConvoId, 'error')) ||
|
||||
!sub.isCurrent()
|
||||
) {
|
||||
return;
|
||||
}
|
||||
logger.log('ResumableSSE', 'Server-sent error event received:', data);
|
||||
run.cancelSteerRetryFrames();
|
||||
sub.close();
|
||||
/** FLUSH (not cancel): the error card below is built from the cache
|
||||
* tail, so queued tokens must land first, and a stale trailing
|
||||
* frame must never overwrite the error write. */
|
||||
flushPendingDeltas();
|
||||
removeActiveJob(currentStreamId);
|
||||
clearAttachedGenerationCreatedAt(terminal);
|
||||
clearPendingApprovalForTerminal(terminal, recoveryConvoId);
|
||||
resetLive(sub.liveSubmission());
|
||||
if (
|
||||
!run.createdStreamIds.has(currentStreamId) &&
|
||||
run.optimisticStreamIds.has(currentStreamId)
|
||||
) {
|
||||
removeConvoFromAllQueries(queryClient, currentStreamId);
|
||||
}
|
||||
|
||||
const errorSupportsV2 = supportsGenerationProtocolV2(data);
|
||||
const errorString =
|
||||
typeof data === 'string' ? data : (data.error ?? data.message ?? JSON.stringify(data));
|
||||
|
||||
// Check if it's a known error type (ViolationTypes or ErrorTypes)
|
||||
let isKnownError = false;
|
||||
try {
|
||||
const parsed = typeof errorString === 'string' ? JSON.parse(errorString) : errorString;
|
||||
const errorType = parsed?.type ?? parsed?.code;
|
||||
if (errorType) {
|
||||
const violationValues = Object.values(ViolationTypes) as string[];
|
||||
const errorTypeValues = Object.values(ErrorTypes) as string[];
|
||||
isKnownError =
|
||||
violationValues.includes(errorType) || errorTypeValues.includes(errorType);
|
||||
}
|
||||
} catch {
|
||||
// Not JSON or parsing failed - treat as generic error
|
||||
}
|
||||
|
||||
logger.log('ResumableSSE', 'Error type check:', { isKnownError, errorString });
|
||||
|
||||
// Display the error to user via errorHandler
|
||||
errorHandler({
|
||||
data: { text: errorString } as unknown as Parameters<typeof errorHandler>[0]['data'],
|
||||
submission: sub.submission as EventSubmission,
|
||||
});
|
||||
|
||||
setIsSubmitting(false);
|
||||
setShowStopButton(false);
|
||||
const recoverySteerId = getRecoverySteerId(sub.submission);
|
||||
if (
|
||||
recoverySteerId != null &&
|
||||
generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
|
||||
errorSupportsV2
|
||||
) {
|
||||
try {
|
||||
/** The error was received on the exact generation-fenced stream.
|
||||
* Read status only now, after terminal was observed: a claim
|
||||
* started before this recovery could return a stale parked copy
|
||||
* after a successful commit and must never authorize redelivery. */
|
||||
const terminalStatus = await fetchStreamStatus(recoveryConvoId);
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
const releasedSource = terminalStatus.unrecoveredSteers?.find(
|
||||
(steer) => steer.steerId === recoverySteerId,
|
||||
);
|
||||
if (
|
||||
terminalStatus.active === false &&
|
||||
supportsGenerationProtocolV2(terminalStatus) &&
|
||||
releasedSource != null
|
||||
) {
|
||||
convertSteersToQueued(recoveryConvoId, [releasedSource], {
|
||||
generationProtocolVersion,
|
||||
allowPreviouslyConvertedIds: [recoverySteerId],
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
logger.warn('ResumableSSE', 'Could not recover source after generation error', {
|
||||
conversationId: recoveryConvoId,
|
||||
error,
|
||||
});
|
||||
}
|
||||
}
|
||||
// The error terminal's backstop parks acked leftovers server-side;
|
||||
// claim it now so a reload can't resurrect the chips converted here.
|
||||
if (generationProtocolVersion === 1 || errorSupportsV2) {
|
||||
convertLocalSteersToQueued(recoveryConvoId, {
|
||||
claimParked: true,
|
||||
generationProtocolVersion,
|
||||
});
|
||||
}
|
||||
setRunEnd({
|
||||
conversationId: recoveryConvoId,
|
||||
outcome: 'error',
|
||||
endedAt: run.scheduler.now(),
|
||||
generationCreatedAt,
|
||||
});
|
||||
setStreamId(null);
|
||||
run.releaseStream(currentStreamId);
|
||||
run.reconnectAttempt = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
// Network failure or unknown HTTP error - attempt reconnection with backoff
|
||||
logger.log('ResumableSSE', 'Stream error (network failure) - will attempt reconnect', {
|
||||
responseCode,
|
||||
hasData: data != null,
|
||||
});
|
||||
|
||||
if (run.reconnectAttempt < MAX_RETRIES) {
|
||||
// Increment counter BEFORE close() so abort handler knows we're reconnecting
|
||||
run.reconnectAttempt++;
|
||||
const delay = getReconnectDelay(run.reconnectAttempt);
|
||||
|
||||
logger.log(
|
||||
'ResumableSSE',
|
||||
`Reconnecting in ${delay}ms (attempt ${run.reconnectAttempt}/${MAX_RETRIES})`,
|
||||
);
|
||||
|
||||
sub.close();
|
||||
|
||||
sub.scheduleReattach(delay);
|
||||
|
||||
// Keep UI in "submitting" state during reconnection attempts
|
||||
// so user knows we're still trying (abort handler may have reset these)
|
||||
setIsSubmitting(true);
|
||||
setShowStopButton(generationCreatedAt != null);
|
||||
} else {
|
||||
logger.error('ResumableSSE', 'Max reconnect attempts reached');
|
||||
sub.close();
|
||||
flushPendingDeltas();
|
||||
const recoveryConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;
|
||||
let status: Awaited<ReturnType<typeof fetchStreamStatus>> | undefined;
|
||||
try {
|
||||
status = await fetchStreamStatus(recoveryConvoId);
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
logger.warn('ResumableSSE', 'Could not determine job state after reconnect limit', {
|
||||
conversationId: recoveryConvoId,
|
||||
error,
|
||||
});
|
||||
}
|
||||
|
||||
if (
|
||||
status != null &&
|
||||
generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
|
||||
!supportsGenerationProtocolV2(status)
|
||||
) {
|
||||
// A legacy responder cannot adjudicate a v2 job during rollout.
|
||||
status = undefined;
|
||||
}
|
||||
|
||||
if (
|
||||
status?.active === true &&
|
||||
status.createdAt != null &&
|
||||
(generationCreatedAt == null || status.createdAt !== generationCreatedAt)
|
||||
) {
|
||||
const handedOff = await handoffToReplacement(terminal, recoveryConvoId, status);
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
if (handedOff) {
|
||||
return;
|
||||
}
|
||||
run.reconnectAttempt = 0;
|
||||
sub.scheduleReattach(30_000);
|
||||
return;
|
||||
}
|
||||
|
||||
if (status?.active === true) {
|
||||
// The transport gave up, not the generation. Keep the active-job
|
||||
// marker and accepted chips intact, then release this stale
|
||||
// submission so useResumeOnLoad can establish a fresh attachment.
|
||||
queryClient.setQueryData(streamStatusQueryKey(recoveryConvoId), status);
|
||||
seedSteerChips(
|
||||
recoveryConvoId,
|
||||
(status.resumeState?.pendingSteers ?? []) as TPendingSteer[],
|
||||
status.createdAt,
|
||||
generationProtocolVersion,
|
||||
);
|
||||
settleAppliedSteerParts(
|
||||
recoveryConvoId,
|
||||
status.resumeState?.aggregatedContent ?? status.aggregatedContent,
|
||||
);
|
||||
if (status.pendingAction ?? status.resumeState?.pendingAction) {
|
||||
applyPendingActionToMessages(
|
||||
(status.pendingAction ?? status.resumeState?.pendingAction) as Agents.PendingAction,
|
||||
);
|
||||
}
|
||||
setIsSubmitting(true);
|
||||
setShowStopButton(generationCreatedAt != null);
|
||||
run.reconnectAttempt = 0;
|
||||
sub.scheduleReattach(30_000);
|
||||
return;
|
||||
}
|
||||
|
||||
if (status == null) {
|
||||
// An inconclusive status read is still not proof of termination.
|
||||
// Preserve server/client ownership and let the normal resume-on-load
|
||||
// status query retry rather than duplicating accepted words.
|
||||
queryClient.invalidateQueries({ queryKey: streamStatusQueryKey(recoveryConvoId) });
|
||||
setIsSubmitting(true);
|
||||
setShowStopButton(generationCreatedAt != null);
|
||||
run.reconnectAttempt = 0;
|
||||
sub.scheduleReattach(30_000);
|
||||
return;
|
||||
}
|
||||
|
||||
// The status endpoint proved the job terminal. Reconcile persisted
|
||||
// content before recovering leftovers, just like the 404 path.
|
||||
run.cancelSteerRetryFrames();
|
||||
let persistedMessages: TMessage[] | undefined;
|
||||
try {
|
||||
const messageQueryKey = [QueryKeys.messages, recoveryConvoId] as const;
|
||||
await queryClient.invalidateQueries({
|
||||
queryKey: messageQueryKey,
|
||||
refetchType: 'none',
|
||||
});
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
const fetched = await queryClient.fetchQuery<TMessage[]>({
|
||||
queryKey: messageQueryKey,
|
||||
});
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
if (Array.isArray(fetched)) {
|
||||
persistedMessages = fetched;
|
||||
}
|
||||
} catch (error) {
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
logger.warn('ResumableSSE', 'Could not reconcile terminal messages after disconnect', {
|
||||
conversationId: recoveryConvoId,
|
||||
error,
|
||||
});
|
||||
}
|
||||
|
||||
const authoritativeValues = [
|
||||
...(persistedMessages ?? []),
|
||||
...(status.resumeState?.aggregatedContent ?? status.aggregatedContent ?? []),
|
||||
];
|
||||
const confirmedV2Terminal =
|
||||
generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
|
||||
supportsGenerationProtocolV2(status);
|
||||
const reconciledIds = new Set(collectAppliedSteerIds(authoritativeValues));
|
||||
settleAppliedSteerParts(recoveryConvoId, authoritativeValues);
|
||||
const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal;
|
||||
const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : [];
|
||||
if (unrecovered.length > 0) {
|
||||
const recoverySteerId = getRecoverySteerId(sub.submission);
|
||||
convertSteersToQueued(recoveryConvoId, unrecovered, {
|
||||
generationProtocolVersion,
|
||||
...(recoverySteerId != null && {
|
||||
allowPreviouslyConvertedIds: [recoverySteerId],
|
||||
}),
|
||||
});
|
||||
for (const steer of unrecovered) {
|
||||
reconciledIds.add(steer.steerId);
|
||||
if (steer.clientSteerId) {
|
||||
reconciledIds.add(steer.clientSteerId);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) {
|
||||
convertLocalSteersToQueued(recoveryConvoId, {
|
||||
excludeSteerIds: reconciledIds,
|
||||
generationProtocolVersion,
|
||||
});
|
||||
}
|
||||
|
||||
resetLive(sub.liveSubmission());
|
||||
removeActiveJob(currentStreamId);
|
||||
clearAttachedGenerationCreatedAt(terminal);
|
||||
clearPendingApprovalForTerminal(terminal, recoveryConvoId);
|
||||
if (
|
||||
!run.createdStreamIds.has(currentStreamId) &&
|
||||
run.optimisticStreamIds.has(currentStreamId)
|
||||
) {
|
||||
removeConvoFromAllQueries(queryClient, currentStreamId);
|
||||
}
|
||||
sub.retired = true;
|
||||
setIsSubmitting(false);
|
||||
setShowStopButton(false);
|
||||
let recoveryOutcome: 'completed' | 'aborted' | 'error' = 'error';
|
||||
if (status.status === 'complete' && persistedMessages != null) {
|
||||
recoveryOutcome = 'completed';
|
||||
} else if (status.status === 'aborted') {
|
||||
recoveryOutcome = 'aborted';
|
||||
}
|
||||
const recoveredResponseMessageId =
|
||||
recoveryOutcome === 'completed'
|
||||
? completedResponseMessageId(
|
||||
persistedMessages,
|
||||
sub.userMessage?.messageId,
|
||||
status?.resumeState?.responseMessageId ??
|
||||
sub.submission.initialResponse?.messageId,
|
||||
)
|
||||
: undefined;
|
||||
setRunEnd({
|
||||
conversationId: recoveryConvoId,
|
||||
outcome: recoveryOutcome,
|
||||
endedAt: run.scheduler.now(),
|
||||
generationCreatedAt: status.createdAt ?? generationCreatedAt,
|
||||
...(recoveredResponseMessageId != null && {
|
||||
responseMessageId: recoveredResponseMessageId,
|
||||
}),
|
||||
});
|
||||
setSubmission(null);
|
||||
setStreamId(null);
|
||||
run.releaseStream(currentStreamId);
|
||||
run.reconnectAttempt = 0;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* The transport emits `abort` only for this hook's own closes. A cancel
|
||||
* the user agent issued (a backgrounded or frozen mobile tab, while the
|
||||
|
|
@ -2277,7 +1671,7 @@ export default function useResumableSSE(
|
|||
sub.connect(transport.stream({ token }), {
|
||||
onOpen: handleOpen,
|
||||
onFrame: (event) => void handleFrame(event),
|
||||
onFailure: (event) => void handleTransportFailure(event),
|
||||
onFailure: (event) => void handleTransportFailure(terminal, event),
|
||||
onClose: handleClose,
|
||||
});
|
||||
|
||||
|
|
@ -2291,7 +1685,7 @@ export default function useResumableSSE(
|
|||
/** Simulate network drop - triggers error event → reconnection */
|
||||
debugWindow.__killNetwork = () => {
|
||||
logger.log('Debug', 'Simulating network drop...');
|
||||
void handleTransportFailure({});
|
||||
void handleTransportFailure(terminal, {});
|
||||
};
|
||||
|
||||
/** Simulate clean close (navigation away) - triggers abort event → no reconnection */
|
||||
|
|
@ -2332,7 +1726,6 @@ export default function useResumableSSE(
|
|||
startupConfig?.balance?.enabled,
|
||||
balanceQuery,
|
||||
removeActiveJob,
|
||||
queryClient,
|
||||
bindResponse,
|
||||
contextHandler,
|
||||
usageHandler,
|
||||
|
|
@ -2351,7 +1744,6 @@ export default function useResumableSSE(
|
|||
settleAppliedSteerParts,
|
||||
convertSteersToQueued,
|
||||
convertLocalSteersToQueued,
|
||||
addActiveJob,
|
||||
setSubmission,
|
||||
updateActiveGenerationCreatedAt,
|
||||
jotaiStore,
|
||||
|
|
|
|||
218
packages/chat/src/core/__tests__/failure.spec.ts
Normal file
218
packages/chat/src/core/__tests__/failure.spec.ts
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
import type { TMessage, TSubmission } from 'librechat-data-provider';
|
||||
import type {
|
||||
ResumableHost,
|
||||
TerminalContext,
|
||||
ResumableScheduler,
|
||||
StreamStatusResponse,
|
||||
} from '../resumable';
|
||||
import { ResumableRun, ResumableSubscription, handleTransportFailure } from '../resumable';
|
||||
|
||||
/** Timers that run only when the test advances the clock. */
|
||||
const createScheduler = () => {
|
||||
let now = 0;
|
||||
const timers = new Map<number, { at: number; callback: () => void }>();
|
||||
let nextId = 1;
|
||||
const scheduler: ResumableScheduler = {
|
||||
setTimeout: (callback, delay) => {
|
||||
const id = nextId++;
|
||||
timers.set(id, { at: now + delay, callback });
|
||||
return id as unknown as ReturnType<typeof setTimeout>;
|
||||
},
|
||||
clearTimeout: (handle) => {
|
||||
timers.delete(handle as unknown as number);
|
||||
},
|
||||
requestFrame: () => nextId++,
|
||||
cancelFrame: () => undefined,
|
||||
now: () => now,
|
||||
};
|
||||
const advance = (ms: number) => {
|
||||
now += ms;
|
||||
for (const [id, timer] of [...timers]) {
|
||||
if (timer.at <= now) {
|
||||
timers.delete(id);
|
||||
timer.callback();
|
||||
}
|
||||
}
|
||||
};
|
||||
return { scheduler, advance };
|
||||
};
|
||||
|
||||
const createHost = (fetchStatus: jest.Mock<Promise<StreamStatusResponse>, [string]>) => {
|
||||
const host: ResumableHost = {
|
||||
logger: { log: jest.fn(), warn: jest.fn(), error: jest.fn() },
|
||||
visibility: { isVisible: () => true, subscribe: () => () => undefined },
|
||||
fetchStatus,
|
||||
subscribe: jest.fn(),
|
||||
cache: {
|
||||
invalidateMessages: jest.fn(async () => undefined),
|
||||
fetchMessages: jest.fn(async () => [] as TMessage[]),
|
||||
refreshConversation: jest.fn(async () => undefined),
|
||||
refreshConversationLists: jest.fn(async () => undefined),
|
||||
refreshPinnedConversations: jest.fn(async () => undefined),
|
||||
setStreamStatus: jest.fn(),
|
||||
removeStreamStatus: jest.fn(),
|
||||
invalidateStreamStatus: jest.fn(),
|
||||
addActiveJob: jest.fn(),
|
||||
removeActiveJob: jest.fn(),
|
||||
fetchConversation: jest.fn(async () => undefined),
|
||||
setConversation: jest.fn(),
|
||||
upsertConversation: jest.fn(),
|
||||
removeConversation: jest.fn(),
|
||||
},
|
||||
pane: {
|
||||
setIsSubmitting: jest.fn(),
|
||||
setShowStopButton: jest.fn(),
|
||||
setSubmission: jest.fn(),
|
||||
setStreamId: jest.fn(),
|
||||
setGenerationEpoch: jest.fn(() => true),
|
||||
setPendingAction: jest.fn(),
|
||||
clearComposerDrafts: jest.fn(),
|
||||
publishRunEnd: jest.fn(),
|
||||
clearDrainAfterAbort: jest.fn(),
|
||||
refreshBalance: jest.fn(),
|
||||
updateConversation: jest.fn(),
|
||||
resolveDetachedEnd: jest.fn(() => ({ resolvable: false, end: null })),
|
||||
},
|
||||
steers: {
|
||||
seed: jest.fn(),
|
||||
appliedIds: jest.fn(() => []),
|
||||
settleApplied: jest.fn(),
|
||||
convertToQueued: jest.fn(),
|
||||
convertLocalToQueued: jest.fn(),
|
||||
},
|
||||
events: {
|
||||
clearStepMaps: jest.fn(),
|
||||
flushPendingDeltas: jest.fn(),
|
||||
resetLive: jest.fn(),
|
||||
showError: jest.fn(),
|
||||
},
|
||||
messages: { get: jest.fn(() => []), set: jest.fn() },
|
||||
};
|
||||
return host;
|
||||
};
|
||||
|
||||
const submission = {
|
||||
userMessage: { messageId: 'u-1', conversationId: 'c-1', isCreatedByUser: true } as TMessage,
|
||||
initialResponse: { messageId: 'u-1_', conversationId: 'c-1' } as TMessage,
|
||||
conversation: { conversationId: 'c-1' },
|
||||
} as TSubmission;
|
||||
|
||||
const setup = () => {
|
||||
const clock = createScheduler();
|
||||
const fetchStatus = jest.fn<Promise<StreamStatusResponse>, [string]>();
|
||||
const host = createHost(fetchStatus);
|
||||
const run = new ResumableRun(clock.scheduler);
|
||||
run.submission = submission;
|
||||
const sub = new ResumableSubscription(run, host, {
|
||||
streamId: 'c-1',
|
||||
submission,
|
||||
isResume: false,
|
||||
generationCreatedAt: 2000,
|
||||
generationProtocolVersion: 2,
|
||||
});
|
||||
sub.claim();
|
||||
const ctx: TerminalContext = { sub, host, applyPendingAction: jest.fn() };
|
||||
return { clock, fetchStatus, host, run, sub, ctx };
|
||||
};
|
||||
|
||||
describe('handleTransportFailure', () => {
|
||||
it('ignores a failure after the final frame', async () => {
|
||||
const { ctx, host } = setup();
|
||||
ctx.sub.finalReceived = true;
|
||||
await handleTransportFailure(ctx, { status: 0 });
|
||||
expect(host.pane.refreshBalance).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('backs off and re-attaches a dropped connection', async () => {
|
||||
const { ctx, host, run, clock } = setup();
|
||||
await handleTransportFailure(ctx, { status: 0 });
|
||||
expect(run.reconnectAttempt).toBe(1);
|
||||
expect(host.pane.setIsSubmitting).toHaveBeenCalledWith(true);
|
||||
expect(host.pane.setShowStopButton).toHaveBeenCalledWith(true);
|
||||
clock.advance(999);
|
||||
expect(host.subscribe).not.toHaveBeenCalled();
|
||||
clock.advance(1);
|
||||
expect(host.subscribe).toHaveBeenCalledWith(expect.objectContaining({ isResume: true }));
|
||||
});
|
||||
|
||||
it('writes a served error once the status proves the run over', async () => {
|
||||
const { ctx, host, fetchStatus } = setup();
|
||||
fetchStatus.mockResolvedValue({ active: false, generationProtocolVersion: 2 });
|
||||
await handleTransportFailure(ctx, {
|
||||
data: { message: 'quota exceeded', generationProtocolVersion: 2 },
|
||||
});
|
||||
expect(ctx.sub.finalReceived).toBe(true);
|
||||
expect(host.events.flushPendingDeltas).toHaveBeenCalled();
|
||||
expect(host.events.showError).toHaveBeenCalledWith('quota exceeded', ctx.sub.submission);
|
||||
expect(host.steers.convertLocalToQueued).toHaveBeenCalledWith('c-1', {
|
||||
claimParked: true,
|
||||
generationProtocolVersion: 2,
|
||||
});
|
||||
expect(host.pane.publishRunEnd).toHaveBeenCalledWith({
|
||||
conversationId: 'c-1',
|
||||
outcome: 'error',
|
||||
endedAt: 0,
|
||||
generationCreatedAt: 2000,
|
||||
});
|
||||
});
|
||||
|
||||
it('settles a missing stream from persisted history', async () => {
|
||||
const { ctx, host, fetchStatus } = setup();
|
||||
const persisted = [submission.userMessage];
|
||||
(host.cache.fetchMessages as jest.Mock).mockResolvedValue(persisted);
|
||||
fetchStatus.mockResolvedValue({ active: false, generationProtocolVersion: 2 });
|
||||
await handleTransportFailure(ctx, { status: 404 });
|
||||
expect(host.cache.fetchMessages).toHaveBeenCalledWith('c-1', true);
|
||||
expect(host.cache.removeStreamStatus).toHaveBeenCalledWith('c-1');
|
||||
expect(host.messages.set).toHaveBeenCalledWith(persisted);
|
||||
expect(ctx.sub.retired).toBe(true);
|
||||
expect(host.pane.publishRunEnd).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ conversationId: 'c-1', outcome: 'aborted' }),
|
||||
{ retainDetached: false },
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps a missing stream attached while its generation is still live', async () => {
|
||||
const { ctx, host, fetchStatus, clock } = setup();
|
||||
fetchStatus.mockResolvedValue({
|
||||
active: true,
|
||||
streamId: 'c-1',
|
||||
createdAt: 2000,
|
||||
generationProtocolVersion: 2,
|
||||
});
|
||||
await handleTransportFailure(ctx, { status: 404 });
|
||||
expect(host.pane.publishRunEnd).not.toHaveBeenCalled();
|
||||
expect(host.steers.seed).toHaveBeenCalledWith('c-1', [], 2000, 2);
|
||||
clock.advance(1000);
|
||||
expect(host.subscribe).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ streamId: 'c-1', generationCreatedAt: 2000 }),
|
||||
);
|
||||
});
|
||||
|
||||
it('waits on a live generation once the retry ceiling is reached', async () => {
|
||||
const { ctx, host, run, fetchStatus } = setup();
|
||||
run.reconnectAttempt = 5;
|
||||
fetchStatus.mockResolvedValue({ active: true, createdAt: 2000, generationProtocolVersion: 2 });
|
||||
await handleTransportFailure(ctx, { status: 0 });
|
||||
expect(host.cache.setStreamStatus).toHaveBeenCalled();
|
||||
expect(run.reconnectAttempt).toBe(0);
|
||||
expect(host.pane.publishRunEnd).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('settles a run the status proves terminal past the retry ceiling', async () => {
|
||||
const { ctx, host, run, fetchStatus } = setup();
|
||||
run.reconnectAttempt = 5;
|
||||
fetchStatus.mockResolvedValue({
|
||||
active: false,
|
||||
status: 'aborted',
|
||||
createdAt: 2000,
|
||||
generationProtocolVersion: 2,
|
||||
});
|
||||
await handleTransportFailure(ctx, { status: 0 });
|
||||
expect(ctx.sub.retired).toBe(true);
|
||||
expect(host.pane.setSubmission).toHaveBeenCalledWith(null);
|
||||
expect(host.pane.publishRunEnd).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ outcome: 'aborted', generationCreatedAt: 2000 }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
@ -60,6 +60,10 @@ const createHost = (fetchStatus: jest.Mock<Promise<StreamStatusResponse>, [strin
|
|||
invalidateStreamStatus: jest.fn(),
|
||||
addActiveJob: jest.fn(),
|
||||
removeActiveJob: jest.fn(),
|
||||
fetchConversation: jest.fn(async () => undefined),
|
||||
setConversation: jest.fn(),
|
||||
upsertConversation: jest.fn(),
|
||||
removeConversation: jest.fn(),
|
||||
},
|
||||
pane: {
|
||||
setIsSubmitting: jest.fn(),
|
||||
|
|
@ -71,6 +75,9 @@ const createHost = (fetchStatus: jest.Mock<Promise<StreamStatusResponse>, [strin
|
|||
clearComposerDrafts: jest.fn(),
|
||||
publishRunEnd: jest.fn(),
|
||||
clearDrainAfterAbort: jest.fn(),
|
||||
refreshBalance: jest.fn(),
|
||||
updateConversation: jest.fn(),
|
||||
resolveDetachedEnd: jest.fn(() => ({ resolvable: false, end: null })),
|
||||
},
|
||||
steers: {
|
||||
seed: jest.fn(),
|
||||
|
|
@ -79,7 +86,13 @@ const createHost = (fetchStatus: jest.Mock<Promise<StreamStatusResponse>, [strin
|
|||
convertToQueued: jest.fn(),
|
||||
convertLocalToQueued: jest.fn(),
|
||||
},
|
||||
events: { clearStepMaps: jest.fn(), flushPendingDeltas: jest.fn(), resetLive: jest.fn() },
|
||||
events: {
|
||||
clearStepMaps: jest.fn(),
|
||||
flushPendingDeltas: jest.fn(),
|
||||
resetLive: jest.fn(),
|
||||
showError: jest.fn(),
|
||||
},
|
||||
messages: { get: jest.fn(() => []), set: jest.fn() },
|
||||
};
|
||||
return host;
|
||||
};
|
||||
|
|
|
|||
190
packages/chat/src/core/resumable/ceiling.ts
Normal file
190
packages/chat/src/core/resumable/ceiling.ts
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
import type { TMessage } from 'librechat-data-provider';
|
||||
import type { StreamStatusResponse } from './host';
|
||||
import type { TerminalContext } from './terminal';
|
||||
import {
|
||||
restoreLiveStatus,
|
||||
handoffToReplacement,
|
||||
clearPendingApprovalForTerminal,
|
||||
clearAttachedGenerationCreatedAt,
|
||||
} from './terminal';
|
||||
import { GENERATION_PROTOCOL_VERSION, supportsGenerationProtocolV2 } from '../protocol';
|
||||
import { completedResponseMessageId } from './messages';
|
||||
import { getRecoverySteerId } from './start';
|
||||
|
||||
/**
|
||||
* The connection failed past the last retry. The transport gave up, not necessarily the
|
||||
* generation: a status read decides whether to keep waiting on it (or hand it to a newer
|
||||
* epoch), and only a status that proves it terminal settles the run from durable history.
|
||||
*/
|
||||
export async function settleAfterRetryCeiling(ctx: TerminalContext): Promise<void> {
|
||||
const { sub, host } = ctx;
|
||||
const { run, generationCreatedAt, generationProtocolVersion } = sub;
|
||||
const currentStreamId = sub.streamId;
|
||||
host.logger.error('ResumableSSE', 'Max reconnect attempts reached');
|
||||
sub.close();
|
||||
host.events.flushPendingDeltas();
|
||||
const recoveryConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;
|
||||
let status: StreamStatusResponse | undefined;
|
||||
try {
|
||||
status = await host.fetchStatus(recoveryConvoId);
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
host.logger.warn('ResumableSSE', 'Could not determine job state after reconnect limit', {
|
||||
conversationId: recoveryConvoId,
|
||||
error,
|
||||
});
|
||||
}
|
||||
|
||||
if (
|
||||
status != null &&
|
||||
generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
|
||||
!supportsGenerationProtocolV2(status)
|
||||
) {
|
||||
// A legacy responder cannot adjudicate a v2 job during rollout.
|
||||
status = undefined;
|
||||
}
|
||||
|
||||
if (
|
||||
status?.active === true &&
|
||||
status.createdAt != null &&
|
||||
(generationCreatedAt == null || status.createdAt !== generationCreatedAt)
|
||||
) {
|
||||
const handedOff = await handoffToReplacement(ctx, recoveryConvoId, status);
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
if (handedOff) {
|
||||
return;
|
||||
}
|
||||
run.reconnectAttempt = 0;
|
||||
sub.scheduleReattach(30_000);
|
||||
return;
|
||||
}
|
||||
|
||||
if (status?.active === true) {
|
||||
// The transport gave up, not the generation. Keep the active-job
|
||||
// marker and accepted chips intact, then release this stale
|
||||
// submission so useResumeOnLoad can establish a fresh attachment.
|
||||
host.cache.setStreamStatus(recoveryConvoId, status);
|
||||
restoreLiveStatus(ctx, recoveryConvoId, status, status.createdAt, generationProtocolVersion);
|
||||
host.pane.setIsSubmitting(true);
|
||||
host.pane.setShowStopButton(generationCreatedAt != null);
|
||||
run.reconnectAttempt = 0;
|
||||
sub.scheduleReattach(30_000);
|
||||
return;
|
||||
}
|
||||
|
||||
if (status == null) {
|
||||
// An inconclusive status read is still not proof of termination.
|
||||
// Preserve server/client ownership and let the normal resume-on-load
|
||||
// status query retry rather than duplicating accepted words.
|
||||
host.cache.invalidateStreamStatus(recoveryConvoId);
|
||||
host.pane.setIsSubmitting(true);
|
||||
host.pane.setShowStopButton(generationCreatedAt != null);
|
||||
run.reconnectAttempt = 0;
|
||||
sub.scheduleReattach(30_000);
|
||||
return;
|
||||
}
|
||||
|
||||
// The status endpoint proved the job terminal. Reconcile persisted
|
||||
// content before recovering leftovers, just like the 404 path.
|
||||
run.cancelSteerRetryFrames();
|
||||
let persistedMessages: TMessage[] | undefined;
|
||||
try {
|
||||
await host.cache.invalidateMessages(recoveryConvoId, 'none');
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
const fetched = await host.cache.fetchMessages(recoveryConvoId);
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
if (Array.isArray(fetched)) {
|
||||
persistedMessages = fetched;
|
||||
}
|
||||
} catch (error) {
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
host.logger.warn('ResumableSSE', 'Could not reconcile terminal messages after disconnect', {
|
||||
conversationId: recoveryConvoId,
|
||||
error,
|
||||
});
|
||||
}
|
||||
|
||||
const authoritativeValues = [
|
||||
...(persistedMessages ?? []),
|
||||
...(status.resumeState?.aggregatedContent ?? status.aggregatedContent ?? []),
|
||||
];
|
||||
const confirmedV2Terminal =
|
||||
generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
|
||||
supportsGenerationProtocolV2(status);
|
||||
const reconciledIds = new Set(host.steers.appliedIds(authoritativeValues));
|
||||
host.steers.settleApplied(recoveryConvoId, authoritativeValues);
|
||||
const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal;
|
||||
const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : [];
|
||||
if (unrecovered.length > 0) {
|
||||
const recoverySteerId = getRecoverySteerId(sub.submission);
|
||||
host.steers.convertToQueued(recoveryConvoId, unrecovered, {
|
||||
generationProtocolVersion,
|
||||
...(recoverySteerId != null && {
|
||||
allowPreviouslyConvertedIds: [recoverySteerId],
|
||||
}),
|
||||
});
|
||||
for (const steer of unrecovered) {
|
||||
reconciledIds.add(steer.steerId);
|
||||
if (steer.clientSteerId) {
|
||||
reconciledIds.add(steer.clientSteerId);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) {
|
||||
host.steers.convertLocalToQueued(recoveryConvoId, {
|
||||
excludeSteerIds: reconciledIds,
|
||||
generationProtocolVersion,
|
||||
});
|
||||
}
|
||||
|
||||
host.events.resetLive(sub.liveSubmission());
|
||||
host.cache.removeActiveJob(currentStreamId);
|
||||
clearAttachedGenerationCreatedAt(ctx);
|
||||
clearPendingApprovalForTerminal(ctx, recoveryConvoId);
|
||||
if (!run.createdStreamIds.has(currentStreamId) && run.optimisticStreamIds.has(currentStreamId)) {
|
||||
host.cache.removeConversation(currentStreamId);
|
||||
}
|
||||
sub.retired = true;
|
||||
host.pane.setIsSubmitting(false);
|
||||
host.pane.setShowStopButton(false);
|
||||
let recoveryOutcome: 'completed' | 'aborted' | 'error' = 'error';
|
||||
if (status.status === 'complete' && persistedMessages != null) {
|
||||
recoveryOutcome = 'completed';
|
||||
} else if (status.status === 'aborted') {
|
||||
recoveryOutcome = 'aborted';
|
||||
}
|
||||
const recoveredResponseMessageId =
|
||||
recoveryOutcome === 'completed'
|
||||
? completedResponseMessageId(
|
||||
persistedMessages,
|
||||
sub.userMessage?.messageId,
|
||||
status?.resumeState?.responseMessageId ?? sub.submission.initialResponse?.messageId,
|
||||
)
|
||||
: undefined;
|
||||
host.pane.publishRunEnd({
|
||||
conversationId: recoveryConvoId,
|
||||
outcome: recoveryOutcome,
|
||||
endedAt: run.scheduler.now(),
|
||||
generationCreatedAt: status.createdAt ?? generationCreatedAt,
|
||||
...(recoveredResponseMessageId != null && {
|
||||
responseMessageId: recoveredResponseMessageId,
|
||||
}),
|
||||
});
|
||||
host.pane.setSubmission(null);
|
||||
host.pane.setStreamId(null);
|
||||
run.releaseStream(currentStreamId);
|
||||
run.reconnectAttempt = 0;
|
||||
}
|
||||
209
packages/chat/src/core/resumable/failure.ts
Normal file
209
packages/chat/src/core/resumable/failure.ts
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
import { ErrorTypes, ViolationTypes } from 'librechat-data-provider';
|
||||
import type { ChatEvent, ChatErrorData } from 'librechat-data-provider';
|
||||
import type { TerminalContext } from './terminal';
|
||||
import {
|
||||
handoffToReplacement,
|
||||
authorizeTerminalTeardown,
|
||||
clearPendingApprovalForTerminal,
|
||||
clearAttachedGenerationCreatedAt,
|
||||
} from './terminal';
|
||||
import { GENERATION_PROTOCOL_VERSION, supportsGenerationProtocolV2 } from '../protocol';
|
||||
import { MAX_RETRIES, getReconnectDelay, getRecoverySteerId } from './start';
|
||||
import { settleAfterRetryCeiling } from './ceiling';
|
||||
import { recoverMissingStream } from './missing';
|
||||
|
||||
/**
|
||||
* Server-sent error event (event: error with data) - no responseCode.
|
||||
* These are known errors (ErrorTypes, ViolationTypes) that should be displayed to user.
|
||||
*/
|
||||
async function handleServedError(
|
||||
ctx: TerminalContext,
|
||||
data: ChatErrorData | string,
|
||||
): Promise<void> {
|
||||
const { sub, host } = ctx;
|
||||
const { run, generationCreatedAt, generationProtocolVersion } = sub;
|
||||
const currentStreamId = sub.streamId;
|
||||
sub.finalReceived = true;
|
||||
const recoveryConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;
|
||||
if (!(await authorizeTerminalTeardown(ctx, recoveryConvoId, 'error')) || !sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
host.logger.log('ResumableSSE', 'Server-sent error event received:', data);
|
||||
run.cancelSteerRetryFrames();
|
||||
sub.close();
|
||||
/** FLUSH (not cancel): the error card below is built from the cache
|
||||
* tail, so queued tokens must land first, and a stale trailing
|
||||
* frame must never overwrite the error write. */
|
||||
host.events.flushPendingDeltas();
|
||||
host.cache.removeActiveJob(currentStreamId);
|
||||
clearAttachedGenerationCreatedAt(ctx);
|
||||
clearPendingApprovalForTerminal(ctx, recoveryConvoId);
|
||||
host.events.resetLive(sub.liveSubmission());
|
||||
if (!run.createdStreamIds.has(currentStreamId) && run.optimisticStreamIds.has(currentStreamId)) {
|
||||
host.cache.removeConversation(currentStreamId);
|
||||
}
|
||||
|
||||
const errorSupportsV2 = supportsGenerationProtocolV2(data);
|
||||
const errorString =
|
||||
typeof data === 'string' ? data : (data.error ?? data.message ?? JSON.stringify(data));
|
||||
|
||||
// Check if it's a known error type (ViolationTypes or ErrorTypes)
|
||||
let isKnownError = false;
|
||||
try {
|
||||
const parsed = typeof errorString === 'string' ? JSON.parse(errorString) : errorString;
|
||||
const errorType = parsed?.type ?? parsed?.code;
|
||||
if (errorType) {
|
||||
const violationValues = Object.values(ViolationTypes) as string[];
|
||||
const errorTypeValues = Object.values(ErrorTypes) as string[];
|
||||
isKnownError = violationValues.includes(errorType) || errorTypeValues.includes(errorType);
|
||||
}
|
||||
} catch {
|
||||
// Not JSON or parsing failed - treat as generic error
|
||||
}
|
||||
|
||||
host.logger.log('ResumableSSE', 'Error type check:', { isKnownError, errorString });
|
||||
|
||||
// Display the error to user via errorHandler
|
||||
host.events.showError(errorString, sub.submission);
|
||||
|
||||
host.pane.setIsSubmitting(false);
|
||||
host.pane.setShowStopButton(false);
|
||||
const recoverySteerId = getRecoverySteerId(sub.submission);
|
||||
if (
|
||||
recoverySteerId != null &&
|
||||
generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
|
||||
errorSupportsV2
|
||||
) {
|
||||
try {
|
||||
/** The error was received on the exact generation-fenced stream.
|
||||
* Read status only now, after terminal was observed: a claim
|
||||
* started before this recovery could return a stale parked copy
|
||||
* after a successful commit and must never authorize redelivery. */
|
||||
const terminalStatus = await host.fetchStatus(recoveryConvoId);
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
const releasedSource = terminalStatus.unrecoveredSteers?.find(
|
||||
(steer) => steer.steerId === recoverySteerId,
|
||||
);
|
||||
if (
|
||||
terminalStatus.active === false &&
|
||||
supportsGenerationProtocolV2(terminalStatus) &&
|
||||
releasedSource != null
|
||||
) {
|
||||
host.steers.convertToQueued(recoveryConvoId, [releasedSource], {
|
||||
generationProtocolVersion,
|
||||
allowPreviouslyConvertedIds: [recoverySteerId],
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
host.logger.warn('ResumableSSE', 'Could not recover source after generation error', {
|
||||
conversationId: recoveryConvoId,
|
||||
error,
|
||||
});
|
||||
}
|
||||
}
|
||||
// The error terminal's backstop parks acked leftovers server-side;
|
||||
// claim it now so a reload can't resurrect the chips converted here.
|
||||
if (generationProtocolVersion === 1 || errorSupportsV2) {
|
||||
host.steers.convertLocalToQueued(recoveryConvoId, {
|
||||
claimParked: true,
|
||||
generationProtocolVersion,
|
||||
});
|
||||
}
|
||||
host.pane.publishRunEnd({
|
||||
conversationId: recoveryConvoId,
|
||||
outcome: 'error',
|
||||
endedAt: run.scheduler.now(),
|
||||
generationCreatedAt,
|
||||
});
|
||||
host.pane.setStreamId(null);
|
||||
run.releaseStream(currentStreamId);
|
||||
run.reconnectAttempt = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Error event handler - handles BOTH:
|
||||
* 1. HTTP-level errors (status present) - 404, 409, network failures (0)
|
||||
* 2. Server-sent error events (event: error with data) - known errors like ViolationTypes/ErrorTypes
|
||||
*
|
||||
* Order matters: check the status first since HTTP errors may also include data.
|
||||
* The transport has already retried a 401 with a refreshed token.
|
||||
*/
|
||||
export async function handleTransportFailure(
|
||||
ctx: TerminalContext,
|
||||
{ status: responseCode, data }: Pick<Extract<ChatEvent, { type: 'error' }>, 'status' | 'data'>,
|
||||
): Promise<void> {
|
||||
const { sub, host } = ctx;
|
||||
const { run, generationCreatedAt } = sub;
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (sub.finalReceived) {
|
||||
host.logger.log('ResumableSSE', 'Ignoring error after FINAL event', {
|
||||
responseCode,
|
||||
hasData: data != null,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
host.pane.refreshBalance();
|
||||
|
||||
if (responseCode === 409) {
|
||||
const handedOff = await handoffToReplacement(ctx, sub.conversationId());
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
if (handedOff) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// 404 → job completed & was cleaned up; messages are persisted in DB.
|
||||
if (responseCode === 404) {
|
||||
await recoverMissingStream(ctx);
|
||||
return;
|
||||
}
|
||||
|
||||
/**
|
||||
* Only check the data if there's no HTTP status, since HTTP errors may also have body data.
|
||||
* Note: responseCode === 0 means transport failure (connection dropped) - treat as network error,
|
||||
* not a server-sent error payload. Use `== null` to only match undefined/null (no HTTP status).
|
||||
*/
|
||||
if (responseCode == null && data != null) {
|
||||
await handleServedError(ctx, data);
|
||||
return;
|
||||
}
|
||||
|
||||
// Network failure or unknown HTTP error - attempt reconnection with backoff
|
||||
host.logger.log('ResumableSSE', 'Stream error (network failure) - will attempt reconnect', {
|
||||
responseCode,
|
||||
hasData: data != null,
|
||||
});
|
||||
|
||||
if (run.reconnectAttempt < MAX_RETRIES) {
|
||||
// Increment counter BEFORE close() so abort handler knows we're reconnecting
|
||||
run.reconnectAttempt++;
|
||||
const delay = getReconnectDelay(run.reconnectAttempt);
|
||||
|
||||
host.logger.log(
|
||||
'ResumableSSE',
|
||||
`Reconnecting in ${delay}ms (attempt ${run.reconnectAttempt}/${MAX_RETRIES})`,
|
||||
);
|
||||
|
||||
sub.close();
|
||||
|
||||
sub.scheduleReattach(delay);
|
||||
|
||||
// Keep UI in "submitting" state during reconnection attempts
|
||||
// so user knows we're still trying (abort handler may have reset these)
|
||||
host.pane.setIsSubmitting(true);
|
||||
host.pane.setShowStopButton(generationCreatedAt != null);
|
||||
} else {
|
||||
await settleAfterRetryCeiling(ctx);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,4 +1,10 @@
|
|||
import type { Agents, TSubmission, TPendingSteer } from 'librechat-data-provider';
|
||||
import type {
|
||||
Agents,
|
||||
TMessage,
|
||||
TSubmission,
|
||||
TConversation,
|
||||
TPendingSteer,
|
||||
} from 'librechat-data-provider';
|
||||
import type { GenerationProtocolVersion } from '../protocol';
|
||||
import type { RunEnd } from '../steer/types';
|
||||
|
||||
|
|
@ -139,6 +145,13 @@ export interface ResumableCache {
|
|||
invalidateStreamStatus: (conversationId: string) => void;
|
||||
addActiveJob: (streamId: string) => void;
|
||||
removeActiveJob: (streamId: string) => void;
|
||||
/** Reads the conversation's saved record; rejects with the HTTP failure, a 404 included. */
|
||||
fetchConversation: (conversationId: string) => Promise<TConversation | undefined>;
|
||||
setConversation: (conversationId: string, conversation: TConversation) => void;
|
||||
/** Places the conversation's snapshot into every conversation list. */
|
||||
upsertConversation: (conversation: TConversation) => void;
|
||||
/** Drops the conversation from every conversation list. */
|
||||
removeConversation: (conversationId: string) => void;
|
||||
}
|
||||
|
||||
/** The pane's controls and run bookkeeping a resumable run drives. */
|
||||
|
|
@ -164,6 +177,18 @@ export interface ResumablePane {
|
|||
publishRunEnd: (end: RunEnd, options?: { retainDetached?: boolean }) => void;
|
||||
/** Consumes an armed "interrupt & send" for exactly this generation. */
|
||||
clearDrainAfterAbort: (conversationId: string, generationCreatedAt?: number) => void;
|
||||
/** Refreshes the token balance, when the deployment tracks one. */
|
||||
refreshBalance: () => void;
|
||||
/** Updates the pane's conversation; absent for a pane that only adds a comparison response. */
|
||||
updateConversation?: (update: (current: TConversation | null) => TConversation | null) => void;
|
||||
/**
|
||||
* The end of a run this pane left mid-stream and is now reattached to, as `messages` record
|
||||
* it. `resolvable` is false when no such run is remembered or the user stopped it.
|
||||
*/
|
||||
resolveDetachedEnd: (
|
||||
conversationId: string,
|
||||
messages: TMessage[] | undefined,
|
||||
) => { resolvable: boolean; end: RunEnd | null };
|
||||
}
|
||||
|
||||
/** What the event handlers that render the stream expose to the run. */
|
||||
|
|
@ -172,6 +197,14 @@ export interface ResumableEvents {
|
|||
flushPendingDeltas: () => void;
|
||||
/** Discards the live usage estimate for a run that stopped streaming. */
|
||||
resetLive: (submission: TSubmission) => void;
|
||||
/** Writes the failed turn's error card from the text (or raw error value) the server sent. */
|
||||
showError: (text: string | boolean | { message?: string }, submission: TSubmission) => void;
|
||||
}
|
||||
|
||||
/** The pane's message list. */
|
||||
export interface ResumableMessages {
|
||||
get: () => TMessage[] | undefined;
|
||||
set: (messages: TMessage[]) => void;
|
||||
}
|
||||
|
||||
/** Everything a resumable run needs from the app it runs in. */
|
||||
|
|
@ -180,4 +213,5 @@ export interface ResumableHost extends SubscriptionHost {
|
|||
pane: ResumablePane;
|
||||
steers: ResumableSteers;
|
||||
events: ResumableEvents;
|
||||
messages: ResumableMessages;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,3 +8,6 @@ export * from './host';
|
|||
export * from './subscription';
|
||||
export * from './terminal';
|
||||
export * from './reconcile';
|
||||
export * from './failure';
|
||||
export * from './missing';
|
||||
export * from './ceiling';
|
||||
|
|
|
|||
242
packages/chat/src/core/resumable/missing.ts
Normal file
242
packages/chat/src/core/resumable/missing.ts
Normal file
|
|
@ -0,0 +1,242 @@
|
|||
import { Constants } from 'librechat-data-provider';
|
||||
import type { TMessage } from 'librechat-data-provider';
|
||||
import type { TerminalContext } from './terminal';
|
||||
import {
|
||||
restoreLiveStatus,
|
||||
handoffToReplacement,
|
||||
clearPendingApprovalForTerminal,
|
||||
clearAttachedGenerationCreatedAt,
|
||||
} from './terminal';
|
||||
import { GENERATION_PROTOCOL_VERSION, supportsGenerationProtocolV2 } from '../protocol';
|
||||
import { isInitialNewConversation, keepLocalCodeApprovalMode } from './messages';
|
||||
import { getRecoverySteerId, toStartGenerationError } from './start';
|
||||
|
||||
/**
|
||||
* A 404 from the stream route: the job completed and was cleaned up, and its messages are
|
||||
* persisted. Refetches and reconciles them before recovering local steers (the final response
|
||||
* may contain an applied steer whose live event was missed), unless a status read shows the
|
||||
* generation, or a newer one, still running.
|
||||
*/
|
||||
export async function recoverMissingStream(ctx: TerminalContext): Promise<void> {
|
||||
const { sub, host } = ctx;
|
||||
const { run, generationCreatedAt, generationProtocolVersion } = sub;
|
||||
const currentStreamId = sub.streamId;
|
||||
const convoId = sub.submission.conversation?.conversationId;
|
||||
host.logger.log('ResumableSSE', 'Stream 404, invalidating messages for:', convoId);
|
||||
// Keep the UI non-idle while the status read decides whether this is
|
||||
// terminal cleanup or a handoff to a newer generation epoch.
|
||||
run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);
|
||||
run.cancelSteerRetryFrames();
|
||||
sub.close();
|
||||
/** Terminal: drop any in-flight live estimate so the gauge doesn't
|
||||
* keep counting stale streamed output after the stream ends */
|
||||
host.events.resetLive(sub.liveSubmission());
|
||||
host.pane.clearComposerDrafts(convoId, {
|
||||
includeNewChatDraft:
|
||||
!convoId ||
|
||||
convoId === Constants.NEW_CONVO ||
|
||||
run.optimisticStreamIds.has(currentStreamId) ||
|
||||
isInitialNewConversation(sub.submission),
|
||||
});
|
||||
host.events.clearStepMaps();
|
||||
let persistedMessages: TMessage[] | undefined;
|
||||
if (convoId) {
|
||||
try {
|
||||
await host.cache.invalidateMessages(convoId, 'none');
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
// The first stream can finish before CREATED mounts the saved-chat query.
|
||||
const fetched = await host.cache.fetchMessages(convoId, true);
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
if (Array.isArray(fetched)) {
|
||||
persistedMessages = fetched;
|
||||
}
|
||||
} catch (error) {
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
host.logger.warn('ResumableSSE', 'Could not reconcile persisted messages after 404', {
|
||||
conversationId: convoId,
|
||||
error,
|
||||
});
|
||||
}
|
||||
host.cache.removeStreamStatus(convoId);
|
||||
}
|
||||
const recoveryConvoId = convoId ?? currentStreamId;
|
||||
const reconciledIds = new Set(persistedMessages ? host.steers.appliedIds(persistedMessages) : []);
|
||||
if (persistedMessages) {
|
||||
host.steers.settleApplied(recoveryConvoId, persistedMessages);
|
||||
}
|
||||
|
||||
// The status replay is owner-gated and non-destructive; creating a
|
||||
// recovered follow-up consumes its exact source. Prefer these
|
||||
// authoritative leftovers over guessing from local chips.
|
||||
let confirmedV2Terminal = false;
|
||||
try {
|
||||
const status = await host.fetchStatus(recoveryConvoId);
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
|
||||
!supportsGenerationProtocolV2(status)
|
||||
) {
|
||||
sub.scheduleReattach(1_000);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
status.active === true &&
|
||||
status.createdAt != null &&
|
||||
(generationCreatedAt == null || status.createdAt !== generationCreatedAt)
|
||||
) {
|
||||
const handedOff = await handoffToReplacement(ctx, recoveryConvoId, status);
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
if (handedOff) {
|
||||
return;
|
||||
}
|
||||
/** Legacy generations have no replacement-handoff contract.
|
||||
* Never attach this submission to a different epoch. */
|
||||
sub.scheduleReattach(1_000);
|
||||
return;
|
||||
}
|
||||
if (status.active === true) {
|
||||
/** A stream lookup can briefly lose its routing record while the
|
||||
* authoritative job status still proves the same generation is
|
||||
* alive. Preserve ownership and retry the attachment instead of
|
||||
* publishing a terminal run-end (which could auto-drain queued
|
||||
* text into a still-running generation). */
|
||||
const activeStreamId = status.streamId ?? currentStreamId;
|
||||
host.cache.setStreamStatus(recoveryConvoId, status);
|
||||
if (status.createdAt != null) {
|
||||
host.pane.setGenerationEpoch(
|
||||
recoveryConvoId,
|
||||
status.createdAt,
|
||||
undefined,
|
||||
generationProtocolVersion,
|
||||
);
|
||||
}
|
||||
host.cache.addActiveJob(activeStreamId);
|
||||
restoreLiveStatus(ctx, recoveryConvoId, status, status.createdAt, generationProtocolVersion);
|
||||
host.pane.setIsSubmitting(true);
|
||||
host.pane.setShowStopButton(status.createdAt != null);
|
||||
run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);
|
||||
sub.scheduleReattach(1_000, activeStreamId, status.createdAt);
|
||||
return;
|
||||
}
|
||||
confirmedV2Terminal =
|
||||
generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
|
||||
supportsGenerationProtocolV2(status);
|
||||
const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal;
|
||||
const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : [];
|
||||
if (unrecovered.length > 0) {
|
||||
const recoverySteerId = getRecoverySteerId(sub.submission);
|
||||
host.steers.convertToQueued(recoveryConvoId, unrecovered, {
|
||||
generationProtocolVersion,
|
||||
...(recoverySteerId != null && {
|
||||
allowPreviouslyConvertedIds: [recoverySteerId],
|
||||
}),
|
||||
});
|
||||
for (const steer of unrecovered) {
|
||||
reconciledIds.add(steer.steerId);
|
||||
if (steer.clientSteerId) {
|
||||
reconciledIds.add(steer.clientSteerId);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (!sub.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
host.logger.warn('ResumableSSE', 'Could not recover parked steers after 404', {
|
||||
conversationId: recoveryConvoId,
|
||||
error,
|
||||
});
|
||||
}
|
||||
|
||||
host.cache.removeActiveJob(currentStreamId);
|
||||
clearAttachedGenerationCreatedAt(ctx);
|
||||
clearPendingApprovalForTerminal(ctx, recoveryConvoId);
|
||||
if (!run.createdStreamIds.has(currentStreamId) && run.optimisticStreamIds.has(currentStreamId)) {
|
||||
// Both fresh and resumed subscriptions can miss every event of a fast turn.
|
||||
// A missing job proves neither that the conversation was saved nor that it failed.
|
||||
try {
|
||||
const persisted = await host.cache.fetchConversation(recoveryConvoId);
|
||||
if (!sub.isCurrent()) return;
|
||||
if (persisted?.conversationId === recoveryConvoId) {
|
||||
host.cache.setConversation(recoveryConvoId, persisted);
|
||||
host.cache.upsertConversation(persisted);
|
||||
host.pane.updateConversation?.((current) => {
|
||||
if (
|
||||
current?.conversationId != null &&
|
||||
current.conversationId !== Constants.NEW_CONVO &&
|
||||
current.conversationId !== recoveryConvoId
|
||||
)
|
||||
return current;
|
||||
return keepLocalCodeApprovalMode(
|
||||
{ ...current, ...persisted },
|
||||
current,
|
||||
current?.conversationId,
|
||||
);
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
if (!sub.isCurrent()) return;
|
||||
if (toStartGenerationError(error)?.response?.status === 404) {
|
||||
host.cache.removeConversation(currentStreamId);
|
||||
}
|
||||
}
|
||||
// An inconclusive read must not turn a saved chat into a phantom.
|
||||
void host.cache.refreshConversationLists();
|
||||
void host.cache.refreshPinnedConversations();
|
||||
}
|
||||
if (persistedMessages) {
|
||||
host.messages.set(persistedMessages);
|
||||
}
|
||||
sub.retired = true;
|
||||
host.pane.setIsSubmitting(false);
|
||||
host.pane.setShowStopButton(false);
|
||||
|
||||
// Only infer recovery from local accepted chips after the persisted
|
||||
// final was successfully inspected. If that fetch failed, their
|
||||
// applied-vs-leftover outcome is unknown and converting could send
|
||||
// already-applied words twice.
|
||||
if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) {
|
||||
host.steers.convertLocalToQueued(recoveryConvoId, {
|
||||
excludeSteerIds: reconciledIds,
|
||||
generationProtocolVersion,
|
||||
});
|
||||
}
|
||||
// The true outcome is unknown here (job record already cleaned up):
|
||||
// a non-'completed' outcome releases parked interrupt flags without
|
||||
// auto-sending queued messages the user may not want fired. A run this
|
||||
// pane left and came back to is the exception: the history just read
|
||||
// names its response by epoch, which is the outcome it would have had.
|
||||
// History read before the job finished may not hold that response yet, or
|
||||
// may hold only the unfinished snapshot saved when the pane left, so a
|
||||
// marker that read leaves unresolved or stopped outlives this end for the
|
||||
// next visit, which reads history after the job is known to be done.
|
||||
const detached = host.pane.resolveDetachedEnd(recoveryConvoId, persistedMessages);
|
||||
const detachedResolvable = detached.resolvable;
|
||||
const detachedEnd = detached.end;
|
||||
host.pane.publishRunEnd(
|
||||
{
|
||||
conversationId: recoveryConvoId,
|
||||
outcome: 'aborted',
|
||||
endedAt: run.scheduler.now(),
|
||||
...detachedEnd,
|
||||
startedAsNewConvo: run.optimisticStreamIds.has(currentStreamId),
|
||||
},
|
||||
{
|
||||
retainDetached:
|
||||
detachedResolvable && (detachedEnd == null || detachedEnd.outcome === 'aborted'),
|
||||
},
|
||||
);
|
||||
host.pane.setStreamId(null);
|
||||
run.releaseStream(currentStreamId);
|
||||
run.reconnectAttempt = 0;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue