diff --git a/client/src/hooks/SSE/useResumableSSE.ts b/client/src/hooks/SSE/useResumableSSE.ts index d9008f45fa..de0479ac22 100644 --- a/client/src/hooks/SSE/useResumableSSE.ts +++ b/client/src/hooks/SSE/useResumableSSE.ts @@ -8,7 +8,6 @@ import { ABORT_SWEEP_STATUSES, selectLocalSteersForQueue } from '@librechat/chat import { Constants, QueryKeys, - ErrorTypes, StepEvents, StepTypes, SteerEvents, @@ -20,15 +19,12 @@ import { UsageEvents, createPayload, ApprovalEvents, - ViolationTypes, removeNullishValues, } from 'librechat-data-provider'; import { - MAX_RETRIES, getStepEventId, toStartGenerationError, isOAuthStepEvent, - getReconnectDelay, waitForRetryDelay, applyPendingAction, getRecoverySteerId, @@ -48,7 +44,6 @@ import { keepLocalCodeApprovalMode, hydrateSubmissionMessages, buildResumeEventSubmission, - completedResponseMessageId, getStartGenerationStreamId, buildOptimisticConversation, buildCreatedInitialResponse, @@ -65,9 +60,9 @@ import { ResumableRun, documentVisibility, ResumableSubscription, - handoffToReplacement, authorizeTerminalTeardown, reconcileGenerationLifecycle, + handleTransportFailure, clearPendingApprovalForTerminal, clearAttachedGenerationCreatedAt, } from '@librechat/chat'; @@ -288,6 +283,16 @@ export default function useResumableSSE( queryClient.invalidateQueries({ queryKey: streamStatusQueryKey(conversationId) }), addActiveJob, removeActiveJob, + fetchConversation: (conversationId) => + fetchConvoSnapshot(queryClient, conversationId, () => + dataService.getConversationById(conversationId), + ), + setConversation: (conversationId, conversation) => + queryClient.setQueryData([QueryKeys.conversation, conversationId], conversation), + upsertConversation: (conversation) => + upsertConvoInAllQueries(queryClient, conversation, true, 'snapshot'), + removeConversation: (conversationId) => + removeConvoFromAllQueries(queryClient, conversationId), }), [addActiveJob, queryClient, removeActiveJob], ); @@ -487,6 +492,23 @@ export default function useResumableSSE( clearComposerDrafts(runIndex, conversationId, options), publishRunEnd: setRunEnd, clearDrainAfterAbort, + refreshBalance: () => { + (startupConfig?.balance?.enabled ?? false) && balanceQuery.refetch(); + }, + updateConversation: isAddedRequest ? undefined : setConversation, + resolveDetachedEnd: (conversationId, messages) => { + const detachedRun = jotaiStore.get(detachedRunByConvoId(conversationId)); + const resolvable = + detachedRun != null && + !isDetachedRunManual(jotaiStore.get, conversationId, detachedRun); + return { + resolvable, + end: + resolvable && messages != null + ? resolveDetachedRunEnd(conversationId, detachedRun, messages) + : null, + }; + }, }, steers: { seed: seedSteerChips, @@ -495,7 +517,17 @@ export default function useResumableSSE( convertToQueued: convertSteersToQueued, convertLocalToQueued: convertLocalSteersToQueued, }, - events: { clearStepMaps, flushPendingDeltas, resetLive }, + events: { + clearStepMaps, + flushPendingDeltas, + resetLive, + showError: (text, errorSubmission) => + errorHandler({ + data: { text } as unknown as Parameters[0]['data'], + submission: errorSubmission as EventSubmission, + }), + }, + messages: { get: getMessages, set: setMessages }, }; const sub = new ResumableSubscription(run, host, target); const { @@ -1587,644 +1619,6 @@ export default function useResumableSSE( } }; - /** - * Error event handler - handles BOTH: - * 1. HTTP-level errors (status present) - 404, 409, network failures (0) - * 2. Server-sent error events (event: error with data) - known errors like ViolationTypes/ErrorTypes - * - * Order matters: check the status first since HTTP errors may also include data. - * The transport has already retried a 401 with a refreshed token. - */ - const handleTransportFailure = async ({ - status: responseCode, - data, - }: Pick, 'status' | 'data'>) => { - if (!sub.isCurrent()) { - return; - } - - if (sub.finalReceived) { - logger.log('ResumableSSE', 'Ignoring error after FINAL event', { - responseCode, - hasData: data != null, - }); - return; - } - - (startupConfig?.balance?.enabled ?? false) && balanceQuery.refetch(); - - if (responseCode === 409) { - const replacementConvoId = sub.submission.conversation?.conversationId ?? currentStreamId; - const handedOff = await handoffToReplacement(terminal, replacementConvoId); - if (!sub.isCurrent()) { - return; - } - if (handedOff) { - return; - } - } - - // 404 → job completed & was cleaned up; messages are persisted in DB. - // Refetch and reconcile them before recovering local steers: the final - // response may contain an applied steer whose live event was missed. - if (responseCode === 404) { - const convoId = sub.submission.conversation?.conversationId; - logger.log('ResumableSSE', 'Stream 404, invalidating messages for:', convoId); - // Keep the UI non-idle while the status read decides whether this is - // terminal cleanup or a handoff to a newer generation epoch. - run.reconnectAttempt = Math.max(run.reconnectAttempt, 1); - run.cancelSteerRetryFrames(); - sub.close(); - /** Terminal: drop any in-flight live estimate so the gauge doesn't - * keep counting stale streamed output after the stream ends */ - resetLive(sub.liveSubmission()); - clearComposerDrafts(runIndex, convoId, { - includeNewChatDraft: - !convoId || - convoId === Constants.NEW_CONVO || - run.optimisticStreamIds.has(currentStreamId) || - isInitialNewConversation(sub.submission), - }); - clearStepMaps(); - let persistedMessages: TMessage[] | undefined; - if (convoId) { - try { - const messageQueryKey = [QueryKeys.messages, convoId] as const; - await queryClient.invalidateQueries({ - queryKey: messageQueryKey, - refetchType: 'none', - }); - if (!sub.isCurrent()) { - return; - } - const fetched = await queryClient.fetchQuery({ - queryKey: messageQueryKey, - // The first stream can finish before CREATED mounts the saved-chat query. - queryFn: () => fetchConversationMessages(convoId), - }); - if (!sub.isCurrent()) { - return; - } - if (Array.isArray(fetched)) { - persistedMessages = fetched; - } - } catch (error) { - if (!sub.isCurrent()) { - return; - } - logger.warn('ResumableSSE', 'Could not reconcile persisted messages after 404', { - conversationId: convoId, - error, - }); - } - queryClient.removeQueries({ queryKey: streamStatusQueryKey(convoId) }); - } - const recoveryConvoId = convoId ?? currentStreamId; - const reconciledIds = new Set( - persistedMessages ? collectAppliedSteerIds(persistedMessages) : [], - ); - if (persistedMessages) { - settleAppliedSteerParts(recoveryConvoId, persistedMessages); - } - - // The status replay is owner-gated and non-destructive; creating a - // recovered follow-up consumes its exact source. Prefer these - // authoritative leftovers over guessing from local chips. - let confirmedV2Terminal = false; - try { - const status = await fetchStreamStatus(recoveryConvoId); - if (!sub.isCurrent()) { - return; - } - if ( - generationProtocolVersion === GENERATION_PROTOCOL_VERSION && - !supportsGenerationProtocolV2(status) - ) { - sub.scheduleReattach(1_000); - return; - } - if ( - status.active === true && - status.createdAt != null && - (generationCreatedAt == null || status.createdAt !== generationCreatedAt) - ) { - const handedOff = await handoffToReplacement(terminal, recoveryConvoId, status); - if (!sub.isCurrent()) { - return; - } - if (handedOff) { - return; - } - /** Legacy generations have no replacement-handoff contract. - * Never attach this submission to a different epoch. */ - sub.scheduleReattach(1_000); - return; - } - if (status.active === true) { - /** A stream lookup can briefly lose its routing record while the - * authoritative job status still proves the same generation is - * alive. Preserve ownership and retry the attachment instead of - * publishing a terminal run-end (which could auto-drain queued - * text into a still-running generation). */ - const activeStreamId = status.streamId ?? currentStreamId; - queryClient.setQueryData(streamStatusQueryKey(recoveryConvoId), status); - if (status.createdAt != null) { - updateActiveGenerationCreatedAt( - recoveryConvoId, - status.createdAt, - undefined, - generationProtocolVersion, - ); - } - addActiveJob(activeStreamId); - seedSteerChips( - recoveryConvoId, - (status.resumeState?.pendingSteers ?? []) as TPendingSteer[], - status.createdAt, - generationProtocolVersion, - ); - settleAppliedSteerParts( - recoveryConvoId, - status.resumeState?.aggregatedContent ?? status.aggregatedContent, - ); - if (status.pendingAction ?? status.resumeState?.pendingAction) { - applyPendingActionToMessages( - (status.pendingAction ?? - status.resumeState?.pendingAction) as Agents.PendingAction, - ); - } - setIsSubmitting(true); - setShowStopButton(status.createdAt != null); - run.reconnectAttempt = Math.max(run.reconnectAttempt, 1); - sub.scheduleReattach(1_000, activeStreamId, status.createdAt); - return; - } - confirmedV2Terminal = - generationProtocolVersion === GENERATION_PROTOCOL_VERSION && - supportsGenerationProtocolV2(status); - const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal; - const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : []; - if (unrecovered.length > 0) { - const recoverySteerId = getRecoverySteerId(sub.submission); - convertSteersToQueued(recoveryConvoId, unrecovered, { - generationProtocolVersion, - ...(recoverySteerId != null && { - allowPreviouslyConvertedIds: [recoverySteerId], - }), - }); - for (const steer of unrecovered) { - reconciledIds.add(steer.steerId); - if (steer.clientSteerId) { - reconciledIds.add(steer.clientSteerId); - } - } - } - } catch (error) { - if (!sub.isCurrent()) { - return; - } - logger.warn('ResumableSSE', 'Could not recover parked steers after 404', { - conversationId: recoveryConvoId, - error, - }); - } - - removeActiveJob(currentStreamId); - clearAttachedGenerationCreatedAt(terminal); - clearPendingApprovalForTerminal(terminal, recoveryConvoId); - if ( - !run.createdStreamIds.has(currentStreamId) && - run.optimisticStreamIds.has(currentStreamId) - ) { - // Both fresh and resumed subscriptions can miss every event of a fast turn. - // A missing job proves neither that the conversation was saved nor that it failed. - try { - const persisted = await fetchConvoSnapshot(queryClient, recoveryConvoId, () => - dataService.getConversationById(recoveryConvoId), - ); - if (!sub.isCurrent()) return; - if (persisted?.conversationId === recoveryConvoId) { - queryClient.setQueryData([QueryKeys.conversation, recoveryConvoId], persisted); - upsertConvoInAllQueries(queryClient, persisted, true, 'snapshot'); - if (!isAddedRequest) { - setConversation?.((current) => { - if ( - current?.conversationId != null && - current.conversationId !== Constants.NEW_CONVO && - current.conversationId !== recoveryConvoId - ) - return current; - return keepLocalCodeApprovalMode( - { ...current, ...persisted }, - current, - current?.conversationId, - ); - }); - } - } - } catch (error) { - if (!sub.isCurrent()) return; - if (toStartGenerationError(error)?.response?.status === 404) { - removeConvoFromAllQueries(queryClient, currentStreamId); - } - } - // An inconclusive read must not turn a saved chat into a phantom. - invalidateConversationLists(queryClient); - queryClient.invalidateQueries({ queryKey: [QueryKeys.pinnedConversations] }); - } - if (persistedMessages) { - setMessages(persistedMessages); - } - sub.retired = true; - setIsSubmitting(false); - setShowStopButton(false); - - // Only infer recovery from local accepted chips after the persisted - // final was successfully inspected. If that fetch failed, their - // applied-vs-leftover outcome is unknown and converting could send - // already-applied words twice. - if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) { - convertLocalSteersToQueued(recoveryConvoId, { - excludeSteerIds: reconciledIds, - generationProtocolVersion, - }); - } - // The true outcome is unknown here (job record already cleaned up): - // a non-'completed' outcome releases parked interrupt flags without - // auto-sending queued messages the user may not want fired. A run this - // pane left and came back to is the exception: the history just read - // names its response by epoch, which is the outcome it would have had. - // History read before the job finished may not hold that response yet, or - // may hold only the unfinished snapshot saved when the pane left, so a - // marker that read leaves unresolved or stopped outlives this end for the - // next visit, which reads history after the job is known to be done. - const detachedRun = jotaiStore.get(detachedRunByConvoId(recoveryConvoId)); - const detachedResolvable = - detachedRun != null && - !isDetachedRunManual(jotaiStore.get, recoveryConvoId, detachedRun); - const detachedEnd = - detachedResolvable && persistedMessages != null - ? resolveDetachedRunEnd(recoveryConvoId, detachedRun, persistedMessages) - : null; - setRunEnd( - { - conversationId: recoveryConvoId, - outcome: 'aborted', - endedAt: run.scheduler.now(), - ...detachedEnd, - startedAsNewConvo: run.optimisticStreamIds.has(currentStreamId), - }, - { - retainDetached: - detachedResolvable && (detachedEnd == null || detachedEnd.outcome === 'aborted'), - }, - ); - setStreamId(null); - run.releaseStream(currentStreamId); - run.reconnectAttempt = 0; - return; - } - - /** - * Server-sent error event (event: error with data) - no responseCode. - * These are known errors (ErrorTypes, ViolationTypes) that should be displayed to user. - * Only check the data if there's no HTTP status, since HTTP errors may also have body data. - * Note: responseCode === 0 means transport failure (connection dropped) - treat as network error, - * not a server-sent error payload. Use `== null` to only match undefined/null (no HTTP status). - */ - if (responseCode == null && data != null) { - sub.finalReceived = true; - const recoveryConvoId = sub.submission.conversation?.conversationId ?? currentStreamId; - if ( - !(await authorizeTerminalTeardown(terminal, recoveryConvoId, 'error')) || - !sub.isCurrent() - ) { - return; - } - logger.log('ResumableSSE', 'Server-sent error event received:', data); - run.cancelSteerRetryFrames(); - sub.close(); - /** FLUSH (not cancel): the error card below is built from the cache - * tail, so queued tokens must land first, and a stale trailing - * frame must never overwrite the error write. */ - flushPendingDeltas(); - removeActiveJob(currentStreamId); - clearAttachedGenerationCreatedAt(terminal); - clearPendingApprovalForTerminal(terminal, recoveryConvoId); - resetLive(sub.liveSubmission()); - if ( - !run.createdStreamIds.has(currentStreamId) && - run.optimisticStreamIds.has(currentStreamId) - ) { - removeConvoFromAllQueries(queryClient, currentStreamId); - } - - const errorSupportsV2 = supportsGenerationProtocolV2(data); - const errorString = - typeof data === 'string' ? data : (data.error ?? data.message ?? JSON.stringify(data)); - - // Check if it's a known error type (ViolationTypes or ErrorTypes) - let isKnownError = false; - try { - const parsed = typeof errorString === 'string' ? JSON.parse(errorString) : errorString; - const errorType = parsed?.type ?? parsed?.code; - if (errorType) { - const violationValues = Object.values(ViolationTypes) as string[]; - const errorTypeValues = Object.values(ErrorTypes) as string[]; - isKnownError = - violationValues.includes(errorType) || errorTypeValues.includes(errorType); - } - } catch { - // Not JSON or parsing failed - treat as generic error - } - - logger.log('ResumableSSE', 'Error type check:', { isKnownError, errorString }); - - // Display the error to user via errorHandler - errorHandler({ - data: { text: errorString } as unknown as Parameters[0]['data'], - submission: sub.submission as EventSubmission, - }); - - setIsSubmitting(false); - setShowStopButton(false); - const recoverySteerId = getRecoverySteerId(sub.submission); - if ( - recoverySteerId != null && - generationProtocolVersion === GENERATION_PROTOCOL_VERSION && - errorSupportsV2 - ) { - try { - /** The error was received on the exact generation-fenced stream. - * Read status only now, after terminal was observed: a claim - * started before this recovery could return a stale parked copy - * after a successful commit and must never authorize redelivery. */ - const terminalStatus = await fetchStreamStatus(recoveryConvoId); - if (!sub.isCurrent()) { - return; - } - const releasedSource = terminalStatus.unrecoveredSteers?.find( - (steer) => steer.steerId === recoverySteerId, - ); - if ( - terminalStatus.active === false && - supportsGenerationProtocolV2(terminalStatus) && - releasedSource != null - ) { - convertSteersToQueued(recoveryConvoId, [releasedSource], { - generationProtocolVersion, - allowPreviouslyConvertedIds: [recoverySteerId], - }); - } - } catch (error) { - if (!sub.isCurrent()) { - return; - } - logger.warn('ResumableSSE', 'Could not recover source after generation error', { - conversationId: recoveryConvoId, - error, - }); - } - } - // The error terminal's backstop parks acked leftovers server-side; - // claim it now so a reload can't resurrect the chips converted here. - if (generationProtocolVersion === 1 || errorSupportsV2) { - convertLocalSteersToQueued(recoveryConvoId, { - claimParked: true, - generationProtocolVersion, - }); - } - setRunEnd({ - conversationId: recoveryConvoId, - outcome: 'error', - endedAt: run.scheduler.now(), - generationCreatedAt, - }); - setStreamId(null); - run.releaseStream(currentStreamId); - run.reconnectAttempt = 0; - return; - } - - // Network failure or unknown HTTP error - attempt reconnection with backoff - logger.log('ResumableSSE', 'Stream error (network failure) - will attempt reconnect', { - responseCode, - hasData: data != null, - }); - - if (run.reconnectAttempt < MAX_RETRIES) { - // Increment counter BEFORE close() so abort handler knows we're reconnecting - run.reconnectAttempt++; - const delay = getReconnectDelay(run.reconnectAttempt); - - logger.log( - 'ResumableSSE', - `Reconnecting in ${delay}ms (attempt ${run.reconnectAttempt}/${MAX_RETRIES})`, - ); - - sub.close(); - - sub.scheduleReattach(delay); - - // Keep UI in "submitting" state during reconnection attempts - // so user knows we're still trying (abort handler may have reset these) - setIsSubmitting(true); - setShowStopButton(generationCreatedAt != null); - } else { - logger.error('ResumableSSE', 'Max reconnect attempts reached'); - sub.close(); - flushPendingDeltas(); - const recoveryConvoId = sub.submission.conversation?.conversationId ?? currentStreamId; - let status: Awaited> | undefined; - try { - status = await fetchStreamStatus(recoveryConvoId); - if (!sub.isCurrent()) { - return; - } - } catch (error) { - if (!sub.isCurrent()) { - return; - } - logger.warn('ResumableSSE', 'Could not determine job state after reconnect limit', { - conversationId: recoveryConvoId, - error, - }); - } - - if ( - status != null && - generationProtocolVersion === GENERATION_PROTOCOL_VERSION && - !supportsGenerationProtocolV2(status) - ) { - // A legacy responder cannot adjudicate a v2 job during rollout. - status = undefined; - } - - if ( - status?.active === true && - status.createdAt != null && - (generationCreatedAt == null || status.createdAt !== generationCreatedAt) - ) { - const handedOff = await handoffToReplacement(terminal, recoveryConvoId, status); - if (!sub.isCurrent()) { - return; - } - if (handedOff) { - return; - } - run.reconnectAttempt = 0; - sub.scheduleReattach(30_000); - return; - } - - if (status?.active === true) { - // The transport gave up, not the generation. Keep the active-job - // marker and accepted chips intact, then release this stale - // submission so useResumeOnLoad can establish a fresh attachment. - queryClient.setQueryData(streamStatusQueryKey(recoveryConvoId), status); - seedSteerChips( - recoveryConvoId, - (status.resumeState?.pendingSteers ?? []) as TPendingSteer[], - status.createdAt, - generationProtocolVersion, - ); - settleAppliedSteerParts( - recoveryConvoId, - status.resumeState?.aggregatedContent ?? status.aggregatedContent, - ); - if (status.pendingAction ?? status.resumeState?.pendingAction) { - applyPendingActionToMessages( - (status.pendingAction ?? status.resumeState?.pendingAction) as Agents.PendingAction, - ); - } - setIsSubmitting(true); - setShowStopButton(generationCreatedAt != null); - run.reconnectAttempt = 0; - sub.scheduleReattach(30_000); - return; - } - - if (status == null) { - // An inconclusive status read is still not proof of termination. - // Preserve server/client ownership and let the normal resume-on-load - // status query retry rather than duplicating accepted words. - queryClient.invalidateQueries({ queryKey: streamStatusQueryKey(recoveryConvoId) }); - setIsSubmitting(true); - setShowStopButton(generationCreatedAt != null); - run.reconnectAttempt = 0; - sub.scheduleReattach(30_000); - return; - } - - // The status endpoint proved the job terminal. Reconcile persisted - // content before recovering leftovers, just like the 404 path. - run.cancelSteerRetryFrames(); - let persistedMessages: TMessage[] | undefined; - try { - const messageQueryKey = [QueryKeys.messages, recoveryConvoId] as const; - await queryClient.invalidateQueries({ - queryKey: messageQueryKey, - refetchType: 'none', - }); - if (!sub.isCurrent()) { - return; - } - const fetched = await queryClient.fetchQuery({ - queryKey: messageQueryKey, - }); - if (!sub.isCurrent()) { - return; - } - if (Array.isArray(fetched)) { - persistedMessages = fetched; - } - } catch (error) { - if (!sub.isCurrent()) { - return; - } - logger.warn('ResumableSSE', 'Could not reconcile terminal messages after disconnect', { - conversationId: recoveryConvoId, - error, - }); - } - - const authoritativeValues = [ - ...(persistedMessages ?? []), - ...(status.resumeState?.aggregatedContent ?? status.aggregatedContent ?? []), - ]; - const confirmedV2Terminal = - generationProtocolVersion === GENERATION_PROTOCOL_VERSION && - supportsGenerationProtocolV2(status); - const reconciledIds = new Set(collectAppliedSteerIds(authoritativeValues)); - settleAppliedSteerParts(recoveryConvoId, authoritativeValues); - const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal; - const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : []; - if (unrecovered.length > 0) { - const recoverySteerId = getRecoverySteerId(sub.submission); - convertSteersToQueued(recoveryConvoId, unrecovered, { - generationProtocolVersion, - ...(recoverySteerId != null && { - allowPreviouslyConvertedIds: [recoverySteerId], - }), - }); - for (const steer of unrecovered) { - reconciledIds.add(steer.steerId); - if (steer.clientSteerId) { - reconciledIds.add(steer.clientSteerId); - } - } - } - if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) { - convertLocalSteersToQueued(recoveryConvoId, { - excludeSteerIds: reconciledIds, - generationProtocolVersion, - }); - } - - resetLive(sub.liveSubmission()); - removeActiveJob(currentStreamId); - clearAttachedGenerationCreatedAt(terminal); - clearPendingApprovalForTerminal(terminal, recoveryConvoId); - if ( - !run.createdStreamIds.has(currentStreamId) && - run.optimisticStreamIds.has(currentStreamId) - ) { - removeConvoFromAllQueries(queryClient, currentStreamId); - } - sub.retired = true; - setIsSubmitting(false); - setShowStopButton(false); - let recoveryOutcome: 'completed' | 'aborted' | 'error' = 'error'; - if (status.status === 'complete' && persistedMessages != null) { - recoveryOutcome = 'completed'; - } else if (status.status === 'aborted') { - recoveryOutcome = 'aborted'; - } - const recoveredResponseMessageId = - recoveryOutcome === 'completed' - ? completedResponseMessageId( - persistedMessages, - sub.userMessage?.messageId, - status?.resumeState?.responseMessageId ?? - sub.submission.initialResponse?.messageId, - ) - : undefined; - setRunEnd({ - conversationId: recoveryConvoId, - outcome: recoveryOutcome, - endedAt: run.scheduler.now(), - generationCreatedAt: status.createdAt ?? generationCreatedAt, - ...(recoveredResponseMessageId != null && { - responseMessageId: recoveredResponseMessageId, - }), - }); - setSubmission(null); - setStreamId(null); - run.releaseStream(currentStreamId); - run.reconnectAttempt = 0; - } - }; - /** * The transport emits `abort` only for this hook's own closes. A cancel * the user agent issued (a backgrounded or frozen mobile tab, while the @@ -2277,7 +1671,7 @@ export default function useResumableSSE( sub.connect(transport.stream({ token }), { onOpen: handleOpen, onFrame: (event) => void handleFrame(event), - onFailure: (event) => void handleTransportFailure(event), + onFailure: (event) => void handleTransportFailure(terminal, event), onClose: handleClose, }); @@ -2291,7 +1685,7 @@ export default function useResumableSSE( /** Simulate network drop - triggers error event → reconnection */ debugWindow.__killNetwork = () => { logger.log('Debug', 'Simulating network drop...'); - void handleTransportFailure({}); + void handleTransportFailure(terminal, {}); }; /** Simulate clean close (navigation away) - triggers abort event → no reconnection */ @@ -2332,7 +1726,6 @@ export default function useResumableSSE( startupConfig?.balance?.enabled, balanceQuery, removeActiveJob, - queryClient, bindResponse, contextHandler, usageHandler, @@ -2351,7 +1744,6 @@ export default function useResumableSSE( settleAppliedSteerParts, convertSteersToQueued, convertLocalSteersToQueued, - addActiveJob, setSubmission, updateActiveGenerationCreatedAt, jotaiStore, diff --git a/packages/chat/src/core/__tests__/failure.spec.ts b/packages/chat/src/core/__tests__/failure.spec.ts new file mode 100644 index 0000000000..ee6d2a342c --- /dev/null +++ b/packages/chat/src/core/__tests__/failure.spec.ts @@ -0,0 +1,218 @@ +import type { TMessage, TSubmission } from 'librechat-data-provider'; +import type { + ResumableHost, + TerminalContext, + ResumableScheduler, + StreamStatusResponse, +} from '../resumable'; +import { ResumableRun, ResumableSubscription, handleTransportFailure } from '../resumable'; + +/** Timers that run only when the test advances the clock. */ +const createScheduler = () => { + let now = 0; + const timers = new Map void }>(); + let nextId = 1; + const scheduler: ResumableScheduler = { + setTimeout: (callback, delay) => { + const id = nextId++; + timers.set(id, { at: now + delay, callback }); + return id as unknown as ReturnType; + }, + clearTimeout: (handle) => { + timers.delete(handle as unknown as number); + }, + requestFrame: () => nextId++, + cancelFrame: () => undefined, + now: () => now, + }; + const advance = (ms: number) => { + now += ms; + for (const [id, timer] of [...timers]) { + if (timer.at <= now) { + timers.delete(id); + timer.callback(); + } + } + }; + return { scheduler, advance }; +}; + +const createHost = (fetchStatus: jest.Mock, [string]>) => { + const host: ResumableHost = { + logger: { log: jest.fn(), warn: jest.fn(), error: jest.fn() }, + visibility: { isVisible: () => true, subscribe: () => () => undefined }, + fetchStatus, + subscribe: jest.fn(), + cache: { + invalidateMessages: jest.fn(async () => undefined), + fetchMessages: jest.fn(async () => [] as TMessage[]), + refreshConversation: jest.fn(async () => undefined), + refreshConversationLists: jest.fn(async () => undefined), + refreshPinnedConversations: jest.fn(async () => undefined), + setStreamStatus: jest.fn(), + removeStreamStatus: jest.fn(), + invalidateStreamStatus: jest.fn(), + addActiveJob: jest.fn(), + removeActiveJob: jest.fn(), + fetchConversation: jest.fn(async () => undefined), + setConversation: jest.fn(), + upsertConversation: jest.fn(), + removeConversation: jest.fn(), + }, + pane: { + setIsSubmitting: jest.fn(), + setShowStopButton: jest.fn(), + setSubmission: jest.fn(), + setStreamId: jest.fn(), + setGenerationEpoch: jest.fn(() => true), + setPendingAction: jest.fn(), + clearComposerDrafts: jest.fn(), + publishRunEnd: jest.fn(), + clearDrainAfterAbort: jest.fn(), + refreshBalance: jest.fn(), + updateConversation: jest.fn(), + resolveDetachedEnd: jest.fn(() => ({ resolvable: false, end: null })), + }, + steers: { + seed: jest.fn(), + appliedIds: jest.fn(() => []), + settleApplied: jest.fn(), + convertToQueued: jest.fn(), + convertLocalToQueued: jest.fn(), + }, + events: { + clearStepMaps: jest.fn(), + flushPendingDeltas: jest.fn(), + resetLive: jest.fn(), + showError: jest.fn(), + }, + messages: { get: jest.fn(() => []), set: jest.fn() }, + }; + return host; +}; + +const submission = { + userMessage: { messageId: 'u-1', conversationId: 'c-1', isCreatedByUser: true } as TMessage, + initialResponse: { messageId: 'u-1_', conversationId: 'c-1' } as TMessage, + conversation: { conversationId: 'c-1' }, +} as TSubmission; + +const setup = () => { + const clock = createScheduler(); + const fetchStatus = jest.fn, [string]>(); + const host = createHost(fetchStatus); + const run = new ResumableRun(clock.scheduler); + run.submission = submission; + const sub = new ResumableSubscription(run, host, { + streamId: 'c-1', + submission, + isResume: false, + generationCreatedAt: 2000, + generationProtocolVersion: 2, + }); + sub.claim(); + const ctx: TerminalContext = { sub, host, applyPendingAction: jest.fn() }; + return { clock, fetchStatus, host, run, sub, ctx }; +}; + +describe('handleTransportFailure', () => { + it('ignores a failure after the final frame', async () => { + const { ctx, host } = setup(); + ctx.sub.finalReceived = true; + await handleTransportFailure(ctx, { status: 0 }); + expect(host.pane.refreshBalance).not.toHaveBeenCalled(); + }); + + it('backs off and re-attaches a dropped connection', async () => { + const { ctx, host, run, clock } = setup(); + await handleTransportFailure(ctx, { status: 0 }); + expect(run.reconnectAttempt).toBe(1); + expect(host.pane.setIsSubmitting).toHaveBeenCalledWith(true); + expect(host.pane.setShowStopButton).toHaveBeenCalledWith(true); + clock.advance(999); + expect(host.subscribe).not.toHaveBeenCalled(); + clock.advance(1); + expect(host.subscribe).toHaveBeenCalledWith(expect.objectContaining({ isResume: true })); + }); + + it('writes a served error once the status proves the run over', async () => { + const { ctx, host, fetchStatus } = setup(); + fetchStatus.mockResolvedValue({ active: false, generationProtocolVersion: 2 }); + await handleTransportFailure(ctx, { + data: { message: 'quota exceeded', generationProtocolVersion: 2 }, + }); + expect(ctx.sub.finalReceived).toBe(true); + expect(host.events.flushPendingDeltas).toHaveBeenCalled(); + expect(host.events.showError).toHaveBeenCalledWith('quota exceeded', ctx.sub.submission); + expect(host.steers.convertLocalToQueued).toHaveBeenCalledWith('c-1', { + claimParked: true, + generationProtocolVersion: 2, + }); + expect(host.pane.publishRunEnd).toHaveBeenCalledWith({ + conversationId: 'c-1', + outcome: 'error', + endedAt: 0, + generationCreatedAt: 2000, + }); + }); + + it('settles a missing stream from persisted history', async () => { + const { ctx, host, fetchStatus } = setup(); + const persisted = [submission.userMessage]; + (host.cache.fetchMessages as jest.Mock).mockResolvedValue(persisted); + fetchStatus.mockResolvedValue({ active: false, generationProtocolVersion: 2 }); + await handleTransportFailure(ctx, { status: 404 }); + expect(host.cache.fetchMessages).toHaveBeenCalledWith('c-1', true); + expect(host.cache.removeStreamStatus).toHaveBeenCalledWith('c-1'); + expect(host.messages.set).toHaveBeenCalledWith(persisted); + expect(ctx.sub.retired).toBe(true); + expect(host.pane.publishRunEnd).toHaveBeenCalledWith( + expect.objectContaining({ conversationId: 'c-1', outcome: 'aborted' }), + { retainDetached: false }, + ); + }); + + it('keeps a missing stream attached while its generation is still live', async () => { + const { ctx, host, fetchStatus, clock } = setup(); + fetchStatus.mockResolvedValue({ + active: true, + streamId: 'c-1', + createdAt: 2000, + generationProtocolVersion: 2, + }); + await handleTransportFailure(ctx, { status: 404 }); + expect(host.pane.publishRunEnd).not.toHaveBeenCalled(); + expect(host.steers.seed).toHaveBeenCalledWith('c-1', [], 2000, 2); + clock.advance(1000); + expect(host.subscribe).toHaveBeenCalledWith( + expect.objectContaining({ streamId: 'c-1', generationCreatedAt: 2000 }), + ); + }); + + it('waits on a live generation once the retry ceiling is reached', async () => { + const { ctx, host, run, fetchStatus } = setup(); + run.reconnectAttempt = 5; + fetchStatus.mockResolvedValue({ active: true, createdAt: 2000, generationProtocolVersion: 2 }); + await handleTransportFailure(ctx, { status: 0 }); + expect(host.cache.setStreamStatus).toHaveBeenCalled(); + expect(run.reconnectAttempt).toBe(0); + expect(host.pane.publishRunEnd).not.toHaveBeenCalled(); + }); + + it('settles a run the status proves terminal past the retry ceiling', async () => { + const { ctx, host, run, fetchStatus } = setup(); + run.reconnectAttempt = 5; + fetchStatus.mockResolvedValue({ + active: false, + status: 'aborted', + createdAt: 2000, + generationProtocolVersion: 2, + }); + await handleTransportFailure(ctx, { status: 0 }); + expect(ctx.sub.retired).toBe(true); + expect(host.pane.setSubmission).toHaveBeenCalledWith(null); + expect(host.pane.publishRunEnd).toHaveBeenCalledWith( + expect.objectContaining({ outcome: 'aborted', generationCreatedAt: 2000 }), + ); + }); +}); diff --git a/packages/chat/src/core/__tests__/terminal.spec.ts b/packages/chat/src/core/__tests__/terminal.spec.ts index eeaa056285..3b9b7d1b00 100644 --- a/packages/chat/src/core/__tests__/terminal.spec.ts +++ b/packages/chat/src/core/__tests__/terminal.spec.ts @@ -60,6 +60,10 @@ const createHost = (fetchStatus: jest.Mock, [strin invalidateStreamStatus: jest.fn(), addActiveJob: jest.fn(), removeActiveJob: jest.fn(), + fetchConversation: jest.fn(async () => undefined), + setConversation: jest.fn(), + upsertConversation: jest.fn(), + removeConversation: jest.fn(), }, pane: { setIsSubmitting: jest.fn(), @@ -71,6 +75,9 @@ const createHost = (fetchStatus: jest.Mock, [strin clearComposerDrafts: jest.fn(), publishRunEnd: jest.fn(), clearDrainAfterAbort: jest.fn(), + refreshBalance: jest.fn(), + updateConversation: jest.fn(), + resolveDetachedEnd: jest.fn(() => ({ resolvable: false, end: null })), }, steers: { seed: jest.fn(), @@ -79,7 +86,13 @@ const createHost = (fetchStatus: jest.Mock, [strin convertToQueued: jest.fn(), convertLocalToQueued: jest.fn(), }, - events: { clearStepMaps: jest.fn(), flushPendingDeltas: jest.fn(), resetLive: jest.fn() }, + events: { + clearStepMaps: jest.fn(), + flushPendingDeltas: jest.fn(), + resetLive: jest.fn(), + showError: jest.fn(), + }, + messages: { get: jest.fn(() => []), set: jest.fn() }, }; return host; }; diff --git a/packages/chat/src/core/resumable/ceiling.ts b/packages/chat/src/core/resumable/ceiling.ts new file mode 100644 index 0000000000..94cbde19d7 --- /dev/null +++ b/packages/chat/src/core/resumable/ceiling.ts @@ -0,0 +1,190 @@ +import type { TMessage } from 'librechat-data-provider'; +import type { StreamStatusResponse } from './host'; +import type { TerminalContext } from './terminal'; +import { + restoreLiveStatus, + handoffToReplacement, + clearPendingApprovalForTerminal, + clearAttachedGenerationCreatedAt, +} from './terminal'; +import { GENERATION_PROTOCOL_VERSION, supportsGenerationProtocolV2 } from '../protocol'; +import { completedResponseMessageId } from './messages'; +import { getRecoverySteerId } from './start'; + +/** + * The connection failed past the last retry. The transport gave up, not necessarily the + * generation: a status read decides whether to keep waiting on it (or hand it to a newer + * epoch), and only a status that proves it terminal settles the run from durable history. + */ +export async function settleAfterRetryCeiling(ctx: TerminalContext): Promise { + const { sub, host } = ctx; + const { run, generationCreatedAt, generationProtocolVersion } = sub; + const currentStreamId = sub.streamId; + host.logger.error('ResumableSSE', 'Max reconnect attempts reached'); + sub.close(); + host.events.flushPendingDeltas(); + const recoveryConvoId = sub.submission.conversation?.conversationId ?? currentStreamId; + let status: StreamStatusResponse | undefined; + try { + status = await host.fetchStatus(recoveryConvoId); + if (!sub.isCurrent()) { + return; + } + } catch (error) { + if (!sub.isCurrent()) { + return; + } + host.logger.warn('ResumableSSE', 'Could not determine job state after reconnect limit', { + conversationId: recoveryConvoId, + error, + }); + } + + if ( + status != null && + generationProtocolVersion === GENERATION_PROTOCOL_VERSION && + !supportsGenerationProtocolV2(status) + ) { + // A legacy responder cannot adjudicate a v2 job during rollout. + status = undefined; + } + + if ( + status?.active === true && + status.createdAt != null && + (generationCreatedAt == null || status.createdAt !== generationCreatedAt) + ) { + const handedOff = await handoffToReplacement(ctx, recoveryConvoId, status); + if (!sub.isCurrent()) { + return; + } + if (handedOff) { + return; + } + run.reconnectAttempt = 0; + sub.scheduleReattach(30_000); + return; + } + + if (status?.active === true) { + // The transport gave up, not the generation. Keep the active-job + // marker and accepted chips intact, then release this stale + // submission so useResumeOnLoad can establish a fresh attachment. + host.cache.setStreamStatus(recoveryConvoId, status); + restoreLiveStatus(ctx, recoveryConvoId, status, status.createdAt, generationProtocolVersion); + host.pane.setIsSubmitting(true); + host.pane.setShowStopButton(generationCreatedAt != null); + run.reconnectAttempt = 0; + sub.scheduleReattach(30_000); + return; + } + + if (status == null) { + // An inconclusive status read is still not proof of termination. + // Preserve server/client ownership and let the normal resume-on-load + // status query retry rather than duplicating accepted words. + host.cache.invalidateStreamStatus(recoveryConvoId); + host.pane.setIsSubmitting(true); + host.pane.setShowStopButton(generationCreatedAt != null); + run.reconnectAttempt = 0; + sub.scheduleReattach(30_000); + return; + } + + // The status endpoint proved the job terminal. Reconcile persisted + // content before recovering leftovers, just like the 404 path. + run.cancelSteerRetryFrames(); + let persistedMessages: TMessage[] | undefined; + try { + await host.cache.invalidateMessages(recoveryConvoId, 'none'); + if (!sub.isCurrent()) { + return; + } + const fetched = await host.cache.fetchMessages(recoveryConvoId); + if (!sub.isCurrent()) { + return; + } + if (Array.isArray(fetched)) { + persistedMessages = fetched; + } + } catch (error) { + if (!sub.isCurrent()) { + return; + } + host.logger.warn('ResumableSSE', 'Could not reconcile terminal messages after disconnect', { + conversationId: recoveryConvoId, + error, + }); + } + + const authoritativeValues = [ + ...(persistedMessages ?? []), + ...(status.resumeState?.aggregatedContent ?? status.aggregatedContent ?? []), + ]; + const confirmedV2Terminal = + generationProtocolVersion === GENERATION_PROTOCOL_VERSION && + supportsGenerationProtocolV2(status); + const reconciledIds = new Set(host.steers.appliedIds(authoritativeValues)); + host.steers.settleApplied(recoveryConvoId, authoritativeValues); + const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal; + const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : []; + if (unrecovered.length > 0) { + const recoverySteerId = getRecoverySteerId(sub.submission); + host.steers.convertToQueued(recoveryConvoId, unrecovered, { + generationProtocolVersion, + ...(recoverySteerId != null && { + allowPreviouslyConvertedIds: [recoverySteerId], + }), + }); + for (const steer of unrecovered) { + reconciledIds.add(steer.steerId); + if (steer.clientSteerId) { + reconciledIds.add(steer.clientSteerId); + } + } + } + if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) { + host.steers.convertLocalToQueued(recoveryConvoId, { + excludeSteerIds: reconciledIds, + generationProtocolVersion, + }); + } + + host.events.resetLive(sub.liveSubmission()); + host.cache.removeActiveJob(currentStreamId); + clearAttachedGenerationCreatedAt(ctx); + clearPendingApprovalForTerminal(ctx, recoveryConvoId); + if (!run.createdStreamIds.has(currentStreamId) && run.optimisticStreamIds.has(currentStreamId)) { + host.cache.removeConversation(currentStreamId); + } + sub.retired = true; + host.pane.setIsSubmitting(false); + host.pane.setShowStopButton(false); + let recoveryOutcome: 'completed' | 'aborted' | 'error' = 'error'; + if (status.status === 'complete' && persistedMessages != null) { + recoveryOutcome = 'completed'; + } else if (status.status === 'aborted') { + recoveryOutcome = 'aborted'; + } + const recoveredResponseMessageId = + recoveryOutcome === 'completed' + ? completedResponseMessageId( + persistedMessages, + sub.userMessage?.messageId, + status?.resumeState?.responseMessageId ?? sub.submission.initialResponse?.messageId, + ) + : undefined; + host.pane.publishRunEnd({ + conversationId: recoveryConvoId, + outcome: recoveryOutcome, + endedAt: run.scheduler.now(), + generationCreatedAt: status.createdAt ?? generationCreatedAt, + ...(recoveredResponseMessageId != null && { + responseMessageId: recoveredResponseMessageId, + }), + }); + host.pane.setSubmission(null); + host.pane.setStreamId(null); + run.releaseStream(currentStreamId); + run.reconnectAttempt = 0; +} diff --git a/packages/chat/src/core/resumable/failure.ts b/packages/chat/src/core/resumable/failure.ts new file mode 100644 index 0000000000..c423df933d --- /dev/null +++ b/packages/chat/src/core/resumable/failure.ts @@ -0,0 +1,209 @@ +import { ErrorTypes, ViolationTypes } from 'librechat-data-provider'; +import type { ChatEvent, ChatErrorData } from 'librechat-data-provider'; +import type { TerminalContext } from './terminal'; +import { + handoffToReplacement, + authorizeTerminalTeardown, + clearPendingApprovalForTerminal, + clearAttachedGenerationCreatedAt, +} from './terminal'; +import { GENERATION_PROTOCOL_VERSION, supportsGenerationProtocolV2 } from '../protocol'; +import { MAX_RETRIES, getReconnectDelay, getRecoverySteerId } from './start'; +import { settleAfterRetryCeiling } from './ceiling'; +import { recoverMissingStream } from './missing'; + +/** + * Server-sent error event (event: error with data) - no responseCode. + * These are known errors (ErrorTypes, ViolationTypes) that should be displayed to user. + */ +async function handleServedError( + ctx: TerminalContext, + data: ChatErrorData | string, +): Promise { + const { sub, host } = ctx; + const { run, generationCreatedAt, generationProtocolVersion } = sub; + const currentStreamId = sub.streamId; + sub.finalReceived = true; + const recoveryConvoId = sub.submission.conversation?.conversationId ?? currentStreamId; + if (!(await authorizeTerminalTeardown(ctx, recoveryConvoId, 'error')) || !sub.isCurrent()) { + return; + } + host.logger.log('ResumableSSE', 'Server-sent error event received:', data); + run.cancelSteerRetryFrames(); + sub.close(); + /** FLUSH (not cancel): the error card below is built from the cache + * tail, so queued tokens must land first, and a stale trailing + * frame must never overwrite the error write. */ + host.events.flushPendingDeltas(); + host.cache.removeActiveJob(currentStreamId); + clearAttachedGenerationCreatedAt(ctx); + clearPendingApprovalForTerminal(ctx, recoveryConvoId); + host.events.resetLive(sub.liveSubmission()); + if (!run.createdStreamIds.has(currentStreamId) && run.optimisticStreamIds.has(currentStreamId)) { + host.cache.removeConversation(currentStreamId); + } + + const errorSupportsV2 = supportsGenerationProtocolV2(data); + const errorString = + typeof data === 'string' ? data : (data.error ?? data.message ?? JSON.stringify(data)); + + // Check if it's a known error type (ViolationTypes or ErrorTypes) + let isKnownError = false; + try { + const parsed = typeof errorString === 'string' ? JSON.parse(errorString) : errorString; + const errorType = parsed?.type ?? parsed?.code; + if (errorType) { + const violationValues = Object.values(ViolationTypes) as string[]; + const errorTypeValues = Object.values(ErrorTypes) as string[]; + isKnownError = violationValues.includes(errorType) || errorTypeValues.includes(errorType); + } + } catch { + // Not JSON or parsing failed - treat as generic error + } + + host.logger.log('ResumableSSE', 'Error type check:', { isKnownError, errorString }); + + // Display the error to user via errorHandler + host.events.showError(errorString, sub.submission); + + host.pane.setIsSubmitting(false); + host.pane.setShowStopButton(false); + const recoverySteerId = getRecoverySteerId(sub.submission); + if ( + recoverySteerId != null && + generationProtocolVersion === GENERATION_PROTOCOL_VERSION && + errorSupportsV2 + ) { + try { + /** The error was received on the exact generation-fenced stream. + * Read status only now, after terminal was observed: a claim + * started before this recovery could return a stale parked copy + * after a successful commit and must never authorize redelivery. */ + const terminalStatus = await host.fetchStatus(recoveryConvoId); + if (!sub.isCurrent()) { + return; + } + const releasedSource = terminalStatus.unrecoveredSteers?.find( + (steer) => steer.steerId === recoverySteerId, + ); + if ( + terminalStatus.active === false && + supportsGenerationProtocolV2(terminalStatus) && + releasedSource != null + ) { + host.steers.convertToQueued(recoveryConvoId, [releasedSource], { + generationProtocolVersion, + allowPreviouslyConvertedIds: [recoverySteerId], + }); + } + } catch (error) { + if (!sub.isCurrent()) { + return; + } + host.logger.warn('ResumableSSE', 'Could not recover source after generation error', { + conversationId: recoveryConvoId, + error, + }); + } + } + // The error terminal's backstop parks acked leftovers server-side; + // claim it now so a reload can't resurrect the chips converted here. + if (generationProtocolVersion === 1 || errorSupportsV2) { + host.steers.convertLocalToQueued(recoveryConvoId, { + claimParked: true, + generationProtocolVersion, + }); + } + host.pane.publishRunEnd({ + conversationId: recoveryConvoId, + outcome: 'error', + endedAt: run.scheduler.now(), + generationCreatedAt, + }); + host.pane.setStreamId(null); + run.releaseStream(currentStreamId); + run.reconnectAttempt = 0; +} + +/** + * Error event handler - handles BOTH: + * 1. HTTP-level errors (status present) - 404, 409, network failures (0) + * 2. Server-sent error events (event: error with data) - known errors like ViolationTypes/ErrorTypes + * + * Order matters: check the status first since HTTP errors may also include data. + * The transport has already retried a 401 with a refreshed token. + */ +export async function handleTransportFailure( + ctx: TerminalContext, + { status: responseCode, data }: Pick, 'status' | 'data'>, +): Promise { + const { sub, host } = ctx; + const { run, generationCreatedAt } = sub; + if (!sub.isCurrent()) { + return; + } + + if (sub.finalReceived) { + host.logger.log('ResumableSSE', 'Ignoring error after FINAL event', { + responseCode, + hasData: data != null, + }); + return; + } + + host.pane.refreshBalance(); + + if (responseCode === 409) { + const handedOff = await handoffToReplacement(ctx, sub.conversationId()); + if (!sub.isCurrent()) { + return; + } + if (handedOff) { + return; + } + } + + // 404 → job completed & was cleaned up; messages are persisted in DB. + if (responseCode === 404) { + await recoverMissingStream(ctx); + return; + } + + /** + * Only check the data if there's no HTTP status, since HTTP errors may also have body data. + * Note: responseCode === 0 means transport failure (connection dropped) - treat as network error, + * not a server-sent error payload. Use `== null` to only match undefined/null (no HTTP status). + */ + if (responseCode == null && data != null) { + await handleServedError(ctx, data); + return; + } + + // Network failure or unknown HTTP error - attempt reconnection with backoff + host.logger.log('ResumableSSE', 'Stream error (network failure) - will attempt reconnect', { + responseCode, + hasData: data != null, + }); + + if (run.reconnectAttempt < MAX_RETRIES) { + // Increment counter BEFORE close() so abort handler knows we're reconnecting + run.reconnectAttempt++; + const delay = getReconnectDelay(run.reconnectAttempt); + + host.logger.log( + 'ResumableSSE', + `Reconnecting in ${delay}ms (attempt ${run.reconnectAttempt}/${MAX_RETRIES})`, + ); + + sub.close(); + + sub.scheduleReattach(delay); + + // Keep UI in "submitting" state during reconnection attempts + // so user knows we're still trying (abort handler may have reset these) + host.pane.setIsSubmitting(true); + host.pane.setShowStopButton(generationCreatedAt != null); + } else { + await settleAfterRetryCeiling(ctx); + } +} diff --git a/packages/chat/src/core/resumable/host.ts b/packages/chat/src/core/resumable/host.ts index 299a00a9dc..6594ec4fcf 100644 --- a/packages/chat/src/core/resumable/host.ts +++ b/packages/chat/src/core/resumable/host.ts @@ -1,4 +1,10 @@ -import type { Agents, TSubmission, TPendingSteer } from 'librechat-data-provider'; +import type { + Agents, + TMessage, + TSubmission, + TConversation, + TPendingSteer, +} from 'librechat-data-provider'; import type { GenerationProtocolVersion } from '../protocol'; import type { RunEnd } from '../steer/types'; @@ -139,6 +145,13 @@ export interface ResumableCache { invalidateStreamStatus: (conversationId: string) => void; addActiveJob: (streamId: string) => void; removeActiveJob: (streamId: string) => void; + /** Reads the conversation's saved record; rejects with the HTTP failure, a 404 included. */ + fetchConversation: (conversationId: string) => Promise; + setConversation: (conversationId: string, conversation: TConversation) => void; + /** Places the conversation's snapshot into every conversation list. */ + upsertConversation: (conversation: TConversation) => void; + /** Drops the conversation from every conversation list. */ + removeConversation: (conversationId: string) => void; } /** The pane's controls and run bookkeeping a resumable run drives. */ @@ -164,6 +177,18 @@ export interface ResumablePane { publishRunEnd: (end: RunEnd, options?: { retainDetached?: boolean }) => void; /** Consumes an armed "interrupt & send" for exactly this generation. */ clearDrainAfterAbort: (conversationId: string, generationCreatedAt?: number) => void; + /** Refreshes the token balance, when the deployment tracks one. */ + refreshBalance: () => void; + /** Updates the pane's conversation; absent for a pane that only adds a comparison response. */ + updateConversation?: (update: (current: TConversation | null) => TConversation | null) => void; + /** + * The end of a run this pane left mid-stream and is now reattached to, as `messages` record + * it. `resolvable` is false when no such run is remembered or the user stopped it. + */ + resolveDetachedEnd: ( + conversationId: string, + messages: TMessage[] | undefined, + ) => { resolvable: boolean; end: RunEnd | null }; } /** What the event handlers that render the stream expose to the run. */ @@ -172,6 +197,14 @@ export interface ResumableEvents { flushPendingDeltas: () => void; /** Discards the live usage estimate for a run that stopped streaming. */ resetLive: (submission: TSubmission) => void; + /** Writes the failed turn's error card from the text (or raw error value) the server sent. */ + showError: (text: string | boolean | { message?: string }, submission: TSubmission) => void; +} + +/** The pane's message list. */ +export interface ResumableMessages { + get: () => TMessage[] | undefined; + set: (messages: TMessage[]) => void; } /** Everything a resumable run needs from the app it runs in. */ @@ -180,4 +213,5 @@ export interface ResumableHost extends SubscriptionHost { pane: ResumablePane; steers: ResumableSteers; events: ResumableEvents; + messages: ResumableMessages; } diff --git a/packages/chat/src/core/resumable/index.ts b/packages/chat/src/core/resumable/index.ts index 3ba5c14d0a..d94f8f3c90 100644 --- a/packages/chat/src/core/resumable/index.ts +++ b/packages/chat/src/core/resumable/index.ts @@ -8,3 +8,6 @@ export * from './host'; export * from './subscription'; export * from './terminal'; export * from './reconcile'; +export * from './failure'; +export * from './missing'; +export * from './ceiling'; diff --git a/packages/chat/src/core/resumable/missing.ts b/packages/chat/src/core/resumable/missing.ts new file mode 100644 index 0000000000..a6ee87261a --- /dev/null +++ b/packages/chat/src/core/resumable/missing.ts @@ -0,0 +1,242 @@ +import { Constants } from 'librechat-data-provider'; +import type { TMessage } from 'librechat-data-provider'; +import type { TerminalContext } from './terminal'; +import { + restoreLiveStatus, + handoffToReplacement, + clearPendingApprovalForTerminal, + clearAttachedGenerationCreatedAt, +} from './terminal'; +import { GENERATION_PROTOCOL_VERSION, supportsGenerationProtocolV2 } from '../protocol'; +import { isInitialNewConversation, keepLocalCodeApprovalMode } from './messages'; +import { getRecoverySteerId, toStartGenerationError } from './start'; + +/** + * A 404 from the stream route: the job completed and was cleaned up, and its messages are + * persisted. Refetches and reconciles them before recovering local steers (the final response + * may contain an applied steer whose live event was missed), unless a status read shows the + * generation, or a newer one, still running. + */ +export async function recoverMissingStream(ctx: TerminalContext): Promise { + const { sub, host } = ctx; + const { run, generationCreatedAt, generationProtocolVersion } = sub; + const currentStreamId = sub.streamId; + const convoId = sub.submission.conversation?.conversationId; + host.logger.log('ResumableSSE', 'Stream 404, invalidating messages for:', convoId); + // Keep the UI non-idle while the status read decides whether this is + // terminal cleanup or a handoff to a newer generation epoch. + run.reconnectAttempt = Math.max(run.reconnectAttempt, 1); + run.cancelSteerRetryFrames(); + sub.close(); + /** Terminal: drop any in-flight live estimate so the gauge doesn't + * keep counting stale streamed output after the stream ends */ + host.events.resetLive(sub.liveSubmission()); + host.pane.clearComposerDrafts(convoId, { + includeNewChatDraft: + !convoId || + convoId === Constants.NEW_CONVO || + run.optimisticStreamIds.has(currentStreamId) || + isInitialNewConversation(sub.submission), + }); + host.events.clearStepMaps(); + let persistedMessages: TMessage[] | undefined; + if (convoId) { + try { + await host.cache.invalidateMessages(convoId, 'none'); + if (!sub.isCurrent()) { + return; + } + // The first stream can finish before CREATED mounts the saved-chat query. + const fetched = await host.cache.fetchMessages(convoId, true); + if (!sub.isCurrent()) { + return; + } + if (Array.isArray(fetched)) { + persistedMessages = fetched; + } + } catch (error) { + if (!sub.isCurrent()) { + return; + } + host.logger.warn('ResumableSSE', 'Could not reconcile persisted messages after 404', { + conversationId: convoId, + error, + }); + } + host.cache.removeStreamStatus(convoId); + } + const recoveryConvoId = convoId ?? currentStreamId; + const reconciledIds = new Set(persistedMessages ? host.steers.appliedIds(persistedMessages) : []); + if (persistedMessages) { + host.steers.settleApplied(recoveryConvoId, persistedMessages); + } + + // The status replay is owner-gated and non-destructive; creating a + // recovered follow-up consumes its exact source. Prefer these + // authoritative leftovers over guessing from local chips. + let confirmedV2Terminal = false; + try { + const status = await host.fetchStatus(recoveryConvoId); + if (!sub.isCurrent()) { + return; + } + if ( + generationProtocolVersion === GENERATION_PROTOCOL_VERSION && + !supportsGenerationProtocolV2(status) + ) { + sub.scheduleReattach(1_000); + return; + } + if ( + status.active === true && + status.createdAt != null && + (generationCreatedAt == null || status.createdAt !== generationCreatedAt) + ) { + const handedOff = await handoffToReplacement(ctx, recoveryConvoId, status); + if (!sub.isCurrent()) { + return; + } + if (handedOff) { + return; + } + /** Legacy generations have no replacement-handoff contract. + * Never attach this submission to a different epoch. */ + sub.scheduleReattach(1_000); + return; + } + if (status.active === true) { + /** A stream lookup can briefly lose its routing record while the + * authoritative job status still proves the same generation is + * alive. Preserve ownership and retry the attachment instead of + * publishing a terminal run-end (which could auto-drain queued + * text into a still-running generation). */ + const activeStreamId = status.streamId ?? currentStreamId; + host.cache.setStreamStatus(recoveryConvoId, status); + if (status.createdAt != null) { + host.pane.setGenerationEpoch( + recoveryConvoId, + status.createdAt, + undefined, + generationProtocolVersion, + ); + } + host.cache.addActiveJob(activeStreamId); + restoreLiveStatus(ctx, recoveryConvoId, status, status.createdAt, generationProtocolVersion); + host.pane.setIsSubmitting(true); + host.pane.setShowStopButton(status.createdAt != null); + run.reconnectAttempt = Math.max(run.reconnectAttempt, 1); + sub.scheduleReattach(1_000, activeStreamId, status.createdAt); + return; + } + confirmedV2Terminal = + generationProtocolVersion === GENERATION_PROTOCOL_VERSION && + supportsGenerationProtocolV2(status); + const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal; + const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : []; + if (unrecovered.length > 0) { + const recoverySteerId = getRecoverySteerId(sub.submission); + host.steers.convertToQueued(recoveryConvoId, unrecovered, { + generationProtocolVersion, + ...(recoverySteerId != null && { + allowPreviouslyConvertedIds: [recoverySteerId], + }), + }); + for (const steer of unrecovered) { + reconciledIds.add(steer.steerId); + if (steer.clientSteerId) { + reconciledIds.add(steer.clientSteerId); + } + } + } + } catch (error) { + if (!sub.isCurrent()) { + return; + } + host.logger.warn('ResumableSSE', 'Could not recover parked steers after 404', { + conversationId: recoveryConvoId, + error, + }); + } + + host.cache.removeActiveJob(currentStreamId); + clearAttachedGenerationCreatedAt(ctx); + clearPendingApprovalForTerminal(ctx, recoveryConvoId); + if (!run.createdStreamIds.has(currentStreamId) && run.optimisticStreamIds.has(currentStreamId)) { + // Both fresh and resumed subscriptions can miss every event of a fast turn. + // A missing job proves neither that the conversation was saved nor that it failed. + try { + const persisted = await host.cache.fetchConversation(recoveryConvoId); + if (!sub.isCurrent()) return; + if (persisted?.conversationId === recoveryConvoId) { + host.cache.setConversation(recoveryConvoId, persisted); + host.cache.upsertConversation(persisted); + host.pane.updateConversation?.((current) => { + if ( + current?.conversationId != null && + current.conversationId !== Constants.NEW_CONVO && + current.conversationId !== recoveryConvoId + ) + return current; + return keepLocalCodeApprovalMode( + { ...current, ...persisted }, + current, + current?.conversationId, + ); + }); + } + } catch (error) { + if (!sub.isCurrent()) return; + if (toStartGenerationError(error)?.response?.status === 404) { + host.cache.removeConversation(currentStreamId); + } + } + // An inconclusive read must not turn a saved chat into a phantom. + void host.cache.refreshConversationLists(); + void host.cache.refreshPinnedConversations(); + } + if (persistedMessages) { + host.messages.set(persistedMessages); + } + sub.retired = true; + host.pane.setIsSubmitting(false); + host.pane.setShowStopButton(false); + + // Only infer recovery from local accepted chips after the persisted + // final was successfully inspected. If that fetch failed, their + // applied-vs-leftover outcome is unknown and converting could send + // already-applied words twice. + if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) { + host.steers.convertLocalToQueued(recoveryConvoId, { + excludeSteerIds: reconciledIds, + generationProtocolVersion, + }); + } + // The true outcome is unknown here (job record already cleaned up): + // a non-'completed' outcome releases parked interrupt flags without + // auto-sending queued messages the user may not want fired. A run this + // pane left and came back to is the exception: the history just read + // names its response by epoch, which is the outcome it would have had. + // History read before the job finished may not hold that response yet, or + // may hold only the unfinished snapshot saved when the pane left, so a + // marker that read leaves unresolved or stopped outlives this end for the + // next visit, which reads history after the job is known to be done. + const detached = host.pane.resolveDetachedEnd(recoveryConvoId, persistedMessages); + const detachedResolvable = detached.resolvable; + const detachedEnd = detached.end; + host.pane.publishRunEnd( + { + conversationId: recoveryConvoId, + outcome: 'aborted', + endedAt: run.scheduler.now(), + ...detachedEnd, + startedAsNewConvo: run.optimisticStreamIds.has(currentStreamId), + }, + { + retainDetached: + detachedResolvable && (detachedEnd == null || detachedEnd.outcome === 'aborted'), + }, + ); + host.pane.setStreamId(null); + run.releaseStream(currentStreamId); + run.reconnectAttempt = 0; +}