diff --git a/client/src/hooks/SSE/__tests__/useResumableSSE.spec.ts b/client/src/hooks/SSE/__tests__/useResumableSSE.spec.ts index 6a2c5538ba..0c3227d47d 100644 --- a/client/src/hooks/SSE/__tests__/useResumableSSE.spec.ts +++ b/client/src/hooks/SSE/__tests__/useResumableSSE.spec.ts @@ -1,3 +1,4 @@ +import { createElement } from 'react'; import { getDefaultStore } from 'jotai'; import { QueryClient } from '@tanstack/react-query'; import { renderHook, act, waitFor } from '@testing-library/react'; @@ -12,6 +13,7 @@ import { } from 'librechat-data-provider'; import type { TMessage, TSubmission } from 'librechat-data-provider'; import type { Query, QueryKey } from '@tanstack/react-query'; +import type { ReactNode } from 'react'; import { activeUsageResponseIdFamily, liveTokensFamily, @@ -315,6 +317,8 @@ import useResumableSSE, { } from '~/hooks/SSE/useResumableSSE'; import useSSE from '~/hooks/SSE/useSSE'; import type { PendingSteer } from '@librechat/chat'; +import type { Transport } from '~/hooks/Chat/contract'; +import { ChatTransportContext, defaultChatTransport } from '~/Providers/ChatTransportContext'; import { recoveryDispositionsFamily, queuedMessagesByConvoId, @@ -627,6 +631,25 @@ describe('useResumableSSE', () => { }); }); + it('calls a host transport start on the transport itself', async () => { + const receivers: unknown[] = []; + const hostTransport: Transport = { + ...defaultChatTransport, + start(this: Transport, startRequest, options) { + receivers.push(this); + return defaultChatTransport.start(startRequest, options); + }, + }; + const wrapper = ({ children }: { children: ReactNode }) => + createElement(ChatTransportContext.Provider, { value: hostTransport }, children); + const { unmount } = renderHook(() => useResumableSSE(buildSubmission(), buildChatHelpers()), { + wrapper, + }); + await waitFor(() => expect(receivers).toHaveLength(1)); + expect(receivers[0]).toBe(hostTransport); + unmount(); + }); + it.each([useSSE, useResumableSSE])( '%p retains authoritative legacy text response ownership for subsequent usage events', async (useTransport) => { diff --git a/client/src/hooks/SSE/useResumableSSE.ts b/client/src/hooks/SSE/useResumableSSE.ts index d850b124c2..aa8afd77c0 100644 --- a/client/src/hooks/SSE/useResumableSSE.ts +++ b/client/src/hooks/SSE/useResumableSSE.ts @@ -3,6 +3,7 @@ import { v4 } from 'uuid'; import { useSetRecoilState } from 'recoil'; import { useStore, useSetAtom } from 'jotai'; import { useQueryClient } from '@tanstack/react-query'; +import { Constants, QueryKeys, dataService } from 'librechat-data-provider'; import { ABORT_SWEEP_STATUSES, selectLocalSteersForQueue } from '@librechat/chat'; import { resolveDetachedRunEnd, @@ -10,13 +11,6 @@ import { canRestoreRecovery, blockRecovery, } from '@librechat/chat'; -import { - Constants, - QueryKeys, - dataService, - createPayload, - removeNullishValues, -} from 'librechat-data-provider'; import { submissionByIndex, liveAppliedSteerIdsAtom, @@ -34,44 +28,27 @@ import { } from '@librechat/chat/react'; import { toStartGenerationError, - waitForRetryDelay, - getRecoverySteerId, - getRetryAfterDelay, - isServerNotReadyError, getOptimisticMessages, - isResumedStartResponse, - isSettledStartResponse, - isRetryableNetworkError, - isReplacedStartResponse, isInitialNewConversation, getStreamStartFailureText, keepLocalCodeApprovalMode, hydrateSubmissionMessages, - getStartGenerationStreamId, buildOptimisticConversation, - getGenerationProtocolVersion, GENERATION_PROTOCOL_VERSION, - getStartGenerationCreatedAt, - isUnnegotiatedV2StartControl, supportsGenerationProtocolV2, - START_GENERATION_NETWORK_RETRIES, - getStartGenerationConversationId, - getPredecessorMismatchStartResponse, - START_GENERATION_READINESS_TIMEOUT_MS, ResumableRun, documentVisibility, attachResumableStream, + startGeneration, } from '@librechat/chat'; import type { ResumableHost, ResumableCache, SubscriptionTarget, - StartGenerationResult, GenerationProtocolVersion, } from '@librechat/chat'; import type { TMessage, - TPayload, TSubmission, TConversation, TPendingSteer, @@ -435,117 +412,137 @@ export default function useResumableSSE( * Follows same auth pattern as useSSE * @param isResume - If true, adds ?resume=true to trigger sync event from server */ - const subscribeToStream = useCallback( - (target: SubscriptionTarget) => { - const host: ResumableHost = { - logger, - visibility: documentVisibility, - fetchStatus: fetchStreamStatus, - subscribe: (next) => subscribeToStream(next), - cache: resumableCache, - pane: { - setIsSubmitting, - setShowStopButton, - setSubmission, - setStreamId, - setGenerationEpoch: updateActiveGenerationCreatedAt, - setPendingAction: (conversationId, action) => - jotaiStore.set(pendingApprovalActionFamily(conversationId), action), - clearComposerDrafts: (conversationId, options) => - clearComposerDrafts(runIndex, conversationId, options), - publishRunEnd: setRunEnd, - setAbortScroll, - renewActiveRun: () => setActiveRunId(v4()), - runEndTarget: resolveRunEndTarget, - clearDrainAfterAbort, - refreshBalance: () => { - (startupConfig?.balance?.enabled ?? false) && balanceQuery.refetch(); - }, - updateConversation: isAddedRequest ? undefined : setConversation, - resolveDetachedEnd: (conversationId, messages) => { - const detachedRun = jotaiStore.get(detachedRunByConvoId(conversationId)); - const resolvable = - detachedRun != null && - !isDetachedRunManual(jotaiStore.get, conversationId, detachedRun); - return { - resolvable, - end: - resolvable && messages != null - ? resolveDetachedRunEnd(conversationId, detachedRun, messages) - : null, + const buildHost = useCallback( + (): ResumableHost => ({ + logger, + visibility: documentVisibility, + fetchStatus: fetchStreamStatus, + subscribe: (next) => attachResumableStream(run, buildHost(), next), + cache: resumableCache, + pane: { + setIsSubmitting, + setShowStopButton, + setSubmission, + setStreamId, + setGenerationEpoch: updateActiveGenerationCreatedAt, + setPendingAction: (conversationId, action) => + jotaiStore.set(pendingApprovalActionFamily(conversationId), action), + clearComposerDrafts: (conversationId, options) => + clearComposerDrafts(runIndex, conversationId, options), + publishRunEnd: setRunEnd, + setAbortScroll, + renewActiveRun: () => setActiveRunId(v4()), + runEndTarget: resolveRunEndTarget, + clearDrainAfterAbort, + refreshBalance: () => { + (startupConfig?.balance?.enabled ?? false) && balanceQuery.refetch(); + }, + updateConversation: isAddedRequest ? undefined : setConversation, + resolveDetachedEnd: (conversationId, messages) => { + const detachedRun = jotaiStore.get(detachedRunByConvoId(conversationId)); + const resolvable = + detachedRun != null && + !isDetachedRunManual(jotaiStore.get, conversationId, detachedRun); + return { + resolvable, + end: + resolvable && messages != null + ? resolveDetachedRunEnd(conversationId, detachedRun, messages) + : null, + }; + }, + }, + steers: { + resolve: resolveSteerChip, + isRecoveryHeld: (conversationId, recoverySteerId) => + jotaiStore.get(recoveryDispositionsFamily(conversationId))[recoverySteerId] != null, + blockRecovery: (conversationId, recoverySteerId) => + jotaiStore.set(recoveryDispositionsFamily(conversationId), (previous) => + blockRecovery(previous, recoverySteerId), + ), + restoreQueued: restoreQueuedSubmission, + update: updateSteerChips, + markLiveApplied: (steerId) => + setLiveAppliedSteerIds((prev) => appendAppliedSteerIds(prev, [steerId])), + seed: seedSteerChips, + appliedIds: collectAppliedSteerIds, + settleApplied: settleAppliedSteerParts, + convertToQueued: convertSteersToQueued, + convertLocalToQueued: convertLocalSteersToQueued, + }, + events: { + stepHandler, + showStartFailure: (failedSubmission, errorData, reason) => { + let text: string | undefined; + if (reason === 'recovery_held') { + text = localize('com_ui_steer_recovery_held'); + } else if (reopenTermsOnRefusal(queryClient, errorData)) { + text = localize('com_error_terms_not_accepted'); + } + errorHandler({ + data: getStreamStartFailureData(errorData, text), + submission: failedSubmission as EventSubmission, + }); + }, + finalHandler: (data, finalSubmission) => + finalHandler(data as TFinalResData, finalSubmission), + createdHandler, + attachmentHandler, + titleHandler, + contentHandler, + messageHandler, + prunePtcTraces, + resetContentHandler, + syncStepMessage, + clearStepMaps, + flushPendingDeltas, + resetLive, + showError: (text, errorSubmission) => + errorHandler({ + data: { text } as unknown as Parameters[0]['data'], + submission: errorSubmission as EventSubmission, + }), + }, + messages: { get: getMessages, set: setMessages }, + usage: { + bindResponse, + contextHandler, + usageHandler, + tapStream, + tapContent, + finalizeUsage, + backfillUsage, + seedLiveFromContent: (content, usageSubmission) => + seedLive(countTrailingOutputChars(content), usageSubmission), + }, + stream: transport.stream({ token }), + start: (request, options) => transport.start(request, options), + exposeDebugControls: import.meta.env.DEV + ? ({ killNetwork, closeClean }) => { + // Debug hooks for testing reconnection vs clean close behavior (dev only) + const debugWindow = window as Window & { + __killNetwork?: () => void; + __closeClean?: () => void; }; - }, - }, - steers: { - resolve: resolveSteerChip, - update: updateSteerChips, - markLiveApplied: (steerId) => - setLiveAppliedSteerIds((prev) => appendAppliedSteerIds(prev, [steerId])), - seed: seedSteerChips, - appliedIds: collectAppliedSteerIds, - settleApplied: settleAppliedSteerParts, - convertToQueued: convertSteersToQueued, - convertLocalToQueued: convertLocalSteersToQueued, - }, - events: { - stepHandler, - finalHandler: (data, finalSubmission) => - finalHandler(data as TFinalResData, finalSubmission), - createdHandler, - attachmentHandler, - titleHandler, - contentHandler, - messageHandler, - prunePtcTraces, - resetContentHandler, - syncStepMessage, - clearStepMaps, - flushPendingDeltas, - resetLive, - showError: (text, errorSubmission) => - errorHandler({ - data: { text } as unknown as Parameters[0]['data'], - submission: errorSubmission as EventSubmission, - }), - }, - messages: { get: getMessages, set: setMessages }, - usage: { - bindResponse, - contextHandler, - usageHandler, - tapStream, - tapContent, - finalizeUsage, - backfillUsage, - seedLiveFromContent: (content, usageSubmission) => - seedLive(countTrailingOutputChars(content), usageSubmission), - }, - stream: transport.stream({ token }), - exposeDebugControls: import.meta.env.DEV - ? ({ killNetwork, closeClean }) => { - // Debug hooks for testing reconnection vs clean close behavior (dev only) - const debugWindow = window as Window & { - __killNetwork?: () => void; - __closeClean?: () => void; - }; - debugWindow.__killNetwork = killNetwork; - debugWindow.__closeClean = closeClean; - } - : undefined, - parts: { - applySteer: applySteerPart, - findSteerTarget: findSteerMessageIndex, - applyActivityLabel: applyActivityLabelPart, - findActivityLabelTarget: findActivityLabelMessageIndex, - offsetActivityPhaseBoundary, - applyReasoningLabel, - findReasoningLabelTarget: findReasoningLabelMessageIndex, - }, - }; - attachResumableStream(run, host, target); - }, + debugWindow.__killNetwork = killNetwork; + debugWindow.__closeClean = closeClean; + } + : undefined, + parts: { + applySteer: applySteerPart, + findSteerTarget: findSteerMessageIndex, + applyActivityLabel: applyActivityLabelPart, + findActivityLabelTarget: findActivityLabelMessageIndex, + offsetActivityPhaseBoundary, + applyReasoningLabel, + findReasoningLabelTarget: findReasoningLabelMessageIndex, + }, + }), [ run, + localize, + queryClient, + restoreQueuedSubmission, resumableCache, isAddedRequest, setConversation, @@ -597,260 +594,9 @@ export default function useResumableSSE( ], ); - /** - * Start generation (POST request that returns streamId) - * Posts through the host transport, which owns auth refresh. - * Retries transient network failures and startup readiness responses. - * Readiness retries honor Retry-After until cleanup or the readiness window expires. - */ - const startGeneration = useCallback( - async ( - currentSubmission: TSubmission, - signal?: AbortSignal, - ): Promise => { - const payloadData = createPayload(currentSubmission); - let { payload } = payloadData; - payload = removeNullishValues(payload) as TPayload; - - clearStepMaps(); - - const url = payloadData.server; - - let lastError: unknown = null; - let requestAttempts = 0; - let networkAttempts = 0; - let readinessAttempts = 0; - const readinessDeadline = run.scheduler.now() + START_GENERATION_READINESS_TIMEOUT_MS; - - while (!signal?.aborted) { - const recoverySteerId = getRecoverySteerId(currentSubmission); - const conversationId = currentSubmission.conversation?.conversationId; - if ( - recoverySteerId != null && - conversationId && - jotaiStore.get(recoveryDispositionsFamily(conversationId))[recoverySteerId] != null - ) { - restoreQueuedSubmission(currentSubmission); - errorHandler({ - data: getStreamStartFailureData(localize('com_ui_steer_recovery_held')), - submission: currentSubmission as EventSubmission, - }); - setShowStopButton(false); - setIsSubmitting(false); - setSubmission(null); - return null; - } - requestAttempts += 1; - try { - const data = await transport.start({ server: url, payload }, { signal }); - if (signal?.aborted) { - return null; - } - if (isSettledStartResponse(data)) { - const conversationId = getStartGenerationConversationId(data); - if (conversationId) { - logger.log('ResumableSSE', 'Generation already settled:', { conversationId }); - return { - status: 'settled', - conversationId, - generationProtocolVersion: GENERATION_PROTOCOL_VERSION, - }; - } - lastError = { response: { data } }; - break; - } - if (isReplacedStartResponse(data)) { - const streamId = getStartGenerationStreamId(data); - const conversationId = getStartGenerationConversationId(data); - const generationCreatedAt = getStartGenerationCreatedAt(data); - if (streamId && conversationId && generationCreatedAt != null) { - return { - status: 'replaced', - streamId, - conversationId, - generationCreatedAt, - generationProtocolVersion: GENERATION_PROTOCOL_VERSION, - }; - } - lastError = { response: { data } }; - break; - } - if (isUnnegotiatedV2StartControl(data)) { - /** Never reinterpret a v2-only control word as a legacy stream - * response merely because it also carries a stream id. */ - lastError = { response: { data } }; - break; - } - const streamId = getStartGenerationStreamId(data); - if (streamId) { - const resumed = isResumedStartResponse(data); - const generationCreatedAt = getStartGenerationCreatedAt(data); - const generationProtocolVersion = getGenerationProtocolVersion(data); - logger.log('ResumableSSE', 'Generation started:', { - streamId, - resumed, - generationCreatedAt, - generationProtocolVersion, - }); - return { - status: 'stream', - streamId, - resumed, - generationProtocolVersion, - ...(generationCreatedAt != null && { generationCreatedAt }), - }; - } - - lastError = { response: { data } }; - break; - } catch (error) { - if (signal?.aborted) { - return null; - } - - const predecessorMismatch = getPredecessorMismatchStartResponse(error); - if (predecessorMismatch != null) { - /** Admission was atomically rejected before this queued turn - * created anything. Put the exact row back immediately and let - * the caller reconcile or hand off to the generation that won. */ - restoreQueuedSubmission(currentSubmission); - logger.log('ResumableSSE', 'Queued generation lost predecessor fence', { - expectedPredecessorCreatedAt: currentSubmission.expectedPredecessorCreatedAt, - ...predecessorMismatch, - }); - return { - status: 'predecessor_mismatch', - ...predecessorMismatch, - generationProtocolVersion: GENERATION_PROTOCOL_VERSION, - }; - } - - lastError = error; - const isNetworkError = isRetryableNetworkError(error); - const isServerNotReady = isServerNotReadyError(error); - const remainingReadinessMs = readinessDeadline - run.scheduler.now(); - const shouldRetryNetwork = - isNetworkError && networkAttempts < START_GENERATION_NETWORK_RETRIES - 1; - const shouldRetryServerNotReady = isServerNotReady && remainingReadinessMs > 0; - - if (shouldRetryNetwork || shouldRetryServerNotReady) { - networkAttempts += isNetworkError ? 1 : 0; - readinessAttempts += isServerNotReady ? 1 : 0; - const fallbackDelay = Math.min(1000 * Math.pow(2, requestAttempts - 1), 8000); - const retryDelay = isServerNotReady - ? Math.min(getRetryAfterDelay(error, fallbackDelay), remainingReadinessMs) - : fallbackDelay; - const reason = isServerNotReady ? 'Server not ready' : 'Network error'; - const attempt = isServerNotReady ? readinessAttempts : networkAttempts; - const limit = isServerNotReady - ? `${Math.ceil(START_GENERATION_READINESS_TIMEOUT_MS / 1000)}s readiness window` - : `${START_GENERATION_NETWORK_RETRIES}`; - logger.log( - 'ResumableSSE', - `${reason} starting generation, retrying in ${retryDelay}ms (attempt ${attempt}/${limit})`, - ); - const shouldContinue = await waitForRetryDelay(retryDelay, signal); - if (!shouldContinue) { - return null; - } - continue; - } - - // Don't retry: either not a network error or max retries reached - break; - } - } - - if (signal?.aborted) { - return null; - } - - logger.error('ResumableSSE', 'Error starting generation:', lastError); - - const startError = toStartGenerationError(lastError); - const errorData = startError?.response?.data; - const responseStatus = startError?.response?.status; - if (responseStatus != null && responseStatus >= 400 && responseStatus < 500) { - const recoverySteerId = getRecoverySteerId(currentSubmission); - const conversationId = currentSubmission.conversation?.conversationId; - const recoveryRejected = - errorData != null && - typeof errorData === 'object' && - 'code' in errorData && - (errorData.code === 'RECOVERY_PAYLOAD_MISMATCH' || - errorData.code === 'INVALID_RECOVERY_REQUEST'); - if (recoveryRejected && recoverySteerId != null && conversationId) { - jotaiStore.set(recoveryDispositionsFamily(conversationId), (previous) => - blockRecovery(previous, recoverySteerId), - ); - } - // The server rejected admission before exposing a generation. Restore - // the exact queue row/position; ambiguous transport/5xx outcomes must - // first reconcile durable state instead of risking a duplicate start. - restoreQueuedSubmission(currentSubmission); - } else { - const recoverySteerId = getRecoverySteerId(currentSubmission); - const recoveryConvoId = currentSubmission.conversation?.conversationId; - if (recoverySteerId != null && recoveryConvoId) { - try { - const status = await fetchStreamStatus(recoveryConvoId); - if (signal?.aborted) { - return null; - } - const releasedSource = status.unrecoveredSteers?.find( - (steer) => steer.steerId === recoverySteerId, - ); - if ( - status.active === false && - supportsGenerationProtocolV2(status) && - releasedSource != null - ) { - convertSteersToQueued(recoveryConvoId, [releasedSource], { - generationProtocolVersion: GENERATION_PROTOCOL_VERSION, - allowPreviouslyConvertedIds: [recoverySteerId], - }); - } - } catch (error) { - if (signal?.aborted) { - return null; - } - logger.warn('ResumableSSE', 'Could not recover source after start failure', { - conversationId: recoveryConvoId, - error, - }); - } - } - } - if (signal?.aborted) { - return null; - } - const termsRefused = reopenTermsOnRefusal(queryClient, errorData); - errorHandler({ - data: getStreamStartFailureData( - errorData, - termsRefused ? localize('com_error_terms_not_accepted') : undefined, - ), - submission: currentSubmission as EventSubmission, - }); - setShowStopButton(false); - setIsSubmitting(false); - setSubmission(null); - return null; - }, - [ - run, - clearStepMaps, - convertSteersToQueued, - errorHandler, - jotaiStore, - localize, - queryClient, - restoreQueuedSubmission, - setIsSubmitting, - setShowStopButton, - setSubmission, - transport, - ], + const subscribeToStream = useCallback( + (target: SubscriptionTarget) => attachResumableStream(run, buildHost(), target), + [buildHost, run], ); useEffect(() => { @@ -952,7 +698,7 @@ export default function useResumableSSE( } else { // New generation: start and then subscribe logger.log('ResumableSSE', 'Starting NEW generation'); - const startResult = await startGeneration(submission, signal); + const startResult = await startGeneration(run, buildHost(), submission, signal); if (!isCurrentEffect()) { return; } diff --git a/packages/chat/src/core/__fixtures__/resumable.ts b/packages/chat/src/core/__fixtures__/resumable.ts index c87f7051e1..d164237cfb 100644 --- a/packages/chat/src/core/__fixtures__/resumable.ts +++ b/packages/chat/src/core/__fixtures__/resumable.ts @@ -110,6 +110,9 @@ export const createHost = ( resolve: jest.fn(() => []), markLiveApplied: jest.fn(), update: jest.fn(), + isRecoveryHeld: jest.fn(() => false), + blockRecovery: jest.fn(), + restoreQueued: jest.fn(), seed: jest.fn(), appliedIds: jest.fn(() => []), settleApplied: jest.fn(), @@ -131,6 +134,7 @@ export const createHost = ( flushPendingDeltas: jest.fn(), resetLive: jest.fn(), showError: jest.fn(), + showStartFailure: jest.fn(), }, messages: { get: jest.fn(() => messages), @@ -158,6 +162,7 @@ export const createHost = ( seedLiveFromContent: jest.fn(), }, stream: { send: jest.fn(), reconnectToStream: jest.fn(() => ({ closed: false })) }, + start: jest.fn(), }; return host; }; diff --git a/packages/chat/src/core/__tests__/launch.spec.ts b/packages/chat/src/core/__tests__/launch.spec.ts new file mode 100644 index 0000000000..5ed9214e0c --- /dev/null +++ b/packages/chat/src/core/__tests__/launch.spec.ts @@ -0,0 +1,111 @@ +import type { TMessage, TSubmission } from 'librechat-data-provider'; +import { createHost, createScheduler } from '../__fixtures__/resumable'; +import { ResumableRun, startGeneration } from '../resumable'; + +const submission = { + userMessage: { messageId: 'u-1', conversationId: 'c-1', isCreatedByUser: true, text: 'Hi' }, + initialResponse: { messageId: 'u-1_', conversationId: 'c-1' } as TMessage, + conversation: { conversationId: 'c-1', endpoint: 'agents' }, + endpointOption: { endpoint: 'agents' }, +} as unknown as TSubmission; + +const httpError = (status: number, data: unknown) => + Object.assign(new Error('failed'), { response: { status, data, headers: {} } }); + +const setup = () => { + const host = createHost(); + const run = new ResumableRun(createScheduler().scheduler); + return { host, run }; +}; + +describe('startGeneration', () => { + it('resolves to the stream a start names', async () => { + const { host, run } = setup(); + (host.start as jest.Mock).mockResolvedValue({ + streamId: 's-1', + generationCreatedAt: 1000, + generationProtocolVersion: 2, + }); + await expect(startGeneration(run, host, submission)).resolves.toEqual({ + status: 'stream', + streamId: 's-1', + resumed: false, + generationCreatedAt: 1000, + generationProtocolVersion: 2, + }); + expect(host.events.clearStepMaps).toHaveBeenCalled(); + }); + + it('reports a settled v2 start', async () => { + const { host, run } = setup(); + (host.start as jest.Mock).mockResolvedValue({ + status: 'settled', + conversationId: 'c-1', + generationProtocolVersion: 2, + }); + await expect(startGeneration(run, host, submission)).resolves.toEqual({ + status: 'settled', + conversationId: 'c-1', + generationProtocolVersion: 2, + }); + }); + + it('restores the queued turn when the predecessor fence was lost', async () => { + const { host, run } = setup(); + (host.start as jest.Mock).mockRejectedValue( + httpError(409, { + status: 'predecessor_mismatch', + code: 'GENERATION_PREDECESSOR_MISMATCH', + streamId: 'c-1', + conversationId: 'c-1', + generationCreatedAt: 3000, + active: true, + }), + ); + await expect(startGeneration(run, host, submission)).resolves.toMatchObject({ + status: 'predecessor_mismatch', + generationCreatedAt: 3000, + }); + expect(host.steers.restoreQueued).toHaveBeenCalledWith(submission); + }); + + it('writes the error card and idles the pane on a rejected start', async () => { + const { host, run } = setup(); + const body = { message: 'forbidden' }; + (host.start as jest.Mock).mockRejectedValue(httpError(403, body)); + await expect(startGeneration(run, host, submission)).resolves.toBeNull(); + expect(host.steers.restoreQueued).toHaveBeenCalledWith(submission); + expect(host.events.showStartFailure).toHaveBeenCalledWith(submission, body); + expect(host.pane.setIsSubmitting).toHaveBeenCalledWith(false); + expect(host.pane.setSubmission).toHaveBeenCalledWith(null); + }); + + it('blocks a recovery the server refused', async () => { + const { host, run } = setup(); + (host.start as jest.Mock).mockRejectedValue( + httpError(400, { code: 'RECOVERY_PAYLOAD_MISMATCH' }), + ); + await startGeneration(run, host, { ...submission, recoverySteerId: 'steer-1' } as TSubmission); + expect(host.steers.blockRecovery).toHaveBeenCalledWith('c-1', 'steer-1'); + }); + + it('refuses a recovery the user already dealt with', async () => { + const { host, run } = setup(); + (host.steers.isRecoveryHeld as jest.Mock).mockReturnValue(true); + const recovery = { ...submission, recoverySteerId: 'steer-1' } as TSubmission; + await expect(startGeneration(run, host, recovery)).resolves.toBeNull(); + expect(host.start).not.toHaveBeenCalled(); + expect(host.events.showStartFailure).toHaveBeenCalledWith(recovery, undefined, 'recovery_held'); + }); + + it('stops quietly once the pane abandons the start', async () => { + const { host, run } = setup(); + const controller = new AbortController(); + (host.start as jest.Mock).mockImplementation(async () => { + controller.abort(); + throw Object.assign(new Error('offline'), { code: 'ERR_NETWORK' }); + }); + await expect(startGeneration(run, host, submission, controller.signal)).resolves.toBeNull(); + expect(host.events.showStartFailure).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/chat/src/core/resumable/host.ts b/packages/chat/src/core/resumable/host.ts index c1fa5df2bc..4397c8c4f3 100644 --- a/packages/chat/src/core/resumable/host.ts +++ b/packages/chat/src/core/resumable/host.ts @@ -6,6 +6,7 @@ import type { TPendingSteer, TAttachment, ChatTransport, + ChatTransportRequest, ChatTitleFrame, ChatFinalFrame, EventSubmission, @@ -127,6 +128,12 @@ export interface ResumableSteers { markLiveApplied: (steerId: string) => void; /** Patches the chips the server re-labelled (preempt capability); never removes one. */ update: (event: TSteerUpdatedEvent) => void; + /** Whether the user has already dealt with the parked steer a recovery start replays. */ + isRecoveryHeld: (conversationId: string, recoverySteerId: string) => boolean; + /** Refuses further recovery of a parked steer the server rejected. */ + blockRecovery: (conversationId: string, recoverySteerId: string) => void; + /** Puts a queued turn whose start was refused back in its queue position. */ + restoreQueued: (submission: TSubmission, expectedPredecessorCreatedAt?: number) => void; /** Re-seeds the chips from the server's still-queued steers. */ seed: ( conversationId: string, @@ -247,6 +254,13 @@ export interface ResumableEvents { flushPendingDeltas: () => void; /** Discards the live usage estimate for a run that stopped streaming. */ resetLive: (submission: TSubmission) => void; + /** Writes the error card for a start that failed, from the response body it failed with, or + * for a recovery start the user already dealt with. */ + showStartFailure: ( + submission: TSubmission, + errorData?: unknown, + reason?: 'recovery_held', + ) => void; /** Writes the failed turn's error card from the text (or raw error value) the server sent. */ showError: (text: string | boolean | { message?: string }, submission: TSubmission) => void; } @@ -308,6 +322,8 @@ export interface ResumableHost extends SubscriptionHost { usage: ResumableUsage; /** The stream side of the app's transport, bound to the signed-in user. */ stream: ChatTransport; + /** POSTs a turn to the resumable generation route; resolves to the start response. */ + start: (request: ChatTransportRequest, options?: { signal?: AbortSignal }) => Promise; /** Receives the dev-only controls that simulate a dropped or a clean close. */ exposeDebugControls?: (controls: { killNetwork: () => void; closeClean: () => void }) => void; } diff --git a/packages/chat/src/core/resumable/index.ts b/packages/chat/src/core/resumable/index.ts index 7c1b1a400c..db73151282 100644 --- a/packages/chat/src/core/resumable/index.ts +++ b/packages/chat/src/core/resumable/index.ts @@ -15,3 +15,4 @@ export * from './patches'; export * from './frames'; export * from './sync'; export * from './attach'; +export * from './launch'; diff --git a/packages/chat/src/core/resumable/launch.ts b/packages/chat/src/core/resumable/launch.ts new file mode 100644 index 0000000000..e94b205b68 --- /dev/null +++ b/packages/chat/src/core/resumable/launch.ts @@ -0,0 +1,259 @@ +import { createPayload, removeNullishValues } from 'librechat-data-provider'; +import type { TPayload, TSubmission } from 'librechat-data-provider'; +import type { StartGenerationResult } from './start'; +import type { ResumableHost } from './host'; +import type { ResumableRun } from './run'; +import { + getRecoverySteerId, + getRetryAfterDelay, + waitForRetryDelay, + isServerNotReadyError, + isResumedStartResponse, + isSettledStartResponse, + toStartGenerationError, + isRetryableNetworkError, + isReplacedStartResponse, + getStartGenerationStreamId, + getStartGenerationCreatedAt, + isUnnegotiatedV2StartControl, + START_GENERATION_NETWORK_RETRIES, + getStartGenerationConversationId, + getPredecessorMismatchStartResponse, + START_GENERATION_READINESS_TIMEOUT_MS, +} from './start'; +import { + GENERATION_PROTOCOL_VERSION, + getGenerationProtocolVersion, + supportsGenerationProtocolV2, +} from '../protocol'; + +/** + * Starts a generation (the POST that returns the stream to attach to) through the host + * transport, which owns auth refresh. Retries transient network failures and startup + * readiness responses; readiness retries honor Retry-After until `signal` aborts or the + * readiness window expires. Resolves to null when the start failed (its error card written) + * or was abandoned. + */ +export async function startGeneration( + run: ResumableRun, + host: ResumableHost, + currentSubmission: TSubmission, + signal?: AbortSignal, +): Promise { + const payloadData = createPayload(currentSubmission); + let { payload } = payloadData; + payload = removeNullishValues(payload) as TPayload; + + host.events.clearStepMaps(); + + const url = payloadData.server; + + let lastError: unknown = null; + let requestAttempts = 0; + let networkAttempts = 0; + let readinessAttempts = 0; + const readinessDeadline = run.scheduler.now() + START_GENERATION_READINESS_TIMEOUT_MS; + + while (!signal?.aborted) { + const recoverySteerId = getRecoverySteerId(currentSubmission); + const conversationId = currentSubmission.conversation?.conversationId; + if ( + recoverySteerId != null && + conversationId && + host.steers.isRecoveryHeld(conversationId, recoverySteerId) + ) { + host.steers.restoreQueued(currentSubmission); + host.events.showStartFailure(currentSubmission, undefined, 'recovery_held'); + host.pane.setShowStopButton(false); + host.pane.setIsSubmitting(false); + host.pane.setSubmission(null); + return null; + } + requestAttempts += 1; + try { + const data = await host.start({ server: url, payload }, { signal }); + if (signal?.aborted) { + return null; + } + if (isSettledStartResponse(data)) { + const conversationId = getStartGenerationConversationId(data); + if (conversationId) { + host.logger.log('ResumableSSE', 'Generation already settled:', { conversationId }); + return { + status: 'settled', + conversationId, + generationProtocolVersion: GENERATION_PROTOCOL_VERSION, + }; + } + lastError = { response: { data } }; + break; + } + if (isReplacedStartResponse(data)) { + const streamId = getStartGenerationStreamId(data); + const conversationId = getStartGenerationConversationId(data); + const generationCreatedAt = getStartGenerationCreatedAt(data); + if (streamId && conversationId && generationCreatedAt != null) { + return { + status: 'replaced', + streamId, + conversationId, + generationCreatedAt, + generationProtocolVersion: GENERATION_PROTOCOL_VERSION, + }; + } + lastError = { response: { data } }; + break; + } + if (isUnnegotiatedV2StartControl(data)) { + /** Never reinterpret a v2-only control word as a legacy stream + * response merely because it also carries a stream id. */ + lastError = { response: { data } }; + break; + } + const streamId = getStartGenerationStreamId(data); + if (streamId) { + const resumed = isResumedStartResponse(data); + const generationCreatedAt = getStartGenerationCreatedAt(data); + const generationProtocolVersion = getGenerationProtocolVersion(data); + host.logger.log('ResumableSSE', 'Generation started:', { + streamId, + resumed, + generationCreatedAt, + generationProtocolVersion, + }); + return { + status: 'stream', + streamId, + resumed, + generationProtocolVersion, + ...(generationCreatedAt != null && { generationCreatedAt }), + }; + } + + lastError = { response: { data } }; + break; + } catch (error) { + if (signal?.aborted) { + return null; + } + + const predecessorMismatch = getPredecessorMismatchStartResponse(error); + if (predecessorMismatch != null) { + /** Admission was atomically rejected before this queued turn + * created anything. Put the exact row back immediately and let + * the caller reconcile or hand off to the generation that won. */ + host.steers.restoreQueued(currentSubmission); + host.logger.log('ResumableSSE', 'Queued generation lost predecessor fence', { + expectedPredecessorCreatedAt: currentSubmission.expectedPredecessorCreatedAt, + ...predecessorMismatch, + }); + return { + status: 'predecessor_mismatch', + ...predecessorMismatch, + generationProtocolVersion: GENERATION_PROTOCOL_VERSION, + }; + } + + lastError = error; + const isNetworkError = isRetryableNetworkError(error); + const isServerNotReady = isServerNotReadyError(error); + const remainingReadinessMs = readinessDeadline - run.scheduler.now(); + const shouldRetryNetwork = + isNetworkError && networkAttempts < START_GENERATION_NETWORK_RETRIES - 1; + const shouldRetryServerNotReady = isServerNotReady && remainingReadinessMs > 0; + + if (shouldRetryNetwork || shouldRetryServerNotReady) { + networkAttempts += isNetworkError ? 1 : 0; + readinessAttempts += isServerNotReady ? 1 : 0; + const fallbackDelay = Math.min(1000 * Math.pow(2, requestAttempts - 1), 8000); + const retryDelay = isServerNotReady + ? Math.min(getRetryAfterDelay(error, fallbackDelay), remainingReadinessMs) + : fallbackDelay; + const reason = isServerNotReady ? 'Server not ready' : 'Network error'; + const attempt = isServerNotReady ? readinessAttempts : networkAttempts; + const limit = isServerNotReady + ? `${Math.ceil(START_GENERATION_READINESS_TIMEOUT_MS / 1000)}s readiness window` + : `${START_GENERATION_NETWORK_RETRIES}`; + host.logger.log( + 'ResumableSSE', + `${reason} starting generation, retrying in ${retryDelay}ms (attempt ${attempt}/${limit})`, + ); + const shouldContinue = await waitForRetryDelay(retryDelay, signal); + if (!shouldContinue) { + return null; + } + continue; + } + + // Don't retry: either not a network error or max retries reached + break; + } + } + + if (signal?.aborted) { + return null; + } + + host.logger.error('ResumableSSE', 'Error starting generation:', lastError); + + const startError = toStartGenerationError(lastError); + const errorData = startError?.response?.data; + const responseStatus = startError?.response?.status; + if (responseStatus != null && responseStatus >= 400 && responseStatus < 500) { + const recoverySteerId = getRecoverySteerId(currentSubmission); + const conversationId = currentSubmission.conversation?.conversationId; + const recoveryRejected = + errorData != null && + typeof errorData === 'object' && + 'code' in errorData && + (errorData.code === 'RECOVERY_PAYLOAD_MISMATCH' || + errorData.code === 'INVALID_RECOVERY_REQUEST'); + if (recoveryRejected && recoverySteerId != null && conversationId) { + host.steers.blockRecovery(conversationId, recoverySteerId); + } + // The server rejected admission before exposing a generation. Restore + // the exact queue row/position; ambiguous transport/5xx outcomes must + // first reconcile durable state instead of risking a duplicate start. + host.steers.restoreQueued(currentSubmission); + } else { + const recoverySteerId = getRecoverySteerId(currentSubmission); + const recoveryConvoId = currentSubmission.conversation?.conversationId; + if (recoverySteerId != null && recoveryConvoId) { + try { + const status = await host.fetchStatus(recoveryConvoId); + if (signal?.aborted) { + return null; + } + const releasedSource = status.unrecoveredSteers?.find( + (steer) => steer.steerId === recoverySteerId, + ); + if ( + status.active === false && + supportsGenerationProtocolV2(status) && + releasedSource != null + ) { + host.steers.convertToQueued(recoveryConvoId, [releasedSource], { + generationProtocolVersion: GENERATION_PROTOCOL_VERSION, + allowPreviouslyConvertedIds: [recoverySteerId], + }); + } + } catch (error) { + if (signal?.aborted) { + return null; + } + host.logger.warn('ResumableSSE', 'Could not recover source after start failure', { + conversationId: recoveryConvoId, + error, + }); + } + } + } + if (signal?.aborted) { + return null; + } + host.events.showStartFailure(currentSubmission, errorData); + host.pane.setShowStopButton(false); + host.pane.setIsSubmitting(false); + host.pane.setSubmission(null); + return null; +}