diff --git a/client/src/Providers/SideChatHostContext.tsx b/client/src/Providers/SideChatHostContext.tsx index 57f23563e2..2dfd8f74bc 100644 --- a/client/src/Providers/SideChatHostContext.tsx +++ b/client/src/Providers/SideChatHostContext.tsx @@ -176,6 +176,8 @@ function useChat( endpoint, enabled: id !== '', beside, + /** The reader drives this conversation: a run it loses is settled as main chat settles its own. */ + settlesLost: true, onWithdrawn: options?.onWithdrawn, onGeneration, onRunEnd, diff --git a/packages/chat/src/components/side/__tests__/panel.spec.tsx b/packages/chat/src/components/side/__tests__/panel.spec.tsx index 51ecce2568..ae4e14e781 100644 --- a/packages/chat/src/components/side/__tests__/panel.spec.tsx +++ b/packages/chat/src/components/side/__tests__/panel.spec.tsx @@ -219,7 +219,14 @@ function setup({ const openConversation = jest.fn(); const sideChatHost: SideChatHost = { useChat: ({ conversationId: id, endpoint }, options) => - useFollowChat({ id: id ?? '', transport, loadMessages, endpoint, ...options }), + useFollowChat({ + id: id ?? '', + transport, + loadMessages, + endpoint, + settlesLost: true, + ...options, + }), useOpenConversation: () => openConversation, useSheetLayout: () => false, useClearsFooter: () => false, @@ -632,6 +639,45 @@ describe('SideChatPanel', () => { expect(sendMessages).toHaveBeenCalledTimes(2); }); + it('sends the queue when a reply it had been closed on is lost after it was opened again, and its stored reply is whole', async () => { + const view = setup({ steers: true }); + const { store, streams, sendMessages, setRunning } = view; + const asked = await start(view); + act(() => { + streams[0].push({ + type: 'created', + data: { created: true, message: asked, turn: { responseMessageId: 'response-2' } }, + } as ChatEvent); + }); + await settle(); + act(() => store.set(queuedMessagesByConvoId(conversationId), [queued])); + view.unmount(); + setRunning({ active: true, streamId: conversationId, createdAt: 3000 }); + view.remount(); + await waitFor(() => expect(streams).toHaveLength(2)); + + /** Its stream drops, and by the time it is looked for again it has finished. */ + setRunning({ active: false }); + view.loadMessages.mockResolvedValue([ + user, + reply('response-1', 'It shipped.'), + asked, + { + ...reply('response-2', 'Two fixes.', asked.messageId), + createdAt: new Date().toISOString(), + }, + ]); + act(() => streams[1].push({ type: 'error', status: 0 })); + + /** Lost, so its end was not heard; but it was remembered, and the stored reply tells it. */ + await waitFor(() => expect(sendMessages).toHaveBeenCalledTimes(2), { timeout: 5000 }); + expect(sendMessages.mock.calls[1][0].payload).toMatchObject({ + text: 'And then?', + parentMessageId: 'response-2', + }); + expect(store.get(detachedRunByConvoId(conversationId))).toBeNull(); + }); + it('lets go of an earlier run’s markers when another generation is followed', async () => { const view = setup({ steers: true }); const { store } = view; diff --git a/packages/chat/src/core/resumable/steers.ts b/packages/chat/src/core/resumable/steers.ts index 0ffb6e53fc..f98f007d31 100644 --- a/packages/chat/src/core/resumable/steers.ts +++ b/packages/chat/src/core/resumable/steers.ts @@ -91,9 +91,15 @@ export function settleStoredSteers( promoted, recoverySteerId, generationProtocolVersion, + sweep = true, }: { /** The conversation's stored rows, when they could be read. */ messages: TMessage[] | undefined; + /** + * Whether the chips still left may be swept into the queue. Not when nothing has proved + * the run over to a client of its protocol: a chip still waiting may yet be taken. + */ + sweep?: boolean; status?: Pick; promoted?: TPromotedSteer[]; /** The parked steer the ended turn was itself a recovery of. */ @@ -124,7 +130,7 @@ export function settleStoredSteers( } } } - if (messages) { + if (messages && sweep) { steers.convertLocalToQueued(conversationId, { excludeSteerIds: reconciledIds, generationProtocolVersion, diff --git a/packages/chat/src/core/steer/__tests__/queue.spec.ts b/packages/chat/src/core/steer/__tests__/queue.spec.ts index 767bff865c..1bd2ead1f5 100644 --- a/packages/chat/src/core/steer/__tests__/queue.spec.ts +++ b/packages/chat/src/core/steer/__tests__/queue.spec.ts @@ -1,5 +1,10 @@ import type { QueuedMessage, RunEnd } from '../types'; -import { isSameRunEpoch, insertQueuedMessage, mergeQueuedMessages } from '../queue'; +import { + isSameRunEpoch, + insertQueuedMessage, + mergeQueuedMessages, + resolveLostRunOutcome, +} from '../queue'; const item = (id: string, createdAt: number, priority?: boolean): QueuedMessage => ({ id, @@ -47,3 +52,22 @@ describe('isSameRunEpoch', () => { expect(isSameRunEpoch(null, end({}))).toBe(false); }); }); + +describe('resolveLostRunOutcome', () => { + it.each([ + ['completed, with its rows read', { status: 'complete', createdAt: 2000 }, true, 'completed'], + ['completed, by a status that names no generation', { status: 'complete' }, true, 'completed'], + [ + 'completed, but its rows could not be read', + { status: 'complete', createdAt: 2000 }, + false, + 'error', + ], + ['stopped', { status: 'aborted', createdAt: 2000 }, true, 'aborted'], + ['failed', { status: 'error', createdAt: 2000 }, true, 'error'], + ['gone from the server’s records', {}, true, 'error'], + ['another generation’s status', { status: 'complete', createdAt: 9000 }, true, 'error'], + ] as const)('%s', (_case, status, read, outcome) => { + expect(resolveLostRunOutcome(status, 2000, read)).toBe(outcome); + }); +}); diff --git a/packages/chat/src/core/steer/queue.ts b/packages/chat/src/core/steer/queue.ts index e13c162005..32ce9a29b3 100644 --- a/packages/chat/src/core/steer/queue.ts +++ b/packages/chat/src/core/steer/queue.ts @@ -82,6 +82,27 @@ const endedWithoutSummary = (message: TMessage): boolean => { return failed; }; +/** + * How a run ended whose connection gave up before it did, from a status that says nothing is + * running any more. It completed only if the status says so of this very generation and its + * stored rows are in hand; it was stopped if the status says that; anything else, including a + * status that names another generation or none, is not known to have gone well, so what is + * queued behind the run is left for the reader instead of being sent on a guess. + */ +export function resolveLostRunOutcome( + status: { status?: string; createdAt?: number }, + generationCreatedAt: number, + read: boolean, +): RunEnd['outcome'] { + if (status.createdAt != null && status.createdAt !== generationCreatedAt) { + return 'error'; + } + if (status.status === 'complete' && read) { + return 'completed'; + } + return status.status === 'aborted' ? 'aborted' : 'error'; +} + const writtenAt = (message: TMessage): number => { const stamp = message.updatedAt ?? message.createdAt; return stamp == null ? Number.NaN : Date.parse(stamp); diff --git a/packages/chat/src/react/chat/__tests__/driven.spec.tsx b/packages/chat/src/react/chat/__tests__/driven.spec.tsx index 9269415d54..18f2ef6c64 100644 --- a/packages/chat/src/react/chat/__tests__/driven.spec.tsx +++ b/packages/chat/src/react/chat/__tests__/driven.spec.tsx @@ -25,9 +25,16 @@ import { toResumableUsage } from '../../stream/usage'; import { isSubmittingFamily } from '../../submission'; import { useDrivenContract } from '../driven'; import { useFollowChat } from '../../follow'; +import { MAX_RETRIES } from '../../../core'; import { ChatHostProvider } from '../host'; import { filesFamily } from '../../pane'; +/** Attaching again waits between tries on main chat's schedule, which a spec has no use for. */ +jest.mock('../../../core/resumable/start', () => ({ + ...jest.requireActual('../../../core/resumable/start'), + getReconnectDelay: () => 5, +})); + const conversationId = 'conversation-1'; const SIDE_PANE = 7; @@ -121,6 +128,7 @@ const createTransport = (generationProtocolVersion?: number) => { return { transport, streams, + open, sendMessages, abort, reconnectToStream, @@ -238,6 +246,7 @@ function setup({ endpoint: conversation.endpoint, onWithdrawn, beside, + settlesLost: true, ...runs, }); const contract = useDrivenContract({ @@ -769,6 +778,247 @@ describe('useDrivenContract', () => { }); }); + describe('a run whose stream is lost', () => { + const record = () => { + const calls: Array<[string, ...unknown[]]> = []; + const log = + (name: string, result?: unknown) => + (...args: unknown[]): unknown => { + calls.push([name, ...args]); + return result; + }; + const steers = { + resolve: log('resolve', []), + markLiveApplied: log('markLive'), + update: log('update'), + seed: log('seed'), + settleApplied: log('settle'), + appliedIds: log('appliedIds', ['steer-taken']), + adoptPromoted: log('adopt', []), + convertToQueued: log('toQueued'), + convertLocalToQueued: log('sweep'), + restoreQueued: log('restore'), + blockRecovery: log('block'), + isRecoveryHeld: () => false, + } as unknown as NonNullable['steers']>; + return { calls, steers }; + }; + /** Sends a turn and lets the server name it. */ + const begin = async (view: ReturnType) => { + act(() => { + view.result.current.contract.ask({ text: 'Next question' }); + }); + await waitFor(() => expect(view.streams).toHaveLength(1)); + const sent = (view.result.current.contract.getMessages() ?? [])[2]; + act(() => view.streams[0].push(created(sent, 'response-2'))); + await settle(); + return sent; + }; + /** Drops the stream in hand, and each one opened to attach again, `times` in all. */ + const dropping = async (view: ReturnType, times: number) => { + for (let dropped = 0; dropped < times; dropped++) { + await waitFor(() => expect(view.streams.length).toBeGreaterThan(dropped)); + act(() => view.streams[dropped].push({ type: 'error', status: 0 })); + } + }; + const whole = (sent: TMessage) => [ + user, + reply('response-1', 'Hi'), + sent, + reply('response-2', 'Whole', sent.messageId), + ]; + + it('is taken as stopped when nothing is running by the time it is looked for again', async () => { + const { calls, steers } = record(); + const onRunEnd = jest.fn(); + const onIdle = jest.fn(); + const view = setup({ beside: { steers }, runs: { onRunEnd, onIdle } }); + await view.ready(); + const sent = await begin(view); + calls.length = 0; + onIdle.mockClear(); + view.loadMessages.mockResolvedValue(whole(sent)); + /** Even when the server still says it completed: main chat's rule for a stream that is gone. */ + view.setStatus({ active: false, status: 'complete', createdAt: 3000 }); + + await dropping(view, 1); + await waitFor(() => expect(onRunEnd).toHaveBeenCalledTimes(1)); + + expect(calls.map(([name]) => name)).toEqual(['appliedIds', 'settle', 'adopt', 'sweep']); + expect(calls[1]).toEqual(['settle', conversationId, whole(sent)]); + expect(onRunEnd).toHaveBeenCalledWith( + expect.objectContaining({ outcome: 'aborted', generationCreatedAt: 3000 }), + ); + /** + * Before that, the caller is handed the rows the run left, with nothing running: one that + * had remembered this run when it stopped watching it can read its end from them. + */ + expect(onIdle).toHaveBeenCalledTimes(1); + expect(ids(onIdle.mock.calls[0][0])).toEqual(ids(whole(sent))); + expect(onIdle.mock.invocationCallOrder[0]).toBeLessThan(onRunEnd.mock.invocationCallOrder[0]); + }); + + it('follows the generation that is running instead, when one is by the time it is looked for', async () => { + const onRunEnd = jest.fn(); + const onIdle = jest.fn(); + const onGeneration = jest.fn(); + const view = setup({ runs: { onRunEnd, onIdle, onGeneration } }); + await view.ready(); + await begin(view); + onIdle.mockClear(); + /** Nothing to attach to when it is looked for; a newer run by the time its rows are read. */ + const reads = view.loadMessages.mock.calls.length; + const newer = { active: true, streamId: conversationId, createdAt: 9000 }; + view.transport.getStatus = async () => + view.loadMessages.mock.calls.length > reads ? newer : { active: false }; + view.reconnectToStream.mockImplementation(async () => + view.loadMessages.mock.calls.length > reads ? view.open() : null, + ); + + await dropping(view, 1); + + /** No rows are handed over as the lost run's, and nothing is said to have ended. */ + await waitFor(() => expect(view.streams).toHaveLength(2)); + expect(onIdle).not.toHaveBeenCalled(); + expect(onRunEnd).not.toHaveBeenCalled(); + expect(onGeneration).not.toHaveBeenLastCalledWith(null); + }); + + it('sweeps no waiting steers when nothing proved the run over to a client of its protocol', async () => { + const { calls, steers } = record(); + const onRunEnd = jest.fn(); + const harness = createTransport(2); + const view = setup({ beside: { steers }, runs: { onRunEnd }, harness }); + await view.ready(); + const sent = await begin(view); + calls.length = 0; + view.loadMessages.mockResolvedValue(whole(sent)); + /** Gone when it is looked for, and its status cannot be read once its rows are in. */ + const reads = view.loadMessages.mock.calls.length; + view.transport.getStatus = async () => { + if (view.loadMessages.mock.calls.length > reads) { + throw new Error('The status route is down'); + } + return { active: false }; + }; + + await dropping(view, 1); + await waitFor(() => expect(onRunEnd).toHaveBeenCalledTimes(1)); + + /** What the stored rows hold is settled; a chip still waiting may yet be taken, and stays. */ + expect(calls.map(([name]) => name)).toEqual(['appliedIds', 'settle', 'adopt']); + expect(onRunEnd).toHaveBeenCalledWith(expect.objectContaining({ outcome: 'aborted' })); + }); + + it('settles nothing on the word of a server that does not speak the generation’s protocol', async () => { + const { calls, steers } = record(); + const onRunEnd = jest.fn(); + const onGeneration = jest.fn(); + const harness = createTransport(2); + const view = setup({ beside: { steers }, runs: { onRunEnd, onGeneration }, harness }); + await view.ready(); + const sent = await begin(view); + calls.length = 0; + view.setStatus({ active: true, streamId: conversationId, createdAt: 3000 }); + await dropping(view, MAX_RETRIES + 1); + await waitFor(() => expect(view.streams.length).toBe(MAX_RETRIES + 2)); + + /** Past the last retry, an older responder says nothing is running: it cannot say that of this run. */ + view.loadMessages.mockResolvedValue(whole(sent)); + view.setStatus({ active: false, status: 'complete', createdAt: 3000 }); + act(() => view.streams[MAX_RETRIES + 1].push({ type: 'error', status: 0 })); + await settle(); + await settle(); + expect(onRunEnd).not.toHaveBeenCalled(); + expect(calls).toEqual([]); + expect(onGeneration).toHaveBeenLastCalledWith({ createdAt: 3000, protocolVersion: 2 }); + + /** A responder that speaks it says the same, and now it is settled. */ + view.setStatus({ + active: false, + status: 'complete', + createdAt: 3000, + generationProtocolVersion: 2, + }); + await waitFor(() => expect(onRunEnd).toHaveBeenCalledTimes(1)); + expect(calls.map(([name]) => name)).toEqual(['appliedIds', 'settle', 'adopt', 'sweep']); + expect(onGeneration).toHaveBeenLastCalledWith(null); + }); + + it('keeps the generation while it is looked for again, so a later round can still settle it', async () => { + const onRunEnd = jest.fn(); + const view = setup({ runs: { onRunEnd } }); + await view.ready(); + const sent = await begin(view); + view.setStatus({ active: true, streamId: conversationId, createdAt: 3000 }); + await dropping(view, MAX_RETRIES + 2); + const opened = view.reconnectToStream.mock.calls.length; + + /** The next round never opens a stream at all, and by its end the run has finished. */ + view.reconnectToStream.mockRejectedValue(new Error('The stream route is down')); + await waitFor(() => + expect(view.reconnectToStream.mock.calls.length).toBeGreaterThan(opened + 1), + ); + view.loadMessages.mockResolvedValue(whole(sent)); + view.setStatus({ active: false, status: 'complete', createdAt: 3000 }); + + await waitFor(() => expect(onRunEnd).toHaveBeenCalledTimes(1), { timeout: 4000 }); + expect(onRunEnd).toHaveBeenCalledWith(expect.objectContaining({ generationCreatedAt: 3000 })); + }); + + it('is settled by the server’s word once reconnecting has failed past its last retry', async () => { + const { calls, steers } = record(); + const onRunEnd = jest.fn(); + const onGeneration = jest.fn(); + const view = setup({ beside: { steers }, runs: { onRunEnd, onGeneration } }); + await view.ready(); + const sent = await begin(view); + calls.length = 0; + /** Still running as far as the server says, and every attachment drops. */ + view.setStatus({ active: true, streamId: conversationId, createdAt: 3000 }); + await dropping(view, MAX_RETRIES + 1); + expect(onRunEnd).not.toHaveBeenCalled(); + + /** By the last failure it has finished, and the server says how. */ + await waitFor(() => expect(view.streams.length).toBe(MAX_RETRIES + 2)); + view.loadMessages.mockResolvedValue(whole(sent)); + view.setStatus({ active: false, status: 'complete', createdAt: 3000 }); + act(() => view.streams[MAX_RETRIES + 1].push({ type: 'error', status: 0 })); + await waitFor(() => expect(onRunEnd).toHaveBeenCalledTimes(1)); + + expect(ids(view.result.current.contract.getMessages())).toEqual(ids(whole(sent))); + expect(calls.map(([name]) => name)).toEqual(['appliedIds', 'settle', 'adopt', 'sweep']); + expect(calls[1]).toEqual(['settle', conversationId, whole(sent)]); + expect(onRunEnd).toHaveBeenCalledWith({ + outcome: 'completed', + generationCreatedAt: 3000, + responseMessageId: 'response-2', + }); + /** It is over: nothing is addressed as running, and the next turn can be taken. */ + expect(onGeneration).toHaveBeenLastCalledWith(null); + await waitFor(() => expect(view.result.current.contract.isSubmitting).toBe(false)); + expect(view.result.current.followed.isUnattached).toBe(false); + expect(view.result.current.followed.canSend()).toBe(true); + }); + + it('is looked for again, and nothing is said of it, while the server says it is still running', async () => { + const { calls, steers } = record(); + const onRunEnd = jest.fn(); + const view = setup({ beside: { steers }, runs: { onRunEnd } }); + await view.ready(); + await begin(view); + calls.length = 0; + view.setStatus({ active: true, streamId: conversationId, createdAt: 3000 }); + + await dropping(view, MAX_RETRIES + 2); + + /** Past the last retry, and still asked for: the connection gave up, not the generation. */ + await waitFor(() => expect(view.streams.length).toBeGreaterThan(MAX_RETRIES + 2)); + expect(onRunEnd).not.toHaveBeenCalled(); + expect(calls).toEqual([]); + }); + }); + describe('the reader’s steers and queue when a run ends without its reply', () => { /** Main chat's steer bookkeeping, recording what it is called with and in what order. */ const record = (held = new Set()) => { diff --git a/packages/chat/src/react/follow.ts b/packages/chat/src/react/follow.ts index 0063dde87a..cfd7131218 100644 --- a/packages/chat/src/react/follow.ts +++ b/packages/chat/src/react/follow.ts @@ -31,6 +31,7 @@ import { getRecoverySteerId, settleStoredSteers, settleFinishedSteers, + resolveLostRunOutcome, restoreRefusedStart, toStartGenerationError, restoreReleasedSource, @@ -94,6 +95,15 @@ export type FollowChatOptions = { * with a sent turn's own final frame in. Not for a stream that only dropped. */ onRunEnd?: (end: FollowedRunEnd) => void; + /** + * Whether a generation that can no longer be followed is settled from the server's status, as + * main chat's session settles its own. For a conversation the reader drives, whose queue and + * steers wait on how its runs end: it is kept and looked for again while the server says it + * is running or cannot say, and its end is said once the server says it is over. Without it + * a follower stops at its last retry and says the run is out of reach (`isUnattached`), which + * is what a caller with another way to show the run wants. + */ + settlesLost?: boolean; /** The reader asked for the generation in flight to be stopped. */ onStopped?: (generation: FollowedGeneration) => void; /** @@ -104,8 +114,9 @@ export type FollowChatOptions = { */ onDetached?: (run: DetachedRun) => void; /** - * The conversation was opened with nothing running under it. `messages` are its stored rows - * as just read, which hold whatever an earlier run left. + * The conversation was found with nothing running under it: as it opened, or after the stream + * of a run it was following was gone. `messages` are its stored rows as just read, which hold + * whatever an earlier run left. */ onIdle?: (messages: TMessage[]) => void; }; @@ -347,6 +358,7 @@ export function useFollowChat({ onWithdrawn, onGeneration, onRunEnd, + settlesLost = false, onStopped, onDetached, onIdle, @@ -364,6 +376,7 @@ export function useFollowChat({ onWithdrawn, onGeneration, onRunEnd, + settlesLost, onStopped, onDetached, onIdle, @@ -380,6 +393,7 @@ export function useFollowChat({ onWithdrawn, onGeneration, onRunEnd, + settlesLost, onStopped, onDetached, onIdle, @@ -835,6 +849,103 @@ export function useFollowChat({ * stored ones that may not hold them yet, and the attachment is made again. An attachment * that delivered something starts the count of failed ones over. */ + /** + * A generation this chat was following can no longer be followed, in one of two ways. It is + * `gone`: its stream dropped and nothing was running when it was looked for again. Or it is + * `unreachable`: the connection failed past its last retry, which is the transport giving + * up and not the generation ending. Only the server's status says what became of it, as it + * does for main chat's session, and only a status from a server that speaks the + * generation's protocol can say so: during a rollout an older responder cannot adjudicate + * a newer job. + * + * - Unreachable, and the status says it is still running, or cannot say: nothing is settled. + * It is kept, and looked for again after a wait. + * - Gone, and the status names something running or cannot adjudicate: the same, after a + * short wait. A stream can briefly lose its record while its generation lives, and a + * newer generation is one to follow. + * - Otherwise it is over. Unreachable: the stored rows are read and its end is said with + * the outcome the status gives. Gone: nothing says how it ended, so it is taken as + * stopped; but the rows are first handed to a caller that had remembered the run, which + * can read its end from them. Either way its steers are settled against the rows, and + * the chips still left are swept only when the run is proved over. + * + * Returns whether it is kept to be looked for again. + */ + const settleLost = async ( + how: 'gone' | 'unreachable', + current?: Attachment, + ): Promise => { + const known = generation; + const turn = answering; + const status = await Promise.resolve(host.current.transport.getStatus?.(id)).catch( + () => undefined, + ); + if (stale) { + return false; + } + const speaks = + known?.protocolVersion !== GENERATION_PROTOCOL_VERSION || + supportsGenerationProtocolV2(status); + const answer = status != null && speaks ? status : undefined; + const undecided = + how === 'unreachable' + ? answer == null || answer.active === true + : status != null && (answer == null || answer.active === true); + if (known != null && undecided) { + await pause(getReconnectDelay(how === 'unreachable' ? MAX_RETRIES : 1)); + owed = !stale; + return !stale; + } + let rows = how === 'gone'; + if (!rows) { + reading = true; + rows = await read().then( + () => true, + () => false, + ); + if (stale) { + return false; + } + } + if (how === 'gone' && answer != null && answer.active !== true) { + host.current.onIdle?.(chat.messages); + } + const steers = host.current.beside?.steers; + if (steers != null && status?.active !== true) { + settleStoredSteers(steers, id, { + messages: rows ? chat.messages : undefined, + status: answer, + generationProtocolVersion: known?.protocolVersion, + sweep: answer != null || known?.protocolVersion === 1, + }); + } + if (how === 'gone' && current != null) { + runEnded(current, true); + return false; + } + forget(); + if (known == null) { + return false; + } + const outcome = + how === 'unreachable' && answer != null + ? resolveLostRunOutcome(answer, known.createdAt, rows) + : 'aborted'; + host.current.onRunEnd?.({ + outcome, + generationCreatedAt: known.createdAt, + responseMessageId: + outcome === 'completed' + ? completedResponseMessageId( + chat.messages, + turn?.userMessageId, + answer?.resumeState?.responseMessageId ?? turn?.responseMessageId, + ) + : undefined, + }); + return false; + }; + const attach = async (): Promise => { if (stale || !loaded) { return; @@ -855,6 +966,10 @@ export function useFollowChat({ let reached = false; /** The last attachment, whose end is the generation's unless it dropped. */ let last: Attachment | undefined; + /** Whether attaching failed past its last retry. */ + let unreachable = false; + /** Whether the generation is kept to be looked for again, its fate not yet known. */ + let kept = false; try { for (let failures = 0, absences = 0; !stale; ) { last = await follow(); @@ -875,30 +990,49 @@ export function useFollowChat({ } failures = received ? 1 : failures + 1; if (failures > MAX_RETRIES) { - reached = false; - return; + unreachable = true; + break; } await pause(getReconnectDelay(failures)); } if (stale) { return; } - reading = true; - await read(); - if (last != null && unsettled(last)) { - await settleSteers(last); + const settles = host.current.settlesLost; + if (unreachable) { + kept = settles && (await settleLost('unreachable')); + reached = settles && !kept && generation == null; + } else { + reading = true; + await read(); + const gone = last?.finished?.isSettled === true && last.terminal == null; + if (settles && gone) { + kept = await settleLost('gone', last); + } else if (settles && last?.finished == null && generation != null) { + /** Kept from an earlier look, and still nothing to follow. */ + kept = await settleLost('gone'); + } else { + if (last != null && unsettled(last)) { + await settleSteers(last); + } + if (stale) { + return; + } + runEnded(last, true); + } } - if (stale) { - return; - } - runEnded(last, true); } catch { /** The chat reports a stream failure itself; a failed re-read leaves what it streamed. */ } finally { attaching = false; reading = false; - /** Lost, or never read back: nothing is addressed as running any more. */ - forget(); + /** + * Lost, or never read back: nothing is addressed as running any more. Unless it is kept + * to be looked for again, when it is still the one a stop or a steer addresses. + */ + if (!kept) { + forget(); + } if (!stale) { setFollowing(false); setUnattached(!reached);