diff --git a/e2e/specs/mock/scenarios/chat-package.spec.ts b/e2e/specs/mock/scenarios/chat-package.spec.ts index 6587a532ec..9039bb7c1d 100644 --- a/e2e/specs/mock/scenarios/chat-package.spec.ts +++ b/e2e/specs/mock/scenarios/chat-package.spec.ts @@ -338,6 +338,106 @@ test.describe('the built @librechat/chat package', () => { }); }); + test('a host sees a start it stopped before the answer as possibly still running @scenario:a-host-sees-a-stopped-start-as-possibly-still-running', () => { + test.setTimeout(120_000); + const dist = resolve(repoRoot, 'packages/chat/dist'); + expect(existsSync(dist), 'packages/chat/dist is missing: run npm run build').toBe(true); + + /** The start reaches the transport and never answers; the stop cannot name the generation, + * so the turn finishes as a disconnect and every later stop stays the host's to make. */ + const probe = ` + const { Chat } = require('@librechat/chat'); + let messages = []; + const statuses = []; + const finishes = []; + const chat = new Chat({ + id: 'convo-1', + state: { + error: undefined, + connection: 'idle', + get status() { return statuses[statuses.length - 1] || 'ready'; }, + set status(value) { statuses.push(value); }, + get messages() { return messages; }, + replaceMessages(next) { messages = next; }, + }, + transport: { + sendMessages: () => new Promise(() => {}), + reconnectToStream: async () => null, + abort: async () => { throw new Error('no stop should be sent'); }, + }, + prepareRequest: () => ({ + payload: { endpoint: 'agents' }, + messages: [{ messageId: 'r1', text: '' }], + }), + onFinish: ({ isAbort, isDisconnect }) => finishes.push({ isAbort, isDisconnect }), + }); + const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] }); + setTimeout(async () => { + const first = await chat.stop(); + await sent; + const second = await chat.stop(); + process.stdout.write(JSON.stringify({ first, second, finishes, status: chat.status })); + }, 0); + `; + const loaded = run(process.execPath, ['-e', probe], { cwd: resolve(repoRoot, 'client') }); + expect(loaded.status, loaded.output).toBe(0); + expect(JSON.parse(loaded.stdout)).toEqual({ + first: { status: 'unaddressable' }, + second: { status: 'unaddressable' }, + finishes: [{ isAbort: false, isDisconnect: true }], + status: 'error', + }); + }); + + test('a host sees a pause retired when its expiry passes with no later frame @scenario:a-host-sees-an-expired-pause-retired-without-a-frame', () => { + test.setTimeout(120_000); + const dist = resolve(repoRoot, 'packages/chat/dist'); + expect(existsSync(dist), 'packages/chat/dist is missing: run npm run build').toBe(true); + + /** A reload restores a pause that expires shortly; nothing else arrives before it does. */ + const probe = ` + const { Chat } = require('@librechat/chat'); + let messages = []; + const sends = []; + const chat = new Chat({ + id: 'convo-1', + state: { error: undefined, connection: 'idle', status: 'ready', + get messages() { return messages; }, + replaceMessages(next) { messages = next; } }, + transport: { + sendMessages: (request) => { sends.push(request.trigger); return new Promise(() => {}); }, + reconnectToStream: async () => null, + abort: async () => ({ success: true }), + }, + prepareRequest: () => ({ payload: { endpoint: 'agents' } }), + }); + let notified = 0; + chat.subscribe(() => { notified += 1; }); + chat.restorePendingAction({ + actionId: 'action-1', streamId: 'convo-1', conversationId: 'convo-1', createdAt: 1, + expiresAt: Date.now() + 50, + payload: { type: 'ask_user_question', question: { question: 'Which?' }, tool_call_id: 'ask-1' }, + }, 500); + const held = chat.pendingAction != null; + const before = notified; + setTimeout(async () => { + const retired = chat.pendingAction === undefined && notified > before; + chat.sendMessage({ parts: [{ type: 'text', text: 'Next' }] }); + await new Promise((resolve) => setTimeout(resolve, 0)); + process.stdout.write(JSON.stringify({ held, retired, sends })); + chat.dispose(); + process.exit(0); + }, 150); + `; + const loaded = run(process.execPath, ['-e', probe], { cwd: resolve(repoRoot, 'client') }); + expect(loaded.status, loaded.output).toBe(0); + expect(JSON.parse(loaded.stdout)).toEqual({ + held: true, + retired: true, + sends: ['submit-message'], + }); + }); + test('a host answers a pause on a stream that names no generation @scenario:a-host-answers-a-pause-on-a-stream-without-a-generation', () => { test.setTimeout(120_000); const dist = resolve(repoRoot, 'packages/chat/dist'); diff --git a/packages/chat/src/core/__tests__/chat.spec.ts b/packages/chat/src/core/__tests__/chat.spec.ts index d8ef4bc57a..91eddf14f7 100644 --- a/packages/chat/src/core/__tests__/chat.spec.ts +++ b/packages/chat/src/core/__tests__/chat.spec.ts @@ -390,6 +390,28 @@ describe('Chat', () => { payload: { type: 'ask_user_question', question: { question: 'Which?' }, tool_call_id: 'a' }, } as Agents.PendingAction; + it('stays out of the host reducer when it expired', async () => { + const applyEvent = jest.fn(); + const { chat, transport } = setup({ + applyEvent, + resumeSelection: () => ({ endpoint: 'agents' }), + }); + const replay = createStream({ streamId: 'convo-1', generationCreatedAt: 100 }); + transport.reconnectToStream.mockResolvedValueOnce(replay); + const attached = chat.resumeStream({ streamId: 'convo-1', generationCreatedAt: 100 }); + await flush(); + + replay.push({ type: 'pending_action', data: { ...pause, expiresAt: Date.now() - 1_000 } }); + await flush(); + + expect(applyEvent).not.toHaveBeenCalledWith( + expect.objectContaining({ type: 'pending_action' }), + expect.anything(), + ); + replay.push(finalEvent()); + await attached; + }); + it('stays out of the host reducer when this chat already resumed it', async () => { const applyEvent = jest.fn(); const { chat, transport } = setup({ @@ -2627,6 +2649,154 @@ describe('Chat', () => { await sent; }); + it('does not restore an expired pause, and drops the one it held', async () => { + const { chat } = setup(); + chat.restorePendingAction(askAction, 500); + + chat.restorePendingAction({ ...askAction, expiresAt: Date.now() - 1 }, 500); + + expect(chat.pendingAction).toBeUndefined(); + chat.restorePendingAction({ ...approvalAction, expiresAt: Date.now() - 1 }, 500); + expect(chat.pendingAction).toBeUndefined(); + }); + + it('retires a restored pause when its expiry passes without another frame', async () => { + const { chat, transport } = setup(); + const listener = jest.fn(); + chat.subscribe(listener); + chat.restorePendingAction({ ...askAction, expiresAt: Date.now() + 30 }, 500); + expect(chat.pendingAction).toBeDefined(); + listener.mockClear(); + + await new Promise((resolve) => setTimeout(resolve, 60)); + + expect(chat.pendingAction).toBeUndefined(); + expect(listener).toHaveBeenCalled(); + const next = chat.sendMessage({ parts: [{ type: 'text', text: 'Next' }] }); + await flush(); + expect(transport.sendMessages).toHaveBeenCalledTimes(1); + chat.dispose(); + await next; + }); + + it('retires a streamed pause when its expiry passes while the stream stays quiet', async () => { + const { chat, stream, sent } = await pause({ ...askAction, expiresAt: Date.now() + 30 }); + expect(chat.pendingAction?.actionId).toBe(askAction.actionId); + + await new Promise((resolve) => setTimeout(resolve, 60)); + + expect(chat.pendingAction).toBeUndefined(); + stream.push(finalEvent()); + await sent; + }); + + it('leaves an expiring pause to the answer in flight, and drops it when that answer fails', async () => { + let refuse: (error: Error) => void = () => undefined; + const resume = jest.fn( + () => + new Promise((_, reject) => { + refuse = reject; + }), + ); + const { chat } = setup({}, { resume }); + chat.restorePendingAction({ ...askAction, expiresAt: Date.now() + 30 }, 500); + + const answered = chat.addToolOutput({ toolCallId: 'ask-1', output: 'x' }); + await new Promise((resolve) => setTimeout(resolve, 60)); + expect(chat.pendingAction?.actionId).toBe(askAction.actionId); + + refuse(new Error('The pause expired')); + await expect(answered).rejects.toThrow('The pause expired'); + expect(chat.pendingAction).toBeUndefined(); + }); + + it('drops an expired copy of the pause a status read restored while its answer was in flight', async () => { + let refuse: (error: Error) => void = () => undefined; + const resume = jest.fn( + () => + new Promise((_, reject) => { + refuse = reject; + }), + ); + const { chat } = setup({}, { resume }); + const action = { ...askAction, expiresAt: Date.now() + 30 }; + chat.restorePendingAction(action, 500); + + const answered = chat.addToolOutput({ toolCallId: 'ask-1', output: 'x' }); + await flush(); + chat.restorePendingAction({ ...action }, 500); + await new Promise((resolve) => setTimeout(resolve, 60)); + expect(chat.pendingAction?.actionId).toBe(askAction.actionId); + + refuse(Object.assign(new Error('Service unavailable'), { response: { status: 503 } })); + await expect(answered).rejects.toThrow('Service unavailable'); + expect(chat.pendingAction).toBeUndefined(); + }); + + it('stops the expiry of a pause the chat no longer holds or that was disposed', async () => { + const { chat } = setup(); + const listener = jest.fn(); + chat.subscribe(listener); + chat.restorePendingAction({ ...askAction, expiresAt: Date.now() + 30 }, 500); + chat.dispose(); + listener.mockClear(); + + await new Promise((resolve) => setTimeout(resolve, 60)); + + expect(listener).not.toHaveBeenCalled(); + }); + + it('reconciles the pause before a host reducer write notifies subscribers', async () => { + const seen: { event: string; actionId?: string }[] = []; + let folding: string | undefined; + const { chat, stream, sent } = await pause(askAction, { + applyEvent: (event, context) => { + folding = event.type; + context.write([...context.messages]); + folding = undefined; + }, + }); + chat.subscribe(() => { + if (folding) { + seen.push({ event: folding, actionId: chat.pendingAction?.actionId }); + } + }); + + stream.push({ + type: 'sync', + data: { sync: true, conversationId: 'convo-1', resumeState: { runSteps: [] } }, + } as ChatEvent); + await flush(); + stream.push({ type: 'pending_action', data: approvalAction }); + await flush(); + stream.push(finalEvent()); + await sent; + + expect(seen).toEqual([ + { event: 'sync', actionId: undefined }, + { event: 'pending_action', actionId: approvalAction.actionId }, + { event: 'final', actionId: undefined }, + ]); + }); + + it('drops an expired pause a stream replays, and the one it held under that id', async () => { + const { chat, stream, sent } = await pause(askAction); + expect(chat.pendingAction).toBe(askAction); + + stream.push({ type: 'pending_action', data: { ...askAction, expiresAt: Date.now() - 1 } }); + await flush(); + expect(chat.pendingAction).toBeUndefined(); + + stream.push({ type: 'pending_action', data: { ...approvalAction, expiresAt: 1 } }); + await flush(); + expect(chat.pendingAction).toBeUndefined(); + stream.push({ type: 'error', status: 0, data: 'offline' } as ChatEvent); + await sent; + await expect(chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] })).resolves.toBe( + undefined, + ); + }); + it('stops a restored pause of a newer generation, not an older dropped one', async () => { const { chat, stream, sent, transport } = await pause(askAction, { resumeSelection: () => ({ endpoint: 'agents' }), @@ -2991,6 +3161,18 @@ describe('Chat', () => { expect(transport.sendMessages).not.toHaveBeenCalled(); }); + it('drops a restored pause once a newer generation is restored with an expired one', async () => { + const { chat } = setup(); + chat.restorePendingAction({ ...pausedAction, streamId: 'convo-1' }, 500); + + chat.restorePendingAction( + { ...pausedAction, actionId: 'action-6', streamId: 'convo-1', expiresAt: 1 }, + 900, + ); + + expect(chat.pendingAction).toBeUndefined(); + }); + it('sends no stop for a late start it cannot name', async () => { let open: ((result: { status: 'stream'; stream: ChatStream }) => void) | undefined; const late = createStream(); diff --git a/packages/chat/src/core/__tests__/generations.spec.ts b/packages/chat/src/core/__tests__/generations.spec.ts index 362b7c1873..f060838cbd 100644 --- a/packages/chat/src/core/__tests__/generations.spec.ts +++ b/packages/chat/src/core/__tests__/generations.spec.ts @@ -1920,6 +1920,15 @@ describe('Chat status reads out of order', () => { expect(h.chat.pendingAction).toBeUndefined(); }); + it('(2) an expired status read still marks its newer generation, so an older read stays stale', async () => { + const h = createHarness(); + h.chat.restorePendingAction(askAction, 500); + h.chat.restorePendingAction({ ...askAction, actionId: 'action-600', expiresAt: 1 }, 600); + h.chat.restorePendingAction({ ...askAction, actionId: 'action-550' }, 550); + + expect(h.chat.pendingAction).toBeUndefined(); + }); + it('(5) an idle stop answers idle without reading a resume selection that throws', async () => { const h = createHarness({ resumeSelection: () => { @@ -2006,6 +2015,86 @@ describe('Chat answers, successors and pause order', () => { ); }); + it('(2) a status read of an older pause in the same generation keeps the newer one', async () => { + const h = createHarness(); + h.chat.restorePendingAction({ ...askAction, createdAt: 3, actionId: 'action-b' }, 500); + h.chat.restorePendingAction({ ...askAction, createdAt: 2, actionId: 'action-a' }, 500); + + expect(h.chat.pendingAction?.actionId).toBe('action-b'); + }); + + it('(2) a reconnect replaying an older pause of the generation keeps the newer restored one', async () => { + const h = createHarness(); + h.chat.restorePendingAction({ ...askAction, createdAt: 3, actionId: 'action-b' }, 500); + const replay = createStream({ streamId: 'convo-1', generationCreatedAt: 500 }); + h.transport.reconnectToStream.mockResolvedValueOnce(replay); + h.chat.resumeStream({ streamId: 'convo-1', generationCreatedAt: 500 }).then( + () => undefined, + () => undefined, + ); + await flush(); + replay.push({ + type: 'pending_action', + data: { ...askAction, createdAt: 2, actionId: 'action-a' }, + } as ChatEvent); + await flush(); + + expect(h.chat.pendingAction?.actionId).toBe('action-b'); + }); + + it('(2) a reconnect replaying an expired newer pause clears the older restored one and stays ordered', async () => { + const h = createHarness(); + h.chat.restorePendingAction({ ...askAction, createdAt: 2, actionId: 'action-a' }, 500); + const replay = createStream({ streamId: 'convo-1', generationCreatedAt: 500 }); + h.transport.reconnectToStream.mockResolvedValueOnce(replay); + h.chat.resumeStream({ streamId: 'convo-1', generationCreatedAt: 500 }).then( + () => undefined, + () => undefined, + ); + await flush(); + replay.push({ + type: 'pending_action', + data: { ...askAction, createdAt: 3, actionId: 'action-b', expiresAt: 1 }, + } as ChatEvent); + await flush(); + expect(h.chat.pendingAction).toBeUndefined(); + + replay.push({ + type: 'pending_action', + data: { ...askAction, createdAt: 2, actionId: 'action-a' }, + } as ChatEvent); + await flush(); + expect(h.chat.pendingAction).toBeUndefined(); + }); + + it.each([ + ['a status read', 'read'], + ['a replay', 'replay'], + ])( + '(2) of two pauses created in the same millisecond, the one received last stands through %s', + async (_label, via) => { + const h = createHarness(); + const first = { ...askAction, createdAt: 3, actionId: 'action-a' }; + const second = { ...askAction, createdAt: 3, actionId: 'action-b' }; + h.chat.restorePendingAction(second, 500); + if (via === 'read') { + h.chat.restorePendingAction(first, 500); + } else { + const replay = createStream({ streamId: 'convo-1', generationCreatedAt: 500 }); + h.transport.reconnectToStream.mockResolvedValueOnce(replay); + h.chat.resumeStream({ streamId: 'convo-1', generationCreatedAt: 500 }).then( + () => undefined, + () => undefined, + ); + await flush(); + replay.push({ type: 'pending_action', data: first } as ChatEvent); + await flush(); + } + + expect(h.chat.pendingAction?.actionId).toBe('action-a'); + }, + ); + it.each(['ERR_NETWORK', 'ERR_INTERNET_DISCONNECTED'])( '(2) an answer that fails on the network (%s) reattaches to the paused generation', async (code) => { @@ -2047,9 +2136,186 @@ describe('Chat answers, successors and pause order', () => { expect(h.chat.pendingAction?.actionId).toBe('action-1'); }, ); + + it('(2) an expired newer pause in the same generation clears the older one held', async () => { + const h = createHarness(); + h.chat.restorePendingAction({ ...askAction, createdAt: 2, actionId: 'action-a' }, 500); + h.chat.restorePendingAction( + { ...askAction, createdAt: 3, actionId: 'action-b', expiresAt: 1 }, + 500, + ); + + expect(h.chat.pendingAction).toBeUndefined(); + }); + + it('(2) a status read of an older pause than the one the stream delivered keeps the streamed one', async () => { + const h = createHarness(); + await states.attached(h); + h.stream?.push({ + type: 'pending_action', + data: { ...askAction, createdAt: 3, actionId: 'action-b' }, + } as ChatEvent); + await flush(); + h.stream?.push(dropEvent); + await flush(); + h.chat.restorePendingAction({ ...askAction, createdAt: 2, actionId: 'action-a' }, 100); + + expect(h.chat.pendingAction?.actionId).toBe('action-b'); + }); }); describe('Chat sync snapshots and resume conflicts', () => { + const snapshot = (resumeState: Record) => + ({ + type: 'sync', + data: { sync: true, resumeState: { runSteps: [], ...resumeState } }, + }) as ChatEvent; + + const reattached = async (h: Harness, generationCreatedAt = 500) => { + const stream = createStream({ streamId: 'convo-1', generationCreatedAt }); + h.transport.reconnectToStream.mockResolvedValueOnce(stream); + h.chat.resumeStream({ streamId: 'convo-1', generationCreatedAt }).then( + () => undefined, + () => undefined, + ); + await flush(); + return stream; + }; + + it('(4) a sync snapshot installs the pause it carries, which can then be answered', async () => { + const h = createHarness(); + const stream = await reattached(h); + stream.push(snapshot({ pendingAction: askAction })); + await flush(); + + expect(h.chat.pendingAction?.actionId).toBe('action-1'); + await h.chat.addToolOutput({ toolCallId: 'ask-1', output: 'A' }); + expect(h.transport.resume).toHaveBeenCalledTimes(1); + }); + + it('(2) a sync snapshot carrying an expired newer pause clears the older one held and stays ordered', async () => { + const h = createHarness(); + h.chat.restorePendingAction({ ...askAction, createdAt: 2, actionId: 'action-a' }, 500); + const stream = await reattached(h); + stream.push( + snapshot({ + pendingAction: { ...askAction, createdAt: 3, actionId: 'action-b', expiresAt: 1 }, + }), + ); + await flush(); + expect(h.chat.pendingAction).toBeUndefined(); + + stream.push({ + type: 'pending_action', + data: { ...askAction, createdAt: 2, actionId: 'action-a' }, + } as ChatEvent); + await flush(); + expect(h.chat.pendingAction).toBeUndefined(); + }); + + it('(4) a sync snapshot without a pause releases one answered elsewhere', async () => { + const h = createHarness(); + h.chat.restorePendingAction(askAction, 500); + const stream = await reattached(h); + stream.push(snapshot({})); + await flush(); + + expect(h.chat.pendingAction).toBeUndefined(); + }); + + it('(4) a pause a snapshot released and the stream reports again keeps its decisions', async () => { + const approval = { + ...askAction, + payload: { + type: 'tool_approval', + action_requests: [{ tool_call_id: 'tool-1' }, { tool_call_id: 'tool-2' }], + }, + } as unknown as Agents.PendingAction; + const h = createHarness(); + h.chat.restorePendingAction(approval, 500); + await h.chat.addToolApprovalResponse({ id: 'tool-1', approved: true }); + const stream = await reattached(h); + stream.push(snapshot({})); + stream.push({ type: 'pending_action', data: approval } as ChatEvent); + await flush(); + await h.chat.addToolApprovalResponse({ id: 'tool-2', approved: true }); + + expect(h.transport.resume).toHaveBeenCalledWith( + expect.objectContaining({ + decisions: [ + expect.objectContaining({ tool_call_id: 'tool-1' }), + expect.objectContaining({ tool_call_id: 'tool-2' }), + ], + }), + ); + }); + + it('(4) a pause a snapshot released and a status read restores after a drop keeps its decisions', async () => { + const approval = { + ...askAction, + payload: { + type: 'tool_approval', + action_requests: [{ tool_call_id: 'tool-1' }, { tool_call_id: 'tool-2' }], + }, + } as unknown as Agents.PendingAction; + const h = createHarness(); + h.chat.restorePendingAction(approval, 500); + await h.chat.addToolApprovalResponse({ id: 'tool-1', approved: true }); + const stream = await reattached(h); + stream.push(snapshot({})); + await flush(); + stream.push(dropEvent); + await flush(); + h.chat.restorePendingAction(approval, 500); + await h.chat.addToolApprovalResponse({ id: 'tool-2', approved: true }); + + expect(h.transport.resume).toHaveBeenCalledTimes(1); + expect(h.transport.resume).toHaveBeenCalledWith( + expect.objectContaining({ + decisions: [ + expect.objectContaining({ tool_call_id: 'tool-1' }), + expect.objectContaining({ tool_call_id: 'tool-2' }), + ], + }), + ); + }); + + it('(4) a sync snapshot names the response a later unframed end reports', async () => { + const named: TMessage = { ...placeholder, messageId: 'response-9', text: 'Rebuilt' }; + const h = createHarness({ + messages: [userMessage, named, { ...placeholder, messageId: 'other' }], + }); + const stream = await reattached(h, 700); + stream.push(snapshot({ responseMessageId: 'response-9' })); + await flush(); + stream.push(dropEvent); + await flush(); + + expect(h.finishes[h.finishes.length - 1]).toMatchObject({ isResponseUnknown: false }); + expect(h.finishes[h.finishes.length - 1].message?.messageId).toBe('response-9'); + }); + + it('(7) a sync snapshot names the conversation a chat started under new adopts', async () => { + const h = createHarness({ id: 'new' }); + const stream = createStream({ streamId: createdUnder, generationCreatedAt: 700 }); + h.transport.reconnectToStream.mockResolvedValueOnce(stream); + h.chat.resumeStream({ streamId: createdUnder }).then( + () => undefined, + () => undefined, + ); + await flush(); + stream.push(snapshot({ conversationId: createdUnder })); + stream.push(finalEvent()); + await flush(); + + attach(h, createStream({ streamId: createdUnder, generationCreatedAt: 800 })); + send(h); + await flush(); + expect(h.transport.sendMessages).toHaveBeenLastCalledWith( + expect.objectContaining({ chatId: createdUnder }), + ); + }); + it.each([ ['a replaced run', 409, { code: 'RUN_REPLACED' }], ['an action answered elsewhere', 409, { error: 'No live pending action to resume' }], @@ -2372,6 +2638,35 @@ describe('Chat reattachments that fail over HTTP', () => { }); }); +describe('Chat snapshots the host reducer folds in', () => { + it.each([ + ['an expired pause', { ...askAction, expiresAt: 1 }], + ['a pause older than the one held', { ...askAction, actionId: 'action-old', createdAt: 0 }], + ])('(2) a sync snapshot reaches the host reducer without %s', async (_label, pause) => { + const seen: ChatEvent[] = []; + const h = createHarness({ applyEvent: (event) => seen.push(event) }); + h.chat.restorePendingAction({ ...askAction, createdAt: 1 }, 500); + const stream = createStream({ streamId: 'convo-1', generationCreatedAt: 500 }); + h.transport.reconnectToStream.mockResolvedValueOnce(stream); + h.chat.resumeStream({ streamId: 'convo-1', generationCreatedAt: 500 }).then( + () => undefined, + () => undefined, + ); + await flush(); + stream.push({ + type: 'sync', + data: { sync: true, resumeState: { runSteps: [], pendingAction: pause } }, + } as ChatEvent); + await flush(); + + const sync = seen.find((event) => event.type === 'sync') as Extract< + ChatEvent, + { type: 'sync' } + >; + expect(sync.data.resumeState?.pendingAction).toBeUndefined(); + }); +}); + describe('Chat unframed stops while a lost start is owed', () => { it.each([ [ @@ -2782,6 +3077,38 @@ describe('Chat stop exits and the reload signal', () => { }); }); +describe('Chat lost starts across conversations and turns a retirement starts', () => { + it('refuses a turn whose call retired an expired pause and let a subscriber start another', async () => { + const h = createHarness(); + attach(h, createStream({ streamId: 'convo-1', generationCreatedAt: 100 })); + h.chat.restorePendingAction({ ...askAction, expiresAt: Date.now() + 60_000 }, 500); + const later = Date.now() + 120_000; + const now = jest.spyOn(Date, 'now').mockReturnValue(later); + let queued: Promise | undefined; + let started = false; + const unsubscribe = h.chat.subscribe(() => { + if (!h.chat.pendingAction && !started) { + started = true; + queued = h.chat.sendMessage({ parts: [{ type: 'text', text: 'Queued' }] }); + } + }); + + try { + await expect(h.chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] })).rejects.toThrow( + 'A turn is already in flight for this chat', + ); + } finally { + now.mockRestore(); + unsubscribe(); + } + await flush(); + expect(h.transport.sendMessages).toHaveBeenCalledTimes(1); + h.stream?.push(finalEvent()); + await queued; + expect(h.finishes).toHaveLength(1); + }); +}); + /** Table 1: every pause transition, the pause it leaves held, and the decisions it keeps. */ describe('Chat pause lifecycle and its decision markers', () => { const approval = { @@ -2791,6 +3118,12 @@ describe('Chat pause lifecycle and its decision markers', () => { action_requests: [{ tool_call_id: 'tool-1' }, { tool_call_id: 'tool-2' }], }, } as unknown as Agents.PendingAction; + const newer = { ...approval, actionId: 'action-2', createdAt: 2 } as Agents.PendingAction; + const sync = (pendingAction?: Agents.PendingAction) => + ({ + type: 'sync', + data: { sync: true, resumeState: { runSteps: [], ...(pendingAction && { pendingAction }) } }, + }) as ChatEvent; const reattached = async (h: Harness) => { const stream = createStream({ streamId: 'convo-1', generationCreatedAt: 500 }); h.transport.reconnectToStream.mockResolvedValueOnce(stream); @@ -2824,6 +3157,51 @@ describe('Chat pause lifecycle and its decision markers', () => { async (h) => h.chat.restorePendingAction({ ...approval }, 500), { held: 'action-1', submitted: 2 }, ], + [ + 'a status read restores a newer pause of the generation', + async (h) => h.chat.restorePendingAction(newer, 500), + { held: 'action-2', submitted: 0 }, + ], + [ + 'a sync snapshot omits the held pause', + async (h) => { + const stream = await reattached(h); + stream.push(sync()); + await flush(); + }, + { held: null }, + ], + [ + 'a sync snapshot omits the held pause and the stream reports it again', + async (h) => { + const stream = await reattached(h); + stream.push(sync()); + stream.push({ type: 'pending_action', data: approval } as ChatEvent); + await flush(); + }, + { held: 'action-1', submitted: 2 }, + ], + [ + 'a sync snapshot omits the held pause, the stream drops, and a status read restores it', + async (h) => { + const stream = await reattached(h); + stream.push(sync()); + await flush(); + stream.push(dropEvent); + await flush(); + h.chat.restorePendingAction({ ...approval }, 500); + }, + { held: 'action-1', submitted: 2 }, + ], + [ + 'a sync snapshot names a newer pause', + async (h) => { + const stream = await reattached(h); + stream.push(sync(newer)); + await flush(); + }, + { held: 'action-2', submitted: 0 }, + ], [ 'the stream replays the held pause', async (h) => { @@ -2833,6 +3211,42 @@ describe('Chat pause lifecycle and its decision markers', () => { }, { held: 'action-1', submitted: 2 }, ], + [ + 'the stream reports a newer pause', + async (h) => { + const stream = await reattached(h); + stream.push({ type: 'pending_action', data: newer } as ChatEvent); + await flush(); + }, + { held: 'action-2', submitted: 0 }, + ], + [ + 'its expiry passes with no answer in flight', + async (h) => { + h.chat.restorePendingAction({ ...approval, expiresAt: Date.now() + 20 }, 500); + await new Promise((resolve) => setTimeout(resolve, 60)); + }, + { held: null }, + ], + [ + 'its expiry passes while an answer is in flight, and the answer fails', + async (h) => { + h.chat.restorePendingAction({ ...approval, expiresAt: Date.now() + 20 }, 500); + let refuse: (error: Error) => void = () => undefined; + h.transport.resume.mockImplementationOnce( + () => + new Promise((_, reject) => { + refuse = reject; + }), + ); + const answered = h.chat.addToolApprovalResponse({ id: 'tool-2', approved: true }); + await new Promise((resolve) => setTimeout(resolve, 60)); + refuse(Object.assign(new Error('Unavailable'), { response: { status: 503 } })); + await answered.catch(() => undefined); + await flush(); + }, + { held: null }, + ], [ 'the answer is accepted, and a status read of that pause comes late', async (h) => { @@ -2904,6 +3318,24 @@ describe('Chat pause lifecycle and its decision markers', () => { ); expect(before).toBeGreaterThanOrEqual(0); }); + + it.each([ + ['older than the restored one', { ...approval, responseMessageId: 'response-10' }], + ['that expired', { ...newer, responseMessageId: 'response-10', expiresAt: 1 }], + ])( + 'a replayed pause %s does not rename the response a drop reports', + async (_label, replayed) => { + const h = createHarness({ messages: [userMessage, restoredResponse, laterMessage] }); + h.chat.restorePendingAction({ ...newer, responseMessageId: 'response-9' }, 500); + const stream = await reattached(h); + stream.push({ type: 'pending_action', data: replayed } as unknown as ChatEvent); + await flush(); + stream.push(dropEvent); + await flush(); + + expect(h.finishes[h.finishes.length - 1]?.message?.messageId).toBe('response-9'); + }, + ); }); /** Table 2: every path a lost start can take, and what a stop answers after it. */ @@ -2975,6 +3407,24 @@ describe('Chat notify ordering and request ownership', () => { return { attempt, unsubscribe }; }; + it('a request that retires an expired pause yields to the turn a subscriber started', async () => { + const h = createHarness(); + attach(h, createStream({ streamId: 'convo-1', generationCreatedAt: 100 })); + h.chat.restorePendingAction({ ...askAction, expiresAt: Date.now() + 60_000 }, 500); + const now = jest.spyOn(Date, 'now').mockReturnValue(Date.now() + 120_000); + const { unsubscribe } = startOnNotify(h, () => !h.chat.pendingAction); + try { + await expect(h.chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] })).rejects.toThrow( + 'already in flight', + ); + } finally { + now.mockRestore(); + unsubscribe(); + } + await flush(); + expect(h.transport.sendMessages).toHaveBeenCalledTimes(1); + }); + it('a request owns the chat before it notifies that it was submitted', async () => { const h = createHarness(); attach(h, createStream({ streamId: 'convo-1', generationCreatedAt: 100 })); @@ -3111,6 +3561,19 @@ describe('Chat notify ordering and request ownership', () => { expect(h.chat.status).toBe('submitted'); }); + it('an expiry the timer retires notifies with nothing owned, so the turn a subscriber starts runs', async () => { + const h = createHarness(); + attach(h, createStream({ streamId: 'convo-1', generationCreatedAt: 100 })); + const { attempt, unsubscribe } = startOnNotify(h, () => !h.chat.pendingAction); + h.chat.restorePendingAction({ ...askAction, expiresAt: Date.now() + 20 }, 500); + await new Promise((resolve) => setTimeout(resolve, 60)); + await flush(); + unsubscribe(); + + expect(attempt.error).toBeUndefined(); + expect(h.transport.sendMessages).toHaveBeenCalledTimes(1); + }); + it('a dispose notifies after the chat is disposed, so a turn a subscriber starts is refused', async () => { const h = createHarness(); await states.attached(h); @@ -3795,3 +4258,187 @@ describe('Chat pauses the server refused to resume', () => { expect(h.chat.pendingAction).toBeUndefined(); }); }); + +describe('Chat pauses created in the same millisecond', () => { + const held = { ...askAction, actionId: 'action-1', createdAt: 5 } as Agents.PendingAction; + const expired = { + ...askAction, + actionId: 'action-2', + createdAt: 5, + expiresAt: 1, + } as Agents.PendingAction; + + it('an expired successor a status read reports retires the held pause it ties', () => { + const h = createHarness(); + h.chat.restorePendingAction(held, 500); + h.chat.restorePendingAction(expired, 500); + + expect(h.chat.pendingAction).toBeUndefined(); + }); + + it('an expired successor a reattached stream replays retires the held pause it ties', async () => { + const h = createHarness(); + h.chat.restorePendingAction(held, 500); + const stream = createStream({ streamId: 'convo-1', generationCreatedAt: 500 }); + h.transport.reconnectToStream.mockResolvedValueOnce(stream); + h.chat.resumeStream({ streamId: 'convo-1', generationCreatedAt: 500 }).then( + () => undefined, + () => undefined, + ); + await flush(); + stream.push({ type: 'pending_action', data: expired } as unknown as ChatEvent); + await flush(); + + expect(h.chat.pendingAction).toBeUndefined(); + }); +}); + +describe('Chat pauses a sync frame carries or releases while an answer is in flight', () => { + const reattach = async (h: Harness) => { + const stream = createStream({ streamId: 'convo-1', generationCreatedAt: 500 }); + h.transport.reconnectToStream.mockResolvedValueOnce(stream); + h.chat.resumeStream({ streamId: 'convo-1', generationCreatedAt: 500 }).then( + () => undefined, + () => undefined, + ); + await flush(); + return stream; + }; + + it('a pause buffered in the sync frame is held, so its answer resumes it', async () => { + const h = createHarness(); + const stream = await reattach(h); + stream.push({ + type: 'sync', + data: { + sync: true, + resumeState: { runSteps: [] }, + pendingEvents: [{ event: 'on_pending_action', data: askAction }], + }, + } as unknown as ChatEvent); + await flush(); + + expect(h.chat.pendingAction?.actionId).toBe('action-1'); + await h.chat.addToolOutput({ toolCallId: 'ask-1', output: 'A' }); + expect(h.transport.resume).toHaveBeenCalledWith( + expect.objectContaining({ actionId: 'action-1', answer: 'A' }), + ); + }); + + it('an answer accepted after a snapshot released its pause and the stream dropped still follows the continuation', async () => { + const h = createHarness(); + h.chat.restorePendingAction(askAction, 500); + let answer: (() => void) | undefined; + h.transport.resume.mockImplementationOnce( + () => + new Promise((resolve) => { + answer = () => + resolve({ streamId: 'convo-1', conversationId: 'convo-1', status: 'resuming' }); + }), + ); + const answering = h.chat.addToolOutput({ toolCallId: 'ask-1', output: 'A' }); + await flush(); + const stream = await reattach(h); + stream.push({ + type: 'sync', + data: { sync: true, resumeState: { runSteps: [] } }, + } as unknown as ChatEvent); + await flush(); + stream.push(dropEvent); + await flush(); + h.transport.reconnectToStream.mockClear(); + h.transport.reconnectToStream.mockImplementationOnce(() => new Promise(() => undefined)); + answer?.(); + await answering; + await flush(); + + expect(h.transport.reconnectToStream).toHaveBeenCalledTimes(1); + }); + + it('a stale pause buffered in the sync frame never reaches the host reducer', async () => { + const folded: ChatEvent[] = []; + const h = createHarness({ applyEvent: (event) => folded.push(event) }); + const stream = await reattach(h); + const expired = { ...askAction, expiresAt: 1 } as Agents.PendingAction; + stream.push({ + type: 'sync', + data: { + sync: true, + resumeState: { runSteps: [] }, + pendingEvents: [ + { event: 'on_pending_action', data: expired }, + { type: 'text', text: 'x' }, + ], + }, + } as unknown as ChatEvent); + await flush(); + + const sync = folded.find((event) => event.type === 'sync') as Extract< + ChatEvent, + { type: 'sync' } + >; + expect(sync.data.pendingEvents).toEqual([{ type: 'text', text: 'x' }]); + expect(h.chat.pendingAction).toBeUndefined(); + }); + + it('an answer accepted after the generation of its released pause finished starts no continuation', async () => { + const h = createHarness(); + h.chat.restorePendingAction(askAction, 500); + let answer: (() => void) | undefined; + h.transport.resume.mockImplementationOnce( + () => + new Promise((resolve) => { + answer = () => + resolve({ streamId: 'convo-1', conversationId: 'convo-1', status: 'resuming' }); + }), + ); + const answering = h.chat.addToolOutput({ toolCallId: 'ask-1', output: 'A' }); + await flush(); + const stream = await reattach(h); + stream.push({ + type: 'sync', + data: { sync: true, resumeState: { runSteps: [] } }, + } as unknown as ChatEvent); + await flush(); + stream.push(finalEvent()); + await flush(); + const finishes = h.finishes.length; + h.transport.reconnectToStream.mockClear(); + answer?.(); + await answering; + await flush(); + + expect(h.transport.reconnectToStream).not.toHaveBeenCalled(); + expect(h.finishes).toHaveLength(finishes); + }); + + it('a snapshot pause a newer buffered one supersedes never reaches the host reducer', async () => { + const folded: ChatEvent[] = []; + const h = createHarness({ applyEvent: (event) => folded.push(event) }); + const stream = await reattach(h); + const older = { ...askAction, actionId: 'action-1', createdAt: 1 } as Agents.PendingAction; + const newer = { + ...askAction, + actionId: 'action-2', + createdAt: 2, + expiresAt: 1, + } as Agents.PendingAction; + stream.push({ + type: 'sync', + data: { + sync: true, + resumeState: { runSteps: [], pendingAction: older }, + pendingEvents: [{ event: 'on_pending_action', data: newer }], + }, + } as unknown as ChatEvent); + await flush(); + + const sync = folded.find((event) => event.type === 'sync') as Extract< + ChatEvent, + { type: 'sync' } + >; + expect(sync.data.resumeState?.pendingAction).toBeUndefined(); + expect(sync.data.pendingEvents).toEqual([]); + expect(h.chat.pendingAction).toBeUndefined(); + }); +}); diff --git a/packages/chat/src/core/chat.ts b/packages/chat/src/core/chat.ts index 977f76984f..758148fd0c 100644 --- a/packages/chat/src/core/chat.ts +++ b/packages/chat/src/core/chat.ts @@ -1,4 +1,4 @@ -import { Constants, ContentTypes } from 'librechat-data-provider'; +import { Constants, ContentTypes, ApprovalEvents } from 'librechat-data-provider'; import type { Agents, TPayload, @@ -7,6 +7,7 @@ import type { UITextPart, TPendingSteer, ChatFinalFrame, + ChatSyncFrame, ChatErrorData, TMessageContentParts, UIMessageMetadata, @@ -450,6 +451,28 @@ const getSettledAs = (result: ChatAbortResult): ChatStreamAbortResult['terminalS : undefined; }; +/** The longest delay `setTimeout` honors; a longer one fires at once. */ +const MAX_TIMER_DELAY = 2_147_483_647; + +/** + * The pause a sync frame carries: one buffered after the snapshot, the newest of which stands, + * else the snapshot's own. The server sends a live pause in either place. + */ +const getSnapshotPause = (data: ChatSyncFrame): Agents.PendingAction | undefined => { + const buffered = data.pendingEvents ?? []; + for (let i = buffered.length - 1; i >= 0; i--) { + const frame = buffered[i]; + if ('event' in frame && frame.event === ApprovalEvents.ON_PENDING_ACTION) { + return frame.data; + } + } + return data.resumeState?.pendingAction; +}; + +/** A pause past its expiry, whose resume the server refuses. */ +const isExpired = (action: Agents.PendingAction): boolean => + action.expiresAt != null && action.expiresAt <= Date.now(); + /** * What a stop hands back to the host. Steers come back only from a stop that ended the run: one * that settled first leaves them to its own terminal frame, and an unsaved one has none to trust. @@ -468,6 +491,14 @@ const getStopResult = (result: ChatAbortResult, conversationId?: string): ChatSt }; }; +type PauseOrder = Pick; + +/** + * Orders two pauses of one generation by when the server created them. Action ids are random, so + * two created in the same millisecond tie, and the one this chat received last stands. + */ +const comparePauses = (a: PauseOrder, b: PauseOrder): number => a.createdAt - b.createdAt; + /** The ids a frame names for the conversation and the response its generation produces. */ type FrameIdentity = { conversationId?: string | null; responseMessageId?: string }; @@ -796,8 +827,9 @@ export class Chat { private readonly listeners = new Set<() => void>(); private releaseState?: () => void; private active?: ActiveRequest; - /** The pause the running generation waits on, for a tool decision or an answer. */ - private action?: Agents.PendingAction; + private held?: Agents.PendingAction; + /** Fires at the held pause's expiry, so no later frame is needed to retire it. */ + private expiry?: ReturnType; private generationCreatedAt?: number; /** The pause whose resume request is in flight, so a second submit cannot send it again. */ private resuming?: string; @@ -816,8 +848,12 @@ export class Chat { private readonly decisions = new Map(); /** Pauses already resumed, or refused as over, whose replay or later status read is stale. */ private readonly resumed = new Set(); - /** The newest generation a status read named, below which a read is stale. */ + /** The newest generation a status read named, expired or not, below which a read is stale. */ private newestRead?: number; + /** A pause a snapshot found gone, whose decisions stand if the stream reports it again. */ + private releasedPause?: string; + /** The newest pause this chat saw within its generation, which an older one of that generation trails. */ + private newestPause?: { generationCreatedAt: number } & PauseOrder; /** The newest start handed to the transport, whose conversation a late answer must not replace. */ private latestStart?: ActiveRequest; /** @@ -842,6 +878,45 @@ export class Chat { return this.conversation; } + /** The pause the running generation waits on; holding one arms its retirement at expiry. */ + private get action(): Agents.PendingAction | undefined { + return this.held; + } + + private set action(action: Agents.PendingAction | undefined) { + if (action === this.held) { + return; + } + clearTimeout(this.expiry); + this.expiry = undefined; + this.held = action; + if (action?.expiresAt != null && !this.disposed) { + this.armExpiry(action); + } + } + + /** Retires `action` once its expiry passes, unless an answer to it is in flight and decides. */ + private armExpiry(action: Agents.PendingAction): void { + const remaining = (action.expiresAt ?? 0) - Date.now(); + this.expiry = setTimeout( + () => { + this.expiry = undefined; + if (this.held !== action || this.disposed) { + return; + } + if (!isExpired(action)) { + /** A deadline past the longest delay a timer takes is reached in steps. */ + this.armExpiry(action); + return; + } + if (this.resuming !== action.actionId) { + this.clearAction(); + } + }, + Math.min(Math.max(remaining, 0), MAX_TIMER_DELAY), + ); + } + get status(): ChatStatus { return this.init.state.status; } @@ -1055,6 +1130,7 @@ export class Chat { if (!stream && ended) { this.action = undefined; this.decisions.clear(); + this.releasedPause = undefined; this.settle({ kind: 'complete', settled: { conversationId } }, active); return; } @@ -1236,6 +1312,7 @@ export class Chat { this.detached = undefined; this.action = undefined; this.decisions.clear(); + this.releasedPause = undefined; if (this.publishUnframedStop(result, generation)) { return getStopResult(result, generation.stoppedConversationId); } @@ -1482,6 +1559,7 @@ export class Chat { this.detached = undefined; this.action = undefined; this.decisions.clear(); + this.releasedPause = undefined; if (this.publishUnframedStop(result, generation)) { return getStopResult(result, generation.stoppedConversationId); } @@ -1557,6 +1635,7 @@ export class Chat { } this.action = undefined; this.decisions.clear(); + this.releasedPause = undefined; /** A pause a status read restored names no response this chat prepared or saw framed. */ const unframed = { conversationId: generation.conversationId, @@ -1604,6 +1683,14 @@ export class Chat { } } + /** Whether `action` is the held pause, or the one a snapshot released while nothing replaced it. */ + private holdsPause(action: Agents.PendingAction): boolean { + if (this.action) { + return this.action.actionId === action.actionId; + } + return this.releasedPause === action.actionId; + } + private assertNotResuming(action: Agents.PendingAction): void { if (this.disposed) { throw new Error('The chat was disposed'); @@ -1615,6 +1702,7 @@ export class Chat { private clearAction(): void { this.action = undefined; + this.releasedPause = undefined; this.decisions.clear(); this.notify(); } @@ -1630,7 +1718,8 @@ export class Chat { /** * Restores a pause the server reported outside a stream, such as the status check of a reload, - * so its tool decisions and answers can be submitted. Refused while a request is in flight. + * so its tool decisions and answers can be submitted. Refused while a request is in flight; an + * expired pause is not restored. */ restorePendingAction(action: Agents.PendingAction, generationCreatedAt: number): void { if (this.disposed) { @@ -1643,10 +1732,11 @@ export class Chat { /** A status read before the answer landed reports a pause this chat already resumed. */ return; } - if (this.isStaleRead(generationCreatedAt)) { + if (this.isStaleRead(generationCreatedAt) || this.isOlderPause(action, generationCreatedAt)) { return; } this.newestRead = generationCreatedAt; + this.seePause(action, generationCreatedAt); const dropped = this.detached?.target.generationCreatedAt; if (this.detached && (dropped == null || generationCreatedAt > dropped)) { /** @@ -1660,9 +1750,25 @@ export class Chat { * an older one, so only an answer about the start itself clears it. */ this.learnConversation(action.conversationId); - if (this.action?.actionId !== action.actionId) { + if (isExpired(action)) { + /** + * An expired pause is stale: the server will refuse its resume. It also supersedes a pause + * restored for an older generation, which no longer runs. + */ + const superseded = + this.action != null && + (generationCreatedAt !== this.generationCreatedAt || + this.isSupersededBy(this.action, action)); + if (this.action?.actionId === action.actionId || superseded) { + this.clearAction(); + } + return; + } + if (this.action?.actionId !== action.actionId && this.releasedPause !== action.actionId) { + /** A pause a snapshot released keeps its decisions when a read restores it, as a frame does. */ this.decisions.clear(); } + this.releasedPause = undefined; this.action = action; this.generationCreatedAt = generationCreatedAt; if (this.restored?.createdAt !== generationCreatedAt) { @@ -1777,6 +1883,8 @@ export class Chat { return; } this.disposed = true; + clearTimeout(this.expiry); + this.expiry = undefined; this.lifetime.abort(new Error('The chat was disposed')); const active = this.active; this.active = undefined; @@ -1817,10 +1925,22 @@ export class Chat { if (this.active) { throw new Error('A turn is already in flight for this chat'); } + if (this.action && isExpired(this.action) && this.resuming !== this.action.actionId) { + /** An expired pause whose answer failed blocks nothing: the server refuses its resume. */ + this.clearAction(); + /** A subscriber told the pause is gone may have disposed the chat or started its own turn. */ + if (this.disposed) { + throw new Error('The chat was disposed'); + } + if (this.active) { + throw new Error('A turn is already in flight for this chat'); + } + } if (this.action) { throw new Error('A paused generation is waiting for a decision; answer or stop it first'); } const active = createActiveRequest({ responseless: true }); + this.active = active; try { this.init.state.error = undefined; @@ -2109,7 +2229,12 @@ export class Chat { /** Applies one event; returns the turn's outcome when the event ends it. */ private apply(event: ChatEvent, active: ActiveRequest): Outcome | undefined { - /** A pause this chat already resumed is stale, so no card is drawn for it. */ + /** A pause that expired or that this chat already resumed is stale, so no card is drawn for it. */ + /** A replay of a pause older than one already seen in this generation is stale too. */ + const older = + event.type === 'pending_action' && + this.generationCreatedAt != null && + this.isOlderPause(event.data, this.generationCreatedAt); const stale = event.type === 'pending_action' && this.isStalePause(event.data); const folded = stale ? undefined : this.withoutStalePause(event); const disconnect = @@ -2119,7 +2244,7 @@ export class Chat { * The pause is reconciled before the host reducer folds the event in, so a subscriber its * writes notify never answers a pause the same event ended or replaced. */ - this.reconcilePause(event, stale, disconnect); + this.reconcilePause(event, older, disconnect); if (folded && !this.disposed) { const applyEvent = this.init.applyEvent ?? defaultApplyEvent; applyEvent(folded, { @@ -2161,12 +2286,26 @@ export class Chat { } /** What one event changes in the pause the chat holds and the decisions made for it. */ - private reconcilePause(event: ChatEvent, stale: boolean, disconnect: boolean): void { - if (event.type === 'pending_action' && !stale) { - if (this.action?.actionId !== event.data.actionId) { + private reconcilePause(event: ChatEvent, older: boolean, disconnect: boolean): void { + if (event.type === 'sync' && (event.data.resumeState || event.data.pendingEvents)) { + this.reconcileSnapshotPause(getSnapshotPause(event.data)); + } else if (event.type === 'pending_action' && isExpired(event.data)) { + this.retireExpiredPause(event.data); + } else if ( + event.type === 'pending_action' && + !this.resumed.has(event.data.actionId) && + !older + ) { + /** A pause this chat already resumed is stale when a reattached stream replays it. */ + if ( + this.action?.actionId !== event.data.actionId && + this.releasedPause !== event.data.actionId + ) { /** A reconnect replays the same pause; decisions already made for it still stand. */ this.decisions.clear(); } + this.releasedPause = undefined; + this.seePause(event.data, this.generationCreatedAt); this.action = event.data; } else if (event.type === 'final') { this.action = undefined; @@ -2174,6 +2313,7 @@ export class Chat { /** A dropped connection leaves the server's generation, and its pause, as it was. */ this.action = undefined; this.decisions.clear(); + this.releasedPause = undefined; } } @@ -2263,6 +2403,7 @@ export class Chat { */ this.action = undefined; this.decisions.clear(); + this.releasedPause = undefined; /** Including an attachment that found its generation already finished. */ const ended = active.stream?.generationCreatedAt ?? @@ -2293,7 +2434,7 @@ export class Chat { this.adopt(active.generation, { ...identity, responseMessageId: undefined }); return; } - const snapshot = event.type === 'sync' ? event.data.resumeState?.pendingAction : undefined; + const snapshot = event.type === 'sync' ? getSnapshotPause(event.data) : undefined; if (event.type === 'pending_action') { active.generation.paused = event.data.actionId; } else if (snapshot && !this.isStalePause(snapshot)) { @@ -2338,24 +2479,124 @@ export class Chat { } /** - * The event the host reducer folds in: a sync snapshot drops a pause this chat already resumed, - * as a standalone `pending_action` frame of it would be skipped, so no card is drawn for it. + * Reconciles the pause with a sync snapshot, which carries the live one or none. The server sends + * a live pause either here or as a `pending_action` frame right after, so a snapshot without one + * releases the held pause, keeping its decisions in case that frame reports it again. */ - private withoutStalePause(event: ChatEvent): ChatEvent { - const snapshot = event.type === 'sync' ? event.data.resumeState?.pendingAction : undefined; - if (event.type !== 'sync' || !snapshot) { - return event; + private reconcileSnapshotPause(snapshot: Agents.PendingAction | undefined): void { + const generation = this.generationCreatedAt; + if (!snapshot) { + if (this.action) { + this.releasedPause = this.action.actionId; + this.action = undefined; + this.notify(); + } + return; } - if (!this.isStalePause(snapshot)) { - return event; + if (isExpired(snapshot)) { + this.retireExpiredPause(snapshot); + return; } - const resumeState = { ...event.data.resumeState, pendingAction: undefined }; - return { ...event, data: { ...event.data, resumeState } } as ChatEvent; + const stale = + this.resumed.has(snapshot.actionId) || + (generation != null && this.isOlderPause(snapshot, generation)); + if (stale) { + if (this.action?.actionId === snapshot.actionId) { + this.clearAction(); + } + return; + } + if (this.action?.actionId !== snapshot.actionId && this.releasedPause !== snapshot.actionId) { + this.decisions.clear(); + } + this.releasedPause = undefined; + this.seePause(snapshot, generation); + this.action = snapshot; + this.notify(); } - /** Whether `action` was already resumed by this chat, so a replay of it is stale. */ + /** + * An expired pause a reconnect replays or a snapshot carries is stale, as one a status read + * restores is, and it still supersedes an older pause of its generation that the chat holds. + */ + private retireExpiredPause(expired: Agents.PendingAction): void { + const generation = this.generationCreatedAt; + this.seePause(expired, generation); + const held = this.action; + const superseded = held != null && generation != null && this.isSupersededBy(held, expired); + if (held?.actionId === expired.actionId || superseded) { + this.clearAction(); + } + } + + /** + * The event the host reducer folds in: a sync frame drops a pause that is stale (expired, + * already resumed, or older than one seen), from its snapshot and from the frames it buffers, as a + * standalone `pending_action` frame of it would be skipped, so no card is drawn for a pause the + * chat does not hold. + */ + private withoutStalePause(event: ChatEvent): ChatEvent { + if (event.type !== 'sync') { + return event; + } + const { resumeState, pendingEvents } = event.data; + /** Only the pause the frame stands for, and only while it is live, reaches the reducer. */ + const effective = getSnapshotPause(event.data); + const shown = effective != null && !this.isStalePause(effective) ? effective : undefined; + const snapshot = resumeState?.pendingAction; + const staleSnapshot = snapshot != null && snapshot !== shown; + const live = pendingEvents?.filter( + (frame) => + !('event' in frame) || + frame.event !== ApprovalEvents.ON_PENDING_ACTION || + frame.data === shown, + ); + const staleBuffered = live != null && live.length !== pendingEvents?.length; + if (!staleSnapshot && !staleBuffered) { + return event; + } + const data = { + ...event.data, + ...(staleSnapshot && { resumeState: { ...resumeState, pendingAction: undefined } }), + ...(staleBuffered && { pendingEvents: live }), + }; + return { ...event, data } as ChatEvent; + } + + /** Whether `action` expired, was already resumed, or trails a pause seen in its generation. */ private isStalePause(action: Agents.PendingAction): boolean { - return this.resumed.has(action.actionId); + const generation = this.generationCreatedAt; + return ( + isExpired(action) || + this.resumed.has(action.actionId) || + (generation != null && this.isOlderPause(action, generation)) + ); + } + + /** + * Whether `next`, a pause of the held one's generation, replaces `held`: it was created later, or + * in the same millisecond, where the pause received last stands. + */ + private isSupersededBy(held: Agents.PendingAction, next: Agents.PendingAction): boolean { + return held.actionId !== next.actionId && comparePauses(held, next) <= 0; + } + + /** Whether `action` came before a pause this chat already saw in the same generation. */ + private isOlderPause(action: Agents.PendingAction, generationCreatedAt: number): boolean { + const newest = this.newestPause; + return ( + newest != null && + newest.generationCreatedAt === generationCreatedAt && + comparePauses(action, newest) < 0 + ); + } + + /** Records a pause of `generationCreatedAt` as the newest seen when no later one was. */ + private seePause(action: Agents.PendingAction, generationCreatedAt: number | undefined): void { + if (generationCreatedAt == null || this.isOlderPause(action, generationCreatedAt)) { + return; + } + this.newestPause = { generationCreatedAt, createdAt: action.createdAt }; } /** Adopts the conversation the server created for a chat started under `new`; a real id stays. */ @@ -2482,8 +2723,11 @@ export class Chat { this.resumed.add(action.actionId); /** The answer names the conversation, which a chat started under `new` adopts either way. */ this.learnConversation(result.conversationId); - /** A pause stopped or replaced while the request was in flight is no longer this one's. */ - if (this.action?.actionId !== action.actionId) { + /** + * A pause stopped or replaced while the request was in flight is no longer this one's. One a + * snapshot only released still is: the server took its answer, so its continuation runs. + */ + if (!this.holdsPause(action)) { return; } const reattach = this.active && !this.active.stream ? this.active.reattach : undefined; @@ -2547,12 +2791,7 @@ export class Chat { throw error; } const ambiguous = isRetryableNetworkError(failure); - if ( - ambiguous && - !this.active && - !this.disposed && - this.action?.actionId === action.actionId - ) { + if (ambiguous && !this.active && !this.disposed && this.holdsPause(action)) { /** * The answer may have reached the server, which then runs the generation again. It stays * addressable through a reattachment, which finds the continuation, the pause or the end. @@ -2576,6 +2815,14 @@ export class Chat { } finally { if (this.resuming === action.actionId) { this.resuming = undefined; + const held = this.action; + if (held?.actionId === action.actionId && isExpired(held)) { + /** + * Its expiry passed while the answer was in flight, which then left it unresolved. A + * status read may have restored a copy of it meanwhile, so the held copy is the one read. + */ + this.clearAction(); + } } } }