⏳ feat: Retire Expired and Older Pauses and Reconcile the Held Pause With Sync Snapshots (#17590)

* feat: Retire Expired and Older Pauses and Reconcile the Held Pause With Sync Snapshots

* fix: Let an Expired Pause Retire the Held One It Ties

* fix: Hold a Pause the Sync Frame Buffers and Follow an Answer Whose Pause a Snapshot Released

* fix: Keep a Stale Buffered Pause Out of the Host Reducer

* fix: Retire a Released Pause When Its Generation Ends

* fix: Let Only the Pause a Sync Frame Stands for Reach the Host Reducer
This commit is contained in:
Marco Beretta 2026-10-29 20:50:08 +01:00 • committed by GitHub
parent d4374f02de
commit 80ec78bbfc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 1209 additions and 33 deletions

View file

@ -338,6 +338,106 @@ test.describe('the built @librechat/chat package', () => {
});
});
test('a host sees a start it stopped before the answer as possibly still running @scenario:a-host-sees-a-stopped-start-as-possibly-still-running', () => {
test.setTimeout(120_000);
const dist = resolve(repoRoot, 'packages/chat/dist');
expect(existsSync(dist), 'packages/chat/dist is missing: run npm run build').toBe(true);
/** The start reaches the transport and never answers; the stop cannot name the generation,
* so the turn finishes as a disconnect and every later stop stays the host's to make. */
const probe = `
const { Chat } = require('@librechat/chat');
let messages = [];
const statuses = [];
const finishes = [];
const chat = new Chat({
id: 'convo-1',
state: {
error: undefined,
connection: 'idle',
get status() { return statuses[statuses.length - 1] || 'ready'; },
set status(value) { statuses.push(value); },
get messages() { return messages; },
replaceMessages(next) { messages = next; },
},
transport: {
sendMessages: () => new Promise(() => {}),
reconnectToStream: async () => null,
abort: async () => { throw new Error('no stop should be sent'); },
},
prepareRequest: () => ({
payload: { endpoint: 'agents' },
messages: [{ messageId: 'r1', text: '' }],
}),
onFinish: ({ isAbort, isDisconnect }) => finishes.push({ isAbort, isDisconnect }),
});
const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
setTimeout(async () => {
const first = await chat.stop();
await sent;
const second = await chat.stop();
process.stdout.write(JSON.stringify({ first, second, finishes, status: chat.status }));
}, 0);
`;
const loaded = run(process.execPath, ['-e', probe], { cwd: resolve(repoRoot, 'client') });
expect(loaded.status, loaded.output).toBe(0);
expect(JSON.parse(loaded.stdout)).toEqual({
first: { status: 'unaddressable' },
second: { status: 'unaddressable' },
finishes: [{ isAbort: false, isDisconnect: true }],
status: 'error',
});
});
test('a host sees a pause retired when its expiry passes with no later frame @scenario:a-host-sees-an-expired-pause-retired-without-a-frame', () => {
test.setTimeout(120_000);
const dist = resolve(repoRoot, 'packages/chat/dist');
expect(existsSync(dist), 'packages/chat/dist is missing: run npm run build').toBe(true);
/** A reload restores a pause that expires shortly; nothing else arrives before it does. */
const probe = `
const { Chat } = require('@librechat/chat');
let messages = [];
const sends = [];
const chat = new Chat({
id: 'convo-1',
state: { error: undefined, connection: 'idle', status: 'ready',
get messages() { return messages; },
replaceMessages(next) { messages = next; } },
transport: {
sendMessages: (request) => { sends.push(request.trigger); return new Promise(() => {}); },
reconnectToStream: async () => null,
abort: async () => ({ success: true }),
},
prepareRequest: () => ({ payload: { endpoint: 'agents' } }),
});
let notified = 0;
chat.subscribe(() => { notified += 1; });
chat.restorePendingAction({
actionId: 'action-1', streamId: 'convo-1', conversationId: 'convo-1', createdAt: 1,
expiresAt: Date.now() + 50,
payload: { type: 'ask_user_question', question: { question: 'Which?' }, tool_call_id: 'ask-1' },
}, 500);
const held = chat.pendingAction != null;
const before = notified;
setTimeout(async () => {
const retired = chat.pendingAction === undefined && notified > before;
chat.sendMessage({ parts: [{ type: 'text', text: 'Next' }] });
await new Promise((resolve) => setTimeout(resolve, 0));
process.stdout.write(JSON.stringify({ held, retired, sends }));
chat.dispose();
process.exit(0);
}, 150);
`;
const loaded = run(process.execPath, ['-e', probe], { cwd: resolve(repoRoot, 'client') });
expect(loaded.status, loaded.output).toBe(0);
expect(JSON.parse(loaded.stdout)).toEqual({
held: true,
retired: true,
sends: ['submit-message'],
});
});
test('a host answers a pause on a stream that names no generation @scenario:a-host-answers-a-pause-on-a-stream-without-a-generation', () => {
test.setTimeout(120_000);
const dist = resolve(repoRoot, 'packages/chat/dist');

View file

@ -390,6 +390,28 @@ describe('Chat', () => {
payload: { type: 'ask_user_question', question: { question: 'Which?' }, tool_call_id: 'a' },
} as Agents.PendingAction;
it('stays out of the host reducer when it expired', async () => {
const applyEvent = jest.fn();
const { chat, transport } = setup({
applyEvent,
resumeSelection: () => ({ endpoint: 'agents' }),
});
const replay = createStream({ streamId: 'convo-1', generationCreatedAt: 100 });
transport.reconnectToStream.mockResolvedValueOnce(replay);
const attached = chat.resumeStream({ streamId: 'convo-1', generationCreatedAt: 100 });
await flush();
replay.push({ type: 'pending_action', data: { ...pause, expiresAt: Date.now() - 1_000 } });
await flush();
expect(applyEvent).not.toHaveBeenCalledWith(
expect.objectContaining({ type: 'pending_action' }),
expect.anything(),
);
replay.push(finalEvent());
await attached;
});
it('stays out of the host reducer when this chat already resumed it', async () => {
const applyEvent = jest.fn();
const { chat, transport } = setup({
@ -2627,6 +2649,154 @@ describe('Chat', () => {
await sent;
});
it('does not restore an expired pause, and drops the one it held', async () => {
const { chat } = setup();
chat.restorePendingAction(askAction, 500);
chat.restorePendingAction({ ...askAction, expiresAt: Date.now() - 1 }, 500);
expect(chat.pendingAction).toBeUndefined();
chat.restorePendingAction({ ...approvalAction, expiresAt: Date.now() - 1 }, 500);
expect(chat.pendingAction).toBeUndefined();
});
it('retires a restored pause when its expiry passes without another frame', async () => {
const { chat, transport } = setup();
const listener = jest.fn();
chat.subscribe(listener);
chat.restorePendingAction({ ...askAction, expiresAt: Date.now() + 30 }, 500);
expect(chat.pendingAction).toBeDefined();
listener.mockClear();
await new Promise((resolve) => setTimeout(resolve, 60));
expect(chat.pendingAction).toBeUndefined();
expect(listener).toHaveBeenCalled();
const next = chat.sendMessage({ parts: [{ type: 'text', text: 'Next' }] });
await flush();
expect(transport.sendMessages).toHaveBeenCalledTimes(1);
chat.dispose();
await next;
});
it('retires a streamed pause when its expiry passes while the stream stays quiet', async () => {
const { chat, stream, sent } = await pause({ ...askAction, expiresAt: Date.now() + 30 });
expect(chat.pendingAction?.actionId).toBe(askAction.actionId);
await new Promise((resolve) => setTimeout(resolve, 60));
expect(chat.pendingAction).toBeUndefined();
stream.push(finalEvent());
await sent;
});
it('leaves an expiring pause to the answer in flight, and drops it when that answer fails', async () => {
let refuse: (error: Error) => void = () => undefined;
const resume = jest.fn(
() =>
new Promise<never>((_, reject) => {
refuse = reject;
}),
);
const { chat } = setup({}, { resume });
chat.restorePendingAction({ ...askAction, expiresAt: Date.now() + 30 }, 500);
const answered = chat.addToolOutput({ toolCallId: 'ask-1', output: 'x' });
await new Promise((resolve) => setTimeout(resolve, 60));
expect(chat.pendingAction?.actionId).toBe(askAction.actionId);
refuse(new Error('The pause expired'));
await expect(answered).rejects.toThrow('The pause expired');
expect(chat.pendingAction).toBeUndefined();
});
it('drops an expired copy of the pause a status read restored while its answer was in flight', async () => {
let refuse: (error: Error) => void = () => undefined;
const resume = jest.fn(
() =>
new Promise<never>((_, reject) => {
refuse = reject;
}),
);
const { chat } = setup({}, { resume });
const action = { ...askAction, expiresAt: Date.now() + 30 };
chat.restorePendingAction(action, 500);
const answered = chat.addToolOutput({ toolCallId: 'ask-1', output: 'x' });
await flush();
chat.restorePendingAction({ ...action }, 500);
await new Promise((resolve) => setTimeout(resolve, 60));
expect(chat.pendingAction?.actionId).toBe(askAction.actionId);
refuse(Object.assign(new Error('Service unavailable'), { response: { status: 503 } }));
await expect(answered).rejects.toThrow('Service unavailable');
expect(chat.pendingAction).toBeUndefined();
});
it('stops the expiry of a pause the chat no longer holds or that was disposed', async () => {
const { chat } = setup();
const listener = jest.fn();
chat.subscribe(listener);
chat.restorePendingAction({ ...askAction, expiresAt: Date.now() + 30 }, 500);
chat.dispose();
listener.mockClear();
await new Promise((resolve) => setTimeout(resolve, 60));
expect(listener).not.toHaveBeenCalled();
});
it('reconciles the pause before a host reducer write notifies subscribers', async () => {
const seen: { event: string; actionId?: string }[] = [];
let folding: string | undefined;
const { chat, stream, sent } = await pause(askAction, {
applyEvent: (event, context) => {
folding = event.type;
context.write([...context.messages]);
folding = undefined;
},
});
chat.subscribe(() => {
if (folding) {
seen.push({ event: folding, actionId: chat.pendingAction?.actionId });
}
});
stream.push({
type: 'sync',
data: { sync: true, conversationId: 'convo-1', resumeState: { runSteps: [] } },
} as ChatEvent);
await flush();
stream.push({ type: 'pending_action', data: approvalAction });
await flush();
stream.push(finalEvent());
await sent;
expect(seen).toEqual([
{ event: 'sync', actionId: undefined },
{ event: 'pending_action', actionId: approvalAction.actionId },
{ event: 'final', actionId: undefined },
]);
});
it('drops an expired pause a stream replays, and the one it held under that id', async () => {
const { chat, stream, sent } = await pause(askAction);
expect(chat.pendingAction).toBe(askAction);
stream.push({ type: 'pending_action', data: { ...askAction, expiresAt: Date.now() - 1 } });
await flush();
expect(chat.pendingAction).toBeUndefined();
stream.push({ type: 'pending_action', data: { ...approvalAction, expiresAt: 1 } });
await flush();
expect(chat.pendingAction).toBeUndefined();
stream.push({ type: 'error', status: 0, data: 'offline' } as ChatEvent);
await sent;
await expect(chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] })).resolves.toBe(
undefined,
);
});
it('stops a restored pause of a newer generation, not an older dropped one', async () => {
const { chat, stream, sent, transport } = await pause(askAction, {
resumeSelection: () => ({ endpoint: 'agents' }),
@ -2991,6 +3161,18 @@ describe('Chat', () => {
expect(transport.sendMessages).not.toHaveBeenCalled();
});
it('drops a restored pause once a newer generation is restored with an expired one', async () => {
const { chat } = setup();
chat.restorePendingAction({ ...pausedAction, streamId: 'convo-1' }, 500);
chat.restorePendingAction(
{ ...pausedAction, actionId: 'action-6', streamId: 'convo-1', expiresAt: 1 },
900,
);
expect(chat.pendingAction).toBeUndefined();
});
it('sends no stop for a late start it cannot name', async () => {
let open: ((result: { status: 'stream'; stream: ChatStream }) => void) | undefined;
const late = createStream();

View file

@ -1920,6 +1920,15 @@ describe('Chat status reads out of order', () => {
expect(h.chat.pendingAction).toBeUndefined();
});
it('(2) an expired status read still marks its newer generation, so an older read stays stale', async () => {
const h = createHarness();
h.chat.restorePendingAction(askAction, 500);
h.chat.restorePendingAction({ ...askAction, actionId: 'action-600', expiresAt: 1 }, 600);
h.chat.restorePendingAction({ ...askAction, actionId: 'action-550' }, 550);
expect(h.chat.pendingAction).toBeUndefined();
});
it('(5) an idle stop answers idle without reading a resume selection that throws', async () => {
const h = createHarness({
resumeSelection: () => {
@ -2006,6 +2015,86 @@ describe('Chat answers, successors and pause order', () => {
);
});
it('(2) a status read of an older pause in the same generation keeps the newer one', async () => {
const h = createHarness();
h.chat.restorePendingAction({ ...askAction, createdAt: 3, actionId: 'action-b' }, 500);
h.chat.restorePendingAction({ ...askAction, createdAt: 2, actionId: 'action-a' }, 500);
expect(h.chat.pendingAction?.actionId).toBe('action-b');
});
it('(2) a reconnect replaying an older pause of the generation keeps the newer restored one', async () => {
const h = createHarness();
h.chat.restorePendingAction({ ...askAction, createdAt: 3, actionId: 'action-b' }, 500);
const replay = createStream({ streamId: 'convo-1', generationCreatedAt: 500 });
h.transport.reconnectToStream.mockResolvedValueOnce(replay);
h.chat.resumeStream({ streamId: 'convo-1', generationCreatedAt: 500 }).then(
() => undefined,
() => undefined,
);
await flush();
replay.push({
type: 'pending_action',
data: { ...askAction, createdAt: 2, actionId: 'action-a' },
} as ChatEvent);
await flush();
expect(h.chat.pendingAction?.actionId).toBe('action-b');
});
it('(2) a reconnect replaying an expired newer pause clears the older restored one and stays ordered', async () => {
const h = createHarness();
h.chat.restorePendingAction({ ...askAction, createdAt: 2, actionId: 'action-a' }, 500);
const replay = createStream({ streamId: 'convo-1', generationCreatedAt: 500 });
h.transport.reconnectToStream.mockResolvedValueOnce(replay);
h.chat.resumeStream({ streamId: 'convo-1', generationCreatedAt: 500 }).then(
() => undefined,
() => undefined,
);
await flush();
replay.push({
type: 'pending_action',
data: { ...askAction, createdAt: 3, actionId: 'action-b', expiresAt: 1 },
} as ChatEvent);
await flush();
expect(h.chat.pendingAction).toBeUndefined();
replay.push({
type: 'pending_action',
data: { ...askAction, createdAt: 2, actionId: 'action-a' },
} as ChatEvent);
await flush();
expect(h.chat.pendingAction).toBeUndefined();
});
it.each([
['a status read', 'read'],
['a replay', 'replay'],
])(
'(2) of two pauses created in the same millisecond, the one received last stands through %s',
async (_label, via) => {
const h = createHarness();
const first = { ...askAction, createdAt: 3, actionId: 'action-a' };
const second = { ...askAction, createdAt: 3, actionId: 'action-b' };
h.chat.restorePendingAction(second, 500);
if (via === 'read') {
h.chat.restorePendingAction(first, 500);
} else {
const replay = createStream({ streamId: 'convo-1', generationCreatedAt: 500 });
h.transport.reconnectToStream.mockResolvedValueOnce(replay);
h.chat.resumeStream({ streamId: 'convo-1', generationCreatedAt: 500 }).then(
() => undefined,
() => undefined,
);
await flush();
replay.push({ type: 'pending_action', data: first } as ChatEvent);
await flush();
}
expect(h.chat.pendingAction?.actionId).toBe('action-a');
},
);
it.each(['ERR_NETWORK', 'ERR_INTERNET_DISCONNECTED'])(
'(2) an answer that fails on the network (%s) reattaches to the paused generation',
async (code) => {
@ -2047,9 +2136,186 @@ describe('Chat answers, successors and pause order', () => {
expect(h.chat.pendingAction?.actionId).toBe('action-1');
},
);
it('(2) an expired newer pause in the same generation clears the older one held', async () => {
const h = createHarness();
h.chat.restorePendingAction({ ...askAction, createdAt: 2, actionId: 'action-a' }, 500);
h.chat.restorePendingAction(
{ ...askAction, createdAt: 3, actionId: 'action-b', expiresAt: 1 },
500,
);
expect(h.chat.pendingAction).toBeUndefined();
});
it('(2) a status read of an older pause than the one the stream delivered keeps the streamed one', async () => {
const h = createHarness();
await states.attached(h);
h.stream?.push({
type: 'pending_action',
data: { ...askAction, createdAt: 3, actionId: 'action-b' },
} as ChatEvent);
await flush();
h.stream?.push(dropEvent);
await flush();
h.chat.restorePendingAction({ ...askAction, createdAt: 2, actionId: 'action-a' }, 100);
expect(h.chat.pendingAction?.actionId).toBe('action-b');
});
});
describe('Chat sync snapshots and resume conflicts', () => {
const snapshot = (resumeState: Record<string, unknown>) =>
({
type: 'sync',
data: { sync: true, resumeState: { runSteps: [], ...resumeState } },
}) as ChatEvent;
const reattached = async (h: Harness, generationCreatedAt = 500) => {
const stream = createStream({ streamId: 'convo-1', generationCreatedAt });
h.transport.reconnectToStream.mockResolvedValueOnce(stream);
h.chat.resumeStream({ streamId: 'convo-1', generationCreatedAt }).then(
() => undefined,
() => undefined,
);
await flush();
return stream;
};
it('(4) a sync snapshot installs the pause it carries, which can then be answered', async () => {
const h = createHarness();
const stream = await reattached(h);
stream.push(snapshot({ pendingAction: askAction }));
await flush();
expect(h.chat.pendingAction?.actionId).toBe('action-1');
await h.chat.addToolOutput({ toolCallId: 'ask-1', output: 'A' });
expect(h.transport.resume).toHaveBeenCalledTimes(1);
});
it('(2) a sync snapshot carrying an expired newer pause clears the older one held and stays ordered', async () => {
const h = createHarness();
h.chat.restorePendingAction({ ...askAction, createdAt: 2, actionId: 'action-a' }, 500);
const stream = await reattached(h);
stream.push(
snapshot({
pendingAction: { ...askAction, createdAt: 3, actionId: 'action-b', expiresAt: 1 },
}),
);
await flush();
expect(h.chat.pendingAction).toBeUndefined();
stream.push({
type: 'pending_action',
data: { ...askAction, createdAt: 2, actionId: 'action-a' },
} as ChatEvent);
await flush();
expect(h.chat.pendingAction).toBeUndefined();
});
it('(4) a sync snapshot without a pause releases one answered elsewhere', async () => {
const h = createHarness();
h.chat.restorePendingAction(askAction, 500);
const stream = await reattached(h);
stream.push(snapshot({}));
await flush();
expect(h.chat.pendingAction).toBeUndefined();
});
it('(4) a pause a snapshot released and the stream reports again keeps its decisions', async () => {
const approval = {
...askAction,
payload: {
type: 'tool_approval',
action_requests: [{ tool_call_id: 'tool-1' }, { tool_call_id: 'tool-2' }],
},
} as unknown as Agents.PendingAction;
const h = createHarness();
h.chat.restorePendingAction(approval, 500);
await h.chat.addToolApprovalResponse({ id: 'tool-1', approved: true });
const stream = await reattached(h);
stream.push(snapshot({}));
stream.push({ type: 'pending_action', data: approval } as ChatEvent);
await flush();
await h.chat.addToolApprovalResponse({ id: 'tool-2', approved: true });
expect(h.transport.resume).toHaveBeenCalledWith(
expect.objectContaining({
decisions: [
expect.objectContaining({ tool_call_id: 'tool-1' }),
expect.objectContaining({ tool_call_id: 'tool-2' }),
],
}),
);
});
it('(4) a pause a snapshot released and a status read restores after a drop keeps its decisions', async () => {
const approval = {
...askAction,
payload: {
type: 'tool_approval',
action_requests: [{ tool_call_id: 'tool-1' }, { tool_call_id: 'tool-2' }],
},
} as unknown as Agents.PendingAction;
const h = createHarness();
h.chat.restorePendingAction(approval, 500);
await h.chat.addToolApprovalResponse({ id: 'tool-1', approved: true });
const stream = await reattached(h);
stream.push(snapshot({}));
await flush();
stream.push(dropEvent);
await flush();
h.chat.restorePendingAction(approval, 500);
await h.chat.addToolApprovalResponse({ id: 'tool-2', approved: true });
expect(h.transport.resume).toHaveBeenCalledTimes(1);
expect(h.transport.resume).toHaveBeenCalledWith(
expect.objectContaining({
decisions: [
expect.objectContaining({ tool_call_id: 'tool-1' }),
expect.objectContaining({ tool_call_id: 'tool-2' }),
],
}),
);
});
it('(4) a sync snapshot names the response a later unframed end reports', async () => {
const named: TMessage = { ...placeholder, messageId: 'response-9', text: 'Rebuilt' };
const h = createHarness({
messages: [userMessage, named, { ...placeholder, messageId: 'other' }],
});
const stream = await reattached(h, 700);
stream.push(snapshot({ responseMessageId: 'response-9' }));
await flush();
stream.push(dropEvent);
await flush();
expect(h.finishes[h.finishes.length - 1]).toMatchObject({ isResponseUnknown: false });
expect(h.finishes[h.finishes.length - 1].message?.messageId).toBe('response-9');
});
it('(7) a sync snapshot names the conversation a chat started under new adopts', async () => {
const h = createHarness({ id: 'new' });
const stream = createStream({ streamId: createdUnder, generationCreatedAt: 700 });
h.transport.reconnectToStream.mockResolvedValueOnce(stream);
h.chat.resumeStream({ streamId: createdUnder }).then(
() => undefined,
() => undefined,
);
await flush();
stream.push(snapshot({ conversationId: createdUnder }));
stream.push(finalEvent());
await flush();
attach(h, createStream({ streamId: createdUnder, generationCreatedAt: 800 }));
send(h);
await flush();
expect(h.transport.sendMessages).toHaveBeenLastCalledWith(
expect.objectContaining({ chatId: createdUnder }),
);
});
it.each([
['a replaced run', 409, { code: 'RUN_REPLACED' }],
['an action answered elsewhere', 409, { error: 'No live pending action to resume' }],
@ -2372,6 +2638,35 @@ describe('Chat reattachments that fail over HTTP', () => {
});
});
describe('Chat snapshots the host reducer folds in', () => {
it.each([
['an expired pause', { ...askAction, expiresAt: 1 }],
['a pause older than the one held', { ...askAction, actionId: 'action-old', createdAt: 0 }],
])('(2) a sync snapshot reaches the host reducer without %s', async (_label, pause) => {
const seen: ChatEvent[] = [];
const h = createHarness({ applyEvent: (event) => seen.push(event) });
h.chat.restorePendingAction({ ...askAction, createdAt: 1 }, 500);
const stream = createStream({ streamId: 'convo-1', generationCreatedAt: 500 });
h.transport.reconnectToStream.mockResolvedValueOnce(stream);
h.chat.resumeStream({ streamId: 'convo-1', generationCreatedAt: 500 }).then(
() => undefined,
() => undefined,
);
await flush();
stream.push({
type: 'sync',
data: { sync: true, resumeState: { runSteps: [], pendingAction: pause } },
} as ChatEvent);
await flush();
const sync = seen.find((event) => event.type === 'sync') as Extract<
ChatEvent,
{ type: 'sync' }
>;
expect(sync.data.resumeState?.pendingAction).toBeUndefined();
});
});
describe('Chat unframed stops while a lost start is owed', () => {
it.each([
[
@ -2782,6 +3077,38 @@ describe('Chat stop exits and the reload signal', () => {
});
});
describe('Chat lost starts across conversations and turns a retirement starts', () => {
it('refuses a turn whose call retired an expired pause and let a subscriber start another', async () => {
const h = createHarness();
attach(h, createStream({ streamId: 'convo-1', generationCreatedAt: 100 }));
h.chat.restorePendingAction({ ...askAction, expiresAt: Date.now() + 60_000 }, 500);
const later = Date.now() + 120_000;
const now = jest.spyOn(Date, 'now').mockReturnValue(later);
let queued: Promise<void> | undefined;
let started = false;
const unsubscribe = h.chat.subscribe(() => {
if (!h.chat.pendingAction && !started) {
started = true;
queued = h.chat.sendMessage({ parts: [{ type: 'text', text: 'Queued' }] });
}
});
try {
await expect(h.chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] })).rejects.toThrow(
'A turn is already in flight for this chat',
);
} finally {
now.mockRestore();
unsubscribe();
}
await flush();
expect(h.transport.sendMessages).toHaveBeenCalledTimes(1);
h.stream?.push(finalEvent());
await queued;
expect(h.finishes).toHaveLength(1);
});
});
/** Table 1: every pause transition, the pause it leaves held, and the decisions it keeps. */
describe('Chat pause lifecycle and its decision markers', () => {
const approval = {
@ -2791,6 +3118,12 @@ describe('Chat pause lifecycle and its decision markers', () => {
action_requests: [{ tool_call_id: 'tool-1' }, { tool_call_id: 'tool-2' }],
},
} as unknown as Agents.PendingAction;
const newer = { ...approval, actionId: 'action-2', createdAt: 2 } as Agents.PendingAction;
const sync = (pendingAction?: Agents.PendingAction) =>
({
type: 'sync',
data: { sync: true, resumeState: { runSteps: [], ...(pendingAction && { pendingAction }) } },
}) as ChatEvent;
const reattached = async (h: Harness) => {
const stream = createStream({ streamId: 'convo-1', generationCreatedAt: 500 });
h.transport.reconnectToStream.mockResolvedValueOnce(stream);
@ -2824,6 +3157,51 @@ describe('Chat pause lifecycle and its decision markers', () => {
async (h) => h.chat.restorePendingAction({ ...approval }, 500),
{ held: 'action-1', submitted: 2 },
],
[
'a status read restores a newer pause of the generation',
async (h) => h.chat.restorePendingAction(newer, 500),
{ held: 'action-2', submitted: 0 },
],
[
'a sync snapshot omits the held pause',
async (h) => {
const stream = await reattached(h);
stream.push(sync());
await flush();
},
{ held: null },
],
[
'a sync snapshot omits the held pause and the stream reports it again',
async (h) => {
const stream = await reattached(h);
stream.push(sync());
stream.push({ type: 'pending_action', data: approval } as ChatEvent);
await flush();
},
{ held: 'action-1', submitted: 2 },
],
[
'a sync snapshot omits the held pause, the stream drops, and a status read restores it',
async (h) => {
const stream = await reattached(h);
stream.push(sync());
await flush();
stream.push(dropEvent);
await flush();
h.chat.restorePendingAction({ ...approval }, 500);
},
{ held: 'action-1', submitted: 2 },
],
[
'a sync snapshot names a newer pause',
async (h) => {
const stream = await reattached(h);
stream.push(sync(newer));
await flush();
},
{ held: 'action-2', submitted: 0 },
],
[
'the stream replays the held pause',
async (h) => {
@ -2833,6 +3211,42 @@ describe('Chat pause lifecycle and its decision markers', () => {
},
{ held: 'action-1', submitted: 2 },
],
[
'the stream reports a newer pause',
async (h) => {
const stream = await reattached(h);
stream.push({ type: 'pending_action', data: newer } as ChatEvent);
await flush();
},
{ held: 'action-2', submitted: 0 },
],
[
'its expiry passes with no answer in flight',
async (h) => {
h.chat.restorePendingAction({ ...approval, expiresAt: Date.now() + 20 }, 500);
await new Promise((resolve) => setTimeout(resolve, 60));
},
{ held: null },
],
[
'its expiry passes while an answer is in flight, and the answer fails',
async (h) => {
h.chat.restorePendingAction({ ...approval, expiresAt: Date.now() + 20 }, 500);
let refuse: (error: Error) => void = () => undefined;
h.transport.resume.mockImplementationOnce(
() =>
new Promise<never>((_, reject) => {
refuse = reject;
}),
);
const answered = h.chat.addToolApprovalResponse({ id: 'tool-2', approved: true });
await new Promise((resolve) => setTimeout(resolve, 60));
refuse(Object.assign(new Error('Unavailable'), { response: { status: 503 } }));
await answered.catch(() => undefined);
await flush();
},
{ held: null },
],
[
'the answer is accepted, and a status read of that pause comes late',
async (h) => {
@ -2904,6 +3318,24 @@ describe('Chat pause lifecycle and its decision markers', () => {
);
expect(before).toBeGreaterThanOrEqual(0);
});
it.each([
['older than the restored one', { ...approval, responseMessageId: 'response-10' }],
['that expired', { ...newer, responseMessageId: 'response-10', expiresAt: 1 }],
])(
'a replayed pause %s does not rename the response a drop reports',
async (_label, replayed) => {
const h = createHarness({ messages: [userMessage, restoredResponse, laterMessage] });
h.chat.restorePendingAction({ ...newer, responseMessageId: 'response-9' }, 500);
const stream = await reattached(h);
stream.push({ type: 'pending_action', data: replayed } as unknown as ChatEvent);
await flush();
stream.push(dropEvent);
await flush();
expect(h.finishes[h.finishes.length - 1]?.message?.messageId).toBe('response-9');
},
);
});
/** Table 2: every path a lost start can take, and what a stop answers after it. */
@ -2975,6 +3407,24 @@ describe('Chat notify ordering and request ownership', () => {
return { attempt, unsubscribe };
};
it('a request that retires an expired pause yields to the turn a subscriber started', async () => {
const h = createHarness();
attach(h, createStream({ streamId: 'convo-1', generationCreatedAt: 100 }));
h.chat.restorePendingAction({ ...askAction, expiresAt: Date.now() + 60_000 }, 500);
const now = jest.spyOn(Date, 'now').mockReturnValue(Date.now() + 120_000);
const { unsubscribe } = startOnNotify(h, () => !h.chat.pendingAction);
try {
await expect(h.chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] })).rejects.toThrow(
'already in flight',
);
} finally {
now.mockRestore();
unsubscribe();
}
await flush();
expect(h.transport.sendMessages).toHaveBeenCalledTimes(1);
});
it('a request owns the chat before it notifies that it was submitted', async () => {
const h = createHarness();
attach(h, createStream({ streamId: 'convo-1', generationCreatedAt: 100 }));
@ -3111,6 +3561,19 @@ describe('Chat notify ordering and request ownership', () => {
expect(h.chat.status).toBe('submitted');
});
it('an expiry the timer retires notifies with nothing owned, so the turn a subscriber starts runs', async () => {
const h = createHarness();
attach(h, createStream({ streamId: 'convo-1', generationCreatedAt: 100 }));
const { attempt, unsubscribe } = startOnNotify(h, () => !h.chat.pendingAction);
h.chat.restorePendingAction({ ...askAction, expiresAt: Date.now() + 20 }, 500);
await new Promise((resolve) => setTimeout(resolve, 60));
await flush();
unsubscribe();
expect(attempt.error).toBeUndefined();
expect(h.transport.sendMessages).toHaveBeenCalledTimes(1);
});
it('a dispose notifies after the chat is disposed, so a turn a subscriber starts is refused', async () => {
const h = createHarness();
await states.attached(h);
@ -3795,3 +4258,187 @@ describe('Chat pauses the server refused to resume', () => {
expect(h.chat.pendingAction).toBeUndefined();
});
});
describe('Chat pauses created in the same millisecond', () => {
const held = { ...askAction, actionId: 'action-1', createdAt: 5 } as Agents.PendingAction;
const expired = {
...askAction,
actionId: 'action-2',
createdAt: 5,
expiresAt: 1,
} as Agents.PendingAction;
it('an expired successor a status read reports retires the held pause it ties', () => {
const h = createHarness();
h.chat.restorePendingAction(held, 500);
h.chat.restorePendingAction(expired, 500);
expect(h.chat.pendingAction).toBeUndefined();
});
it('an expired successor a reattached stream replays retires the held pause it ties', async () => {
const h = createHarness();
h.chat.restorePendingAction(held, 500);
const stream = createStream({ streamId: 'convo-1', generationCreatedAt: 500 });
h.transport.reconnectToStream.mockResolvedValueOnce(stream);
h.chat.resumeStream({ streamId: 'convo-1', generationCreatedAt: 500 }).then(
() => undefined,
() => undefined,
);
await flush();
stream.push({ type: 'pending_action', data: expired } as unknown as ChatEvent);
await flush();
expect(h.chat.pendingAction).toBeUndefined();
});
});
describe('Chat pauses a sync frame carries or releases while an answer is in flight', () => {
const reattach = async (h: Harness) => {
const stream = createStream({ streamId: 'convo-1', generationCreatedAt: 500 });
h.transport.reconnectToStream.mockResolvedValueOnce(stream);
h.chat.resumeStream({ streamId: 'convo-1', generationCreatedAt: 500 }).then(
() => undefined,
() => undefined,
);
await flush();
return stream;
};
it('a pause buffered in the sync frame is held, so its answer resumes it', async () => {
const h = createHarness();
const stream = await reattach(h);
stream.push({
type: 'sync',
data: {
sync: true,
resumeState: { runSteps: [] },
pendingEvents: [{ event: 'on_pending_action', data: askAction }],
},
} as unknown as ChatEvent);
await flush();
expect(h.chat.pendingAction?.actionId).toBe('action-1');
await h.chat.addToolOutput({ toolCallId: 'ask-1', output: 'A' });
expect(h.transport.resume).toHaveBeenCalledWith(
expect.objectContaining({ actionId: 'action-1', answer: 'A' }),
);
});
it('an answer accepted after a snapshot released its pause and the stream dropped still follows the continuation', async () => {
const h = createHarness();
h.chat.restorePendingAction(askAction, 500);
let answer: (() => void) | undefined;
h.transport.resume.mockImplementationOnce(
() =>
new Promise((resolve) => {
answer = () =>
resolve({ streamId: 'convo-1', conversationId: 'convo-1', status: 'resuming' });
}),
);
const answering = h.chat.addToolOutput({ toolCallId: 'ask-1', output: 'A' });
await flush();
const stream = await reattach(h);
stream.push({
type: 'sync',
data: { sync: true, resumeState: { runSteps: [] } },
} as unknown as ChatEvent);
await flush();
stream.push(dropEvent);
await flush();
h.transport.reconnectToStream.mockClear();
h.transport.reconnectToStream.mockImplementationOnce(() => new Promise(() => undefined));
answer?.();
await answering;
await flush();
expect(h.transport.reconnectToStream).toHaveBeenCalledTimes(1);
});
it('a stale pause buffered in the sync frame never reaches the host reducer', async () => {
const folded: ChatEvent[] = [];
const h = createHarness({ applyEvent: (event) => folded.push(event) });
const stream = await reattach(h);
const expired = { ...askAction, expiresAt: 1 } as Agents.PendingAction;
stream.push({
type: 'sync',
data: {
sync: true,
resumeState: { runSteps: [] },
pendingEvents: [
{ event: 'on_pending_action', data: expired },
{ type: 'text', text: 'x' },
],
},
} as unknown as ChatEvent);
await flush();
const sync = folded.find((event) => event.type === 'sync') as Extract<
ChatEvent,
{ type: 'sync' }
>;
expect(sync.data.pendingEvents).toEqual([{ type: 'text', text: 'x' }]);
expect(h.chat.pendingAction).toBeUndefined();
});
it('an answer accepted after the generation of its released pause finished starts no continuation', async () => {
const h = createHarness();
h.chat.restorePendingAction(askAction, 500);
let answer: (() => void) | undefined;
h.transport.resume.mockImplementationOnce(
() =>
new Promise((resolve) => {
answer = () =>
resolve({ streamId: 'convo-1', conversationId: 'convo-1', status: 'resuming' });
}),
);
const answering = h.chat.addToolOutput({ toolCallId: 'ask-1', output: 'A' });
await flush();
const stream = await reattach(h);
stream.push({
type: 'sync',
data: { sync: true, resumeState: { runSteps: [] } },
} as unknown as ChatEvent);
await flush();
stream.push(finalEvent());
await flush();
const finishes = h.finishes.length;
h.transport.reconnectToStream.mockClear();
answer?.();
await answering;
await flush();
expect(h.transport.reconnectToStream).not.toHaveBeenCalled();
expect(h.finishes).toHaveLength(finishes);
});
it('a snapshot pause a newer buffered one supersedes never reaches the host reducer', async () => {
const folded: ChatEvent[] = [];
const h = createHarness({ applyEvent: (event) => folded.push(event) });
const stream = await reattach(h);
const older = { ...askAction, actionId: 'action-1', createdAt: 1 } as Agents.PendingAction;
const newer = {
...askAction,
actionId: 'action-2',
createdAt: 2,
expiresAt: 1,
} as Agents.PendingAction;
stream.push({
type: 'sync',
data: {
sync: true,
resumeState: { runSteps: [], pendingAction: older },
pendingEvents: [{ event: 'on_pending_action', data: newer }],
},
} as unknown as ChatEvent);
await flush();
const sync = folded.find((event) => event.type === 'sync') as Extract<
ChatEvent,
{ type: 'sync' }
>;
expect(sync.data.resumeState?.pendingAction).toBeUndefined();
expect(sync.data.pendingEvents).toEqual([]);
expect(h.chat.pendingAction).toBeUndefined();
});
});

View file

@ -1,4 +1,4 @@
import { Constants, ContentTypes } from 'librechat-data-provider';
import { Constants, ContentTypes, ApprovalEvents } from 'librechat-data-provider';
import type {
Agents,
TPayload,
@ -7,6 +7,7 @@ import type {
UITextPart,
TPendingSteer,
ChatFinalFrame,
ChatSyncFrame,
ChatErrorData,
TMessageContentParts,
UIMessageMetadata,
@ -450,6 +451,28 @@ const getSettledAs = (result: ChatAbortResult): ChatStreamAbortResult['terminalS
: undefined;
};
/** The longest delay `setTimeout` honors; a longer one fires at once. */
const MAX_TIMER_DELAY = 2_147_483_647;
/**
* The pause a sync frame carries: one buffered after the snapshot, the newest of which stands,
* else the snapshot's own. The server sends a live pause in either place.
*/
const getSnapshotPause = (data: ChatSyncFrame): Agents.PendingAction | undefined => {
const buffered = data.pendingEvents ?? [];
for (let i = buffered.length - 1; i >= 0; i--) {
const frame = buffered[i];
if ('event' in frame && frame.event === ApprovalEvents.ON_PENDING_ACTION) {
return frame.data;
}
}
return data.resumeState?.pendingAction;
};
/** A pause past its expiry, whose resume the server refuses. */
const isExpired = (action: Agents.PendingAction): boolean =>
action.expiresAt != null && action.expiresAt <= Date.now();
/**
* What a stop hands back to the host. Steers come back only from a stop that ended the run: one
* that settled first leaves them to its own terminal frame, and an unsaved one has none to trust.
@ -468,6 +491,14 @@ const getStopResult = (result: ChatAbortResult, conversationId?: string): ChatSt
};
};
type PauseOrder = Pick<Agents.PendingAction, 'createdAt'>;
/**
* Orders two pauses of one generation by when the server created them. Action ids are random, so
* two created in the same millisecond tie, and the one this chat received last stands.
*/
const comparePauses = (a: PauseOrder, b: PauseOrder): number => a.createdAt - b.createdAt;
/** The ids a frame names for the conversation and the response its generation produces. */
type FrameIdentity = { conversationId?: string | null; responseMessageId?: string };
@ -796,8 +827,9 @@ export class Chat {
private readonly listeners = new Set<() => void>();
private releaseState?: () => void;
private active?: ActiveRequest;
/** The pause the running generation waits on, for a tool decision or an answer. */
private action?: Agents.PendingAction;
private held?: Agents.PendingAction;
/** Fires at the held pause's expiry, so no later frame is needed to retire it. */
private expiry?: ReturnType<typeof setTimeout>;
private generationCreatedAt?: number;
/** The pause whose resume request is in flight, so a second submit cannot send it again. */
private resuming?: string;
@ -816,8 +848,12 @@ export class Chat {
private readonly decisions = new Map<string, Agents.ToolApprovalResolution>();
/** Pauses already resumed, or refused as over, whose replay or later status read is stale. */
private readonly resumed = new Set<string>();
/** The newest generation a status read named, below which a read is stale. */
/** The newest generation a status read named, expired or not, below which a read is stale. */
private newestRead?: number;
/** A pause a snapshot found gone, whose decisions stand if the stream reports it again. */
private releasedPause?: string;
/** The newest pause this chat saw within its generation, which an older one of that generation trails. */
private newestPause?: { generationCreatedAt: number } & PauseOrder;
/** The newest start handed to the transport, whose conversation a late answer must not replace. */
private latestStart?: ActiveRequest;
/**
@ -842,6 +878,45 @@ export class Chat {
return this.conversation;
}
/** The pause the running generation waits on; holding one arms its retirement at expiry. */
private get action(): Agents.PendingAction | undefined {
return this.held;
}
private set action(action: Agents.PendingAction | undefined) {
if (action === this.held) {
return;
}
clearTimeout(this.expiry);
this.expiry = undefined;
this.held = action;
if (action?.expiresAt != null && !this.disposed) {
this.armExpiry(action);
}
}
/** Retires `action` once its expiry passes, unless an answer to it is in flight and decides. */
private armExpiry(action: Agents.PendingAction): void {
const remaining = (action.expiresAt ?? 0) - Date.now();
this.expiry = setTimeout(
() => {
this.expiry = undefined;
if (this.held !== action || this.disposed) {
return;
}
if (!isExpired(action)) {
/** A deadline past the longest delay a timer takes is reached in steps. */
this.armExpiry(action);
return;
}
if (this.resuming !== action.actionId) {
this.clearAction();
}
},
Math.min(Math.max(remaining, 0), MAX_TIMER_DELAY),
);
}
get status(): ChatStatus {
return this.init.state.status;
}
@ -1055,6 +1130,7 @@ export class Chat {
if (!stream && ended) {
this.action = undefined;
this.decisions.clear();
this.releasedPause = undefined;
this.settle({ kind: 'complete', settled: { conversationId } }, active);
return;
}
@ -1236,6 +1312,7 @@ export class Chat {
this.detached = undefined;
this.action = undefined;
this.decisions.clear();
this.releasedPause = undefined;
if (this.publishUnframedStop(result, generation)) {
return getStopResult(result, generation.stoppedConversationId);
}
@ -1482,6 +1559,7 @@ export class Chat {
this.detached = undefined;
this.action = undefined;
this.decisions.clear();
this.releasedPause = undefined;
if (this.publishUnframedStop(result, generation)) {
return getStopResult(result, generation.stoppedConversationId);
}
@ -1557,6 +1635,7 @@ export class Chat {
}
this.action = undefined;
this.decisions.clear();
this.releasedPause = undefined;
/** A pause a status read restored names no response this chat prepared or saw framed. */
const unframed = {
conversationId: generation.conversationId,
@ -1604,6 +1683,14 @@ export class Chat {
}
}
/** Whether `action` is the held pause, or the one a snapshot released while nothing replaced it. */
private holdsPause(action: Agents.PendingAction): boolean {
if (this.action) {
return this.action.actionId === action.actionId;
}
return this.releasedPause === action.actionId;
}
private assertNotResuming(action: Agents.PendingAction): void {
if (this.disposed) {
throw new Error('The chat was disposed');
@ -1615,6 +1702,7 @@ export class Chat {
private clearAction(): void {
this.action = undefined;
this.releasedPause = undefined;
this.decisions.clear();
this.notify();
}
@ -1630,7 +1718,8 @@ export class Chat {
/**
* Restores a pause the server reported outside a stream, such as the status check of a reload,
* so its tool decisions and answers can be submitted. Refused while a request is in flight.
* so its tool decisions and answers can be submitted. Refused while a request is in flight; an
* expired pause is not restored.
*/
restorePendingAction(action: Agents.PendingAction, generationCreatedAt: number): void {
if (this.disposed) {
@ -1643,10 +1732,11 @@ export class Chat {
/** A status read before the answer landed reports a pause this chat already resumed. */
return;
}
if (this.isStaleRead(generationCreatedAt)) {
if (this.isStaleRead(generationCreatedAt) || this.isOlderPause(action, generationCreatedAt)) {
return;
}
this.newestRead = generationCreatedAt;
this.seePause(action, generationCreatedAt);
const dropped = this.detached?.target.generationCreatedAt;
if (this.detached && (dropped == null || generationCreatedAt > dropped)) {
/**
@ -1660,9 +1750,25 @@ export class Chat {
* an older one, so only an answer about the start itself clears it.
*/
this.learnConversation(action.conversationId);
if (this.action?.actionId !== action.actionId) {
if (isExpired(action)) {
/**
* An expired pause is stale: the server will refuse its resume. It also supersedes a pause
* restored for an older generation, which no longer runs.
*/
const superseded =
this.action != null &&
(generationCreatedAt !== this.generationCreatedAt ||
this.isSupersededBy(this.action, action));
if (this.action?.actionId === action.actionId || superseded) {
this.clearAction();
}
return;
}
if (this.action?.actionId !== action.actionId && this.releasedPause !== action.actionId) {
/** A pause a snapshot released keeps its decisions when a read restores it, as a frame does. */
this.decisions.clear();
}
this.releasedPause = undefined;
this.action = action;
this.generationCreatedAt = generationCreatedAt;
if (this.restored?.createdAt !== generationCreatedAt) {
@ -1777,6 +1883,8 @@ export class Chat {
return;
}
this.disposed = true;
clearTimeout(this.expiry);
this.expiry = undefined;
this.lifetime.abort(new Error('The chat was disposed'));
const active = this.active;
this.active = undefined;
@ -1817,10 +1925,22 @@ export class Chat {
if (this.active) {
throw new Error('A turn is already in flight for this chat');
}
if (this.action && isExpired(this.action) && this.resuming !== this.action.actionId) {
/** An expired pause whose answer failed blocks nothing: the server refuses its resume. */
this.clearAction();
/** A subscriber told the pause is gone may have disposed the chat or started its own turn. */
if (this.disposed) {
throw new Error('The chat was disposed');
}
if (this.active) {
throw new Error('A turn is already in flight for this chat');
}
}
if (this.action) {
throw new Error('A paused generation is waiting for a decision; answer or stop it first');
}
const active = createActiveRequest({ responseless: true });
this.active = active;
try {
this.init.state.error = undefined;
@ -2109,7 +2229,12 @@ export class Chat {
/** Applies one event; returns the turn's outcome when the event ends it. */
private apply(event: ChatEvent, active: ActiveRequest): Outcome | undefined {
/** A pause this chat already resumed is stale, so no card is drawn for it. */
/** A pause that expired or that this chat already resumed is stale, so no card is drawn for it. */
/** A replay of a pause older than one already seen in this generation is stale too. */
const older =
event.type === 'pending_action' &&
this.generationCreatedAt != null &&
this.isOlderPause(event.data, this.generationCreatedAt);
const stale = event.type === 'pending_action' && this.isStalePause(event.data);
const folded = stale ? undefined : this.withoutStalePause(event);
const disconnect =
@ -2119,7 +2244,7 @@ export class Chat {
* The pause is reconciled before the host reducer folds the event in, so a subscriber its
* writes notify never answers a pause the same event ended or replaced.
*/
this.reconcilePause(event, stale, disconnect);
this.reconcilePause(event, older, disconnect);
if (folded && !this.disposed) {
const applyEvent = this.init.applyEvent ?? defaultApplyEvent;
applyEvent(folded, {
@ -2161,12 +2286,26 @@ export class Chat {
}
/** What one event changes in the pause the chat holds and the decisions made for it. */
private reconcilePause(event: ChatEvent, stale: boolean, disconnect: boolean): void {
if (event.type === 'pending_action' && !stale) {
if (this.action?.actionId !== event.data.actionId) {
private reconcilePause(event: ChatEvent, older: boolean, disconnect: boolean): void {
if (event.type === 'sync' && (event.data.resumeState || event.data.pendingEvents)) {
this.reconcileSnapshotPause(getSnapshotPause(event.data));
} else if (event.type === 'pending_action' && isExpired(event.data)) {
this.retireExpiredPause(event.data);
} else if (
event.type === 'pending_action' &&
!this.resumed.has(event.data.actionId) &&
!older
) {
/** A pause this chat already resumed is stale when a reattached stream replays it. */
if (
this.action?.actionId !== event.data.actionId &&
this.releasedPause !== event.data.actionId
) {
/** A reconnect replays the same pause; decisions already made for it still stand. */
this.decisions.clear();
}
this.releasedPause = undefined;
this.seePause(event.data, this.generationCreatedAt);
this.action = event.data;
} else if (event.type === 'final') {
this.action = undefined;
@ -2174,6 +2313,7 @@ export class Chat {
/** A dropped connection leaves the server's generation, and its pause, as it was. */
this.action = undefined;
this.decisions.clear();
this.releasedPause = undefined;
}
}
@ -2263,6 +2403,7 @@ export class Chat {
*/
this.action = undefined;
this.decisions.clear();
this.releasedPause = undefined;
/** Including an attachment that found its generation already finished. */
const ended =
active.stream?.generationCreatedAt ??
@ -2293,7 +2434,7 @@ export class Chat {
this.adopt(active.generation, { ...identity, responseMessageId: undefined });
return;
}
const snapshot = event.type === 'sync' ? event.data.resumeState?.pendingAction : undefined;
const snapshot = event.type === 'sync' ? getSnapshotPause(event.data) : undefined;
if (event.type === 'pending_action') {
active.generation.paused = event.data.actionId;
} else if (snapshot && !this.isStalePause(snapshot)) {
@ -2338,24 +2479,124 @@ export class Chat {
}
/**
* The event the host reducer folds in: a sync snapshot drops a pause this chat already resumed,
* as a standalone `pending_action` frame of it would be skipped, so no card is drawn for it.
* Reconciles the pause with a sync snapshot, which carries the live one or none. The server sends
* a live pause either here or as a `pending_action` frame right after, so a snapshot without one
* releases the held pause, keeping its decisions in case that frame reports it again.
*/
private withoutStalePause(event: ChatEvent): ChatEvent {
const snapshot = event.type === 'sync' ? event.data.resumeState?.pendingAction : undefined;
if (event.type !== 'sync' || !snapshot) {
return event;
private reconcileSnapshotPause(snapshot: Agents.PendingAction | undefined): void {
const generation = this.generationCreatedAt;
if (!snapshot) {
if (this.action) {
this.releasedPause = this.action.actionId;
this.action = undefined;
this.notify();
}
return;
}
if (!this.isStalePause(snapshot)) {
return event;
if (isExpired(snapshot)) {
this.retireExpiredPause(snapshot);
return;
}
const resumeState = { ...event.data.resumeState, pendingAction: undefined };
return { ...event, data: { ...event.data, resumeState } } as ChatEvent;
const stale =
this.resumed.has(snapshot.actionId) ||
(generation != null && this.isOlderPause(snapshot, generation));
if (stale) {
if (this.action?.actionId === snapshot.actionId) {
this.clearAction();
}
return;
}
if (this.action?.actionId !== snapshot.actionId && this.releasedPause !== snapshot.actionId) {
this.decisions.clear();
}
this.releasedPause = undefined;
this.seePause(snapshot, generation);
this.action = snapshot;
this.notify();
}
/** Whether `action` was already resumed by this chat, so a replay of it is stale. */
/**
* An expired pause a reconnect replays or a snapshot carries is stale, as one a status read
* restores is, and it still supersedes an older pause of its generation that the chat holds.
*/
private retireExpiredPause(expired: Agents.PendingAction): void {
const generation = this.generationCreatedAt;
this.seePause(expired, generation);
const held = this.action;
const superseded = held != null && generation != null && this.isSupersededBy(held, expired);
if (held?.actionId === expired.actionId || superseded) {
this.clearAction();
}
}
/**
* The event the host reducer folds in: a sync frame drops a pause that is stale (expired,
* already resumed, or older than one seen), from its snapshot and from the frames it buffers, as a
* standalone `pending_action` frame of it would be skipped, so no card is drawn for a pause the
* chat does not hold.
*/
private withoutStalePause(event: ChatEvent): ChatEvent {
if (event.type !== 'sync') {
return event;
}
const { resumeState, pendingEvents } = event.data;
/** Only the pause the frame stands for, and only while it is live, reaches the reducer. */
const effective = getSnapshotPause(event.data);
const shown = effective != null && !this.isStalePause(effective) ? effective : undefined;
const snapshot = resumeState?.pendingAction;
const staleSnapshot = snapshot != null && snapshot !== shown;
const live = pendingEvents?.filter(
(frame) =>
!('event' in frame) ||
frame.event !== ApprovalEvents.ON_PENDING_ACTION ||
frame.data === shown,
);
const staleBuffered = live != null && live.length !== pendingEvents?.length;
if (!staleSnapshot && !staleBuffered) {
return event;
}
const data = {
...event.data,
...(staleSnapshot && { resumeState: { ...resumeState, pendingAction: undefined } }),
...(staleBuffered && { pendingEvents: live }),
};
return { ...event, data } as ChatEvent;
}
/** Whether `action` expired, was already resumed, or trails a pause seen in its generation. */
private isStalePause(action: Agents.PendingAction): boolean {
return this.resumed.has(action.actionId);
const generation = this.generationCreatedAt;
return (
isExpired(action) ||
this.resumed.has(action.actionId) ||
(generation != null && this.isOlderPause(action, generation))
);
}
/**
* Whether `next`, a pause of the held one's generation, replaces `held`: it was created later, or
* in the same millisecond, where the pause received last stands.
*/
private isSupersededBy(held: Agents.PendingAction, next: Agents.PendingAction): boolean {
return held.actionId !== next.actionId && comparePauses(held, next) <= 0;
}
/** Whether `action` came before a pause this chat already saw in the same generation. */
private isOlderPause(action: Agents.PendingAction, generationCreatedAt: number): boolean {
const newest = this.newestPause;
return (
newest != null &&
newest.generationCreatedAt === generationCreatedAt &&
comparePauses(action, newest) < 0
);
}
/** Records a pause of `generationCreatedAt` as the newest seen when no later one was. */
private seePause(action: Agents.PendingAction, generationCreatedAt: number | undefined): void {
if (generationCreatedAt == null || this.isOlderPause(action, generationCreatedAt)) {
return;
}
this.newestPause = { generationCreatedAt, createdAt: action.createdAt };
}
/** Adopts the conversation the server created for a chat started under `new`; a real id stays. */
@ -2482,8 +2723,11 @@ export class Chat {
this.resumed.add(action.actionId);
/** The answer names the conversation, which a chat started under `new` adopts either way. */
this.learnConversation(result.conversationId);
/** A pause stopped or replaced while the request was in flight is no longer this one's. */
if (this.action?.actionId !== action.actionId) {
/**
* A pause stopped or replaced while the request was in flight is no longer this one's. One a
* snapshot only released still is: the server took its answer, so its continuation runs.
*/
if (!this.holdsPause(action)) {
return;
}
const reattach = this.active && !this.active.stream ? this.active.reattach : undefined;
@ -2547,12 +2791,7 @@ export class Chat {
throw error;
}
const ambiguous = isRetryableNetworkError(failure);
if (
ambiguous &&
!this.active &&
!this.disposed &&
this.action?.actionId === action.actionId
) {
if (ambiguous && !this.active && !this.disposed && this.holdsPause(action)) {
/**
* The answer may have reached the server, which then runs the generation again. It stays
* addressable through a reattachment, which finds the continuation, the pause or the end.
@ -2576,6 +2815,14 @@ export class Chat {
} finally {
if (this.resuming === action.actionId) {
this.resuming = undefined;
const held = this.action;
if (held?.actionId === action.actionId && isExpired(held)) {
/**
* Its expiry passed while the answer was in flight, which then left it unresolved. A
* status read may have restored a copy of it meanwhile, so the held copy is the one read.
*/
this.clearAction();
}
}
}
}