🔈 fix: Announce Side-Chat Replies to Screen Readers (#18467)

This commit is contained in:
Danny Avila 2026-11-19 08:13:08 -05:00 • committed by GitHub
parent 8e1ec2c650
commit cd1188e174
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 164 additions and 6 deletions

View file

@ -20,6 +20,7 @@ import { retryTransientQuery } from '~/data-provider/retry';
import useScaledMaxWidth from '~/hooks/useScaledMaxWidth';
import { useChatTransport } from './ChatTransportContext';
import { useChatSettings } from './ChatSettingsContext';
import { useLiveAnnouncer } from './AnnouncerContext';
import { fetchStreamStatus } from '~/data-provider';
/**
@ -104,7 +105,12 @@ function useChat(
},
[queryClient, id],
);
const beside = useMemo(() => ({ onAttachment, onSettled }), [onAttachment, onSettled]);
/** A turn here is said to a screen reader as main chat's is: started, still going, done. */
const { announcePolite: announce } = useLiveAnnouncer();
const beside = useMemo(
() => ({ onAttachment, onSettled, announce }),
[onAttachment, onSettled, announce],
);
return useFollowChat({
id,
transport: generation,

View file

@ -13,6 +13,7 @@ import type {
import type { Transport } from '~/hooks/Chat/contract';
import { findConvoInAllQueries, hasServerFetchedReply, hasLocallyCommittedReply } from '~/utils';
import { ChatTransportContext, defaultChatTransport } from '../ChatTransportContext';
import AnnouncerContext from '../AnnouncerContext';
const mockMessages = jest.fn();
const mockAgent = jest.fn();
@ -105,17 +106,21 @@ const setup = (
client = new QueryClient({ defaultOptions: { queries: { retry: false } } }),
) => {
const harness = createTransport();
const announcePolite = jest.fn();
const announcer = { announcePolite, announceAssertive: announcePolite };
const wrapper = ({ children }: { children: React.ReactNode }) => (
<RecoilRoot>
<QueryClientProvider client={client}>
<ChatTransportContext.Provider value={harness.transport}>
{children}
</ChatTransportContext.Provider>
<AnnouncerContext.Provider value={announcer}>
<ChatTransportContext.Provider value={harness.transport}>
{children}
</ChatTransportContext.Provider>
</AnnouncerContext.Provider>
</QueryClientProvider>
</RecoilRoot>
);
const view = renderHook(() => appSideChatHost.useChat(target), { wrapper });
return { ...view, ...harness, client };
return { ...view, ...harness, client, announcePolite };
};
describe("the app's side chat host", () => {
@ -322,6 +327,43 @@ describe("the app's side chat host", () => {
expect(hasLocallyCommittedReply(client, 'side-conversation', repliedAt)).toBe(true);
});
it("says a turn through the app's announcer, as main chat's is said", async () => {
mockStreamStatus.mockResolvedValue({
active: true,
streamId: 'side-conversation',
createdAt: 2000,
generationProtocolVersion: 2,
});
mockMessages.mockResolvedValue([asked]);
const { attachments, announcePolite } = setup();
await waitFor(() => expect(attachments).toHaveLength(1));
const emit = (event: ChatEvent) => act(() => attachments[0].options.onEvent(event));
emit({ type: 'open' });
emit({
type: 'created',
data: { created: true, message: asked, turn: { responseMessageId: 'response-1' } },
} as ChatEvent);
await waitFor(() =>
expect(announcePolite).toHaveBeenCalledWith({ message: 'start', isStatus: true }),
);
mockStreamStatus.mockResolvedValue({ active: false });
emit({
type: 'final',
data: {
final: true,
conversation: { conversationId: 'side-conversation' },
requestMessage: asked,
responseMessage: answered,
},
} as ChatEvent);
await waitFor(() =>
expect(announcePolite).toHaveBeenLastCalledWith({ message: 'Here it is.' }),
);
expect(announcePolite).toHaveBeenCalledWith({ message: 'end', isStatus: true });
});
it('stops a generation through the app, by its conversation and the stream it names', async () => {
mockStreamStatus.mockResolvedValue({
active: true,

View file

@ -173,6 +173,34 @@ test('a reply read in a side chat is acknowledged @scenario:side-chat-reply-is-m
await expect(row.getByRole('button').first()).not.toHaveAccessibleName(/Unread/);
});
test('a reply in a side chat is read out when it ends @scenario:side-chat-reply-is-announced', async ({
page,
}) => {
test.setTimeout(120_000);
const earlier = uniqueLabel('earlier-said');
const followUp = uniqueLabel('said-beside');
const earlierUrl = await startChat(page, earlier);
await startChat(page, uniqueLabel('current-said'));
await openChatHistory(page);
const row = conversationRow(page, earlierUrl);
await row.hover();
await row.getByRole('button', { name: 'Conversation Menu Options' }).click();
await page.getByRole('menuitem', { name: 'Open beside chat' }).click();
const sideChat = page.getByRole('region', { name: 'Side chat' });
await expect(sideChat).toContainText(replyText(earlier), { timeout: 30_000 });
const composer = sideChat.getByRole('textbox', { name: 'Message input' });
await composer.click();
await page.keyboard.type(replyPrompt(followUp));
await page.keyboard.press('Enter');
await expect(sideChat).toContainText(replyText(followUp), { timeout: 30_000 });
/** The page's polite live region, which is how a screen reader hears main chat's replies. */
const spoken = page.locator('.sr-only > [aria-live="polite"]');
await expect(spoken.filter({ hasText: replyText(followUp) })).toHaveCount(1);
});
test('an unread conversation opened beside the chat is caught up with @scenario:side-chat-catches-up-an-unread-conversation', async ({
page,
}) => {

View file

@ -244,6 +244,63 @@ describe('useFollowChat', () => {
expect(held[1]).toEqual(['task-1:user', 'task-1:assistant']);
});
it('says a turn to a screen reader: started, still being written, done, and what it says', async () => {
const clock = jest.spyOn(Date, 'now').mockReturnValue(1_000_000);
const announce = jest.fn();
const { transport, streams } = createTransport(running);
setup({ transport, beside: { announce } });
await waitFor(() => expect(streams).toHaveLength(1));
const step = (): ChatEvent =>
({
type: 'step',
data: {
event: StepEvents.ON_RUN_STEP,
data: {
id: 's0',
runId: 'task-1:assistant',
index: 0,
type: StepTypes.MESSAGE_CREATION,
stepDetails: {
type: StepTypes.MESSAGE_CREATION,
message_creation: { message_id: 's0' },
},
usage: null,
},
},
}) as ChatEvent;
act(() =>
streams[0].push({
type: 'created',
data: {
created: true,
message: brief,
turn: { responseMessageId: 'task-1:assistant' },
},
} as ChatEvent),
);
await settle();
expect(announce.mock.calls).toEqual([[{ message: 'start', isStatus: true }]]);
/** Soon after the start nothing more is said; later, that the reply is still being written. */
act(() => streams[0].push(step()));
await settle();
expect(announce).toHaveBeenCalledTimes(1);
clock.mockReturnValue(1_008_000);
act(() => streams[0].push(token('Hel')));
act(() => streams[0].push(token('lo')));
await settle();
expect(announce.mock.calls.slice(1)).toEqual([[{ message: 'composing', isStatus: true }]]);
act(() => streams[0].push(final('Hello')));
await settle();
expect(announce.mock.calls.slice(2)).toEqual([
[{ message: 'end', isStatus: true }],
[{ message: 'Hello' }],
]);
clock.mockRestore();
});
it('reads again when the generation ended between the status answer and the attach', async () => {
const { transport } = createTransport(running);
(transport.reconnectToStream as jest.Mock).mockResolvedValue(null);

View file

@ -6,6 +6,7 @@ import type {
ChatTransport,
SubmittedTurn,
ChatConnection,
AnnounceOptions,
ChatFinishResult,
EventApplierInit,
EventApplierEffect,
@ -13,6 +14,8 @@ import type {
import {
Chat,
MAX_RETRIES,
getAllContentText,
MESSAGE_UPDATE_INTERVAL,
getReconnectDelay,
rerunTurnMessages,
createEventApplier,
@ -87,6 +90,12 @@ export type FollowBeside = AppliedBeside & {
* of its history proves the reply it holds, so that reading the reply can be acknowledged.
*/
onSettled?: (final?: ChatFinalFrame) => void;
/**
* What main chat says to a screen reader over a turn, said for this conversation's: that a
* reply has started, that it is still being written, and, at its end, that it is done and
* what it says.
*/
announce?: (options: AnnounceOptions) => void;
};
/** The same handlers, looked up when a frame arrives, so the caller's latest are the ones called. */
@ -311,9 +320,17 @@ export function useFollowChat({
ended = undefined;
host.current.beside?.onSettled?.(final);
};
/** When the reader was last told a reply is being written, so they are not told too often. */
let announced = 0;
const announce = (options: AnnounceOptions): void => host.current.beside?.announce?.(options);
const applier = createEventApplier({
schedule: perFrame,
onEffect: (effect) => host.current.onEffect?.(effect),
onEffect: (effect) => {
if (effect.type === 'announce') {
announce({ message: effect.message, isStatus: true });
}
host.current.onEffect?.(effect);
},
...forwardBeside(() => host.current.beside),
/** Not when the server answered the start with a generation that is not this turn. */
turn: () => (attachment?.foreign === true ? undefined : sent?.submission),
@ -359,6 +376,14 @@ export function useFollowChat({
}
} else if (event.type === 'final') {
ended = event.data;
announce({ message: 'end', isStatus: true });
announce({ message: getAllContentText(event.data.responseMessage as TMessage | null) });
} else if (event.type === 'created') {
announced = Date.now();
announce({ message: 'start', isStatus: true });
} else if (event.type === 'step' && Date.now() - announced > MESSAGE_UPDATE_INTERVAL) {
announced = Date.now();
announce({ message: 'composing', isStatus: true });
}
host.current.onEvent?.(event);
},