diff --git a/client/src/Providers/SideChatHostContext.tsx b/client/src/Providers/SideChatHostContext.tsx index 25d136cf50..ebe8a3b7b3 100644 --- a/client/src/Providers/SideChatHostContext.tsx +++ b/client/src/Providers/SideChatHostContext.tsx @@ -20,6 +20,7 @@ import { retryTransientQuery } from '~/data-provider/retry'; import useScaledMaxWidth from '~/hooks/useScaledMaxWidth'; import { useChatTransport } from './ChatTransportContext'; import { useChatSettings } from './ChatSettingsContext'; +import { useLiveAnnouncer } from './AnnouncerContext'; import { fetchStreamStatus } from '~/data-provider'; /** @@ -104,7 +105,12 @@ function useChat( }, [queryClient, id], ); - const beside = useMemo(() => ({ onAttachment, onSettled }), [onAttachment, onSettled]); + /** A turn here is said to a screen reader as main chat's is: started, still going, done. */ + const { announcePolite: announce } = useLiveAnnouncer(); + const beside = useMemo( + () => ({ onAttachment, onSettled, announce }), + [onAttachment, onSettled, announce], + ); return useFollowChat({ id, transport: generation, diff --git a/client/src/Providers/__tests__/SideChatHostContext.spec.tsx b/client/src/Providers/__tests__/SideChatHostContext.spec.tsx index 49effaeb42..8f715e647c 100644 --- a/client/src/Providers/__tests__/SideChatHostContext.spec.tsx +++ b/client/src/Providers/__tests__/SideChatHostContext.spec.tsx @@ -13,6 +13,7 @@ import type { import type { Transport } from '~/hooks/Chat/contract'; import { findConvoInAllQueries, hasServerFetchedReply, hasLocallyCommittedReply } from '~/utils'; import { ChatTransportContext, defaultChatTransport } from '../ChatTransportContext'; +import AnnouncerContext from '../AnnouncerContext'; const mockMessages = jest.fn(); const mockAgent = jest.fn(); @@ -105,17 +106,21 @@ const setup = ( client = new QueryClient({ defaultOptions: { queries: { retry: false } } }), ) => { const harness = createTransport(); + const announcePolite = jest.fn(); + const announcer = { announcePolite, announceAssertive: announcePolite }; const wrapper = ({ children }: { children: React.ReactNode }) => ( - - {children} - + + + {children} + + ); const view = renderHook(() => appSideChatHost.useChat(target), { wrapper }); - return { ...view, ...harness, client }; + return { ...view, ...harness, client, announcePolite }; }; describe("the app's side chat host", () => { @@ -322,6 +327,43 @@ describe("the app's side chat host", () => { expect(hasLocallyCommittedReply(client, 'side-conversation', repliedAt)).toBe(true); }); + it("says a turn through the app's announcer, as main chat's is said", async () => { + mockStreamStatus.mockResolvedValue({ + active: true, + streamId: 'side-conversation', + createdAt: 2000, + generationProtocolVersion: 2, + }); + mockMessages.mockResolvedValue([asked]); + const { attachments, announcePolite } = setup(); + await waitFor(() => expect(attachments).toHaveLength(1)); + const emit = (event: ChatEvent) => act(() => attachments[0].options.onEvent(event)); + + emit({ type: 'open' }); + emit({ + type: 'created', + data: { created: true, message: asked, turn: { responseMessageId: 'response-1' } }, + } as ChatEvent); + await waitFor(() => + expect(announcePolite).toHaveBeenCalledWith({ message: 'start', isStatus: true }), + ); + + mockStreamStatus.mockResolvedValue({ active: false }); + emit({ + type: 'final', + data: { + final: true, + conversation: { conversationId: 'side-conversation' }, + requestMessage: asked, + responseMessage: answered, + }, + } as ChatEvent); + await waitFor(() => + expect(announcePolite).toHaveBeenLastCalledWith({ message: 'Here it is.' }), + ); + expect(announcePolite).toHaveBeenCalledWith({ message: 'end', isStatus: true }); + }); + it('stops a generation through the app, by its conversation and the stream it names', async () => { mockStreamStatus.mockResolvedValue({ active: true, diff --git a/e2e/specs/mock/scenarios/side-chat-open-beside.spec.ts b/e2e/specs/mock/scenarios/side-chat-open-beside.spec.ts index 6fa81f69c9..eb1ba277dc 100644 --- a/e2e/specs/mock/scenarios/side-chat-open-beside.spec.ts +++ b/e2e/specs/mock/scenarios/side-chat-open-beside.spec.ts @@ -173,6 +173,34 @@ test('a reply read in a side chat is acknowledged @scenario:side-chat-reply-is-m await expect(row.getByRole('button').first()).not.toHaveAccessibleName(/Unread/); }); +test('a reply in a side chat is read out when it ends @scenario:side-chat-reply-is-announced', async ({ + page, +}) => { + test.setTimeout(120_000); + const earlier = uniqueLabel('earlier-said'); + const followUp = uniqueLabel('said-beside'); + const earlierUrl = await startChat(page, earlier); + await startChat(page, uniqueLabel('current-said')); + + await openChatHistory(page); + const row = conversationRow(page, earlierUrl); + await row.hover(); + await row.getByRole('button', { name: 'Conversation Menu Options' }).click(); + await page.getByRole('menuitem', { name: 'Open beside chat' }).click(); + const sideChat = page.getByRole('region', { name: 'Side chat' }); + await expect(sideChat).toContainText(replyText(earlier), { timeout: 30_000 }); + + const composer = sideChat.getByRole('textbox', { name: 'Message input' }); + await composer.click(); + await page.keyboard.type(replyPrompt(followUp)); + await page.keyboard.press('Enter'); + await expect(sideChat).toContainText(replyText(followUp), { timeout: 30_000 }); + + /** The page's polite live region, which is how a screen reader hears main chat's replies. */ + const spoken = page.locator('.sr-only > [aria-live="polite"]'); + await expect(spoken.filter({ hasText: replyText(followUp) })).toHaveCount(1); +}); + test('an unread conversation opened beside the chat is caught up with @scenario:side-chat-catches-up-an-unread-conversation', async ({ page, }) => { diff --git a/packages/chat/src/react/__tests__/follow.spec.tsx b/packages/chat/src/react/__tests__/follow.spec.tsx index 3d54e83d29..bcbb3af1c1 100644 --- a/packages/chat/src/react/__tests__/follow.spec.tsx +++ b/packages/chat/src/react/__tests__/follow.spec.tsx @@ -244,6 +244,63 @@ describe('useFollowChat', () => { expect(held[1]).toEqual(['task-1:user', 'task-1:assistant']); }); + it('says a turn to a screen reader: started, still being written, done, and what it says', async () => { + const clock = jest.spyOn(Date, 'now').mockReturnValue(1_000_000); + const announce = jest.fn(); + const { transport, streams } = createTransport(running); + setup({ transport, beside: { announce } }); + await waitFor(() => expect(streams).toHaveLength(1)); + const step = (): ChatEvent => + ({ + type: 'step', + data: { + event: StepEvents.ON_RUN_STEP, + data: { + id: 's0', + runId: 'task-1:assistant', + index: 0, + type: StepTypes.MESSAGE_CREATION, + stepDetails: { + type: StepTypes.MESSAGE_CREATION, + message_creation: { message_id: 's0' }, + }, + usage: null, + }, + }, + }) as ChatEvent; + + act(() => + streams[0].push({ + type: 'created', + data: { + created: true, + message: brief, + turn: { responseMessageId: 'task-1:assistant' }, + }, + } as ChatEvent), + ); + await settle(); + expect(announce.mock.calls).toEqual([[{ message: 'start', isStatus: true }]]); + + /** Soon after the start nothing more is said; later, that the reply is still being written. */ + act(() => streams[0].push(step())); + await settle(); + expect(announce).toHaveBeenCalledTimes(1); + clock.mockReturnValue(1_008_000); + act(() => streams[0].push(token('Hel'))); + act(() => streams[0].push(token('lo'))); + await settle(); + expect(announce.mock.calls.slice(1)).toEqual([[{ message: 'composing', isStatus: true }]]); + + act(() => streams[0].push(final('Hello'))); + await settle(); + expect(announce.mock.calls.slice(2)).toEqual([ + [{ message: 'end', isStatus: true }], + [{ message: 'Hello' }], + ]); + clock.mockRestore(); + }); + it('reads again when the generation ended between the status answer and the attach', async () => { const { transport } = createTransport(running); (transport.reconnectToStream as jest.Mock).mockResolvedValue(null); diff --git a/packages/chat/src/react/follow.ts b/packages/chat/src/react/follow.ts index d548da52a2..90612c832b 100644 --- a/packages/chat/src/react/follow.ts +++ b/packages/chat/src/react/follow.ts @@ -6,6 +6,7 @@ import type { ChatTransport, SubmittedTurn, ChatConnection, + AnnounceOptions, ChatFinishResult, EventApplierInit, EventApplierEffect, @@ -13,6 +14,8 @@ import type { import { Chat, MAX_RETRIES, + getAllContentText, + MESSAGE_UPDATE_INTERVAL, getReconnectDelay, rerunTurnMessages, createEventApplier, @@ -87,6 +90,12 @@ export type FollowBeside = AppliedBeside & { * of its history proves the reply it holds, so that reading the reply can be acknowledged. */ onSettled?: (final?: ChatFinalFrame) => void; + /** + * What main chat says to a screen reader over a turn, said for this conversation's: that a + * reply has started, that it is still being written, and, at its end, that it is done and + * what it says. + */ + announce?: (options: AnnounceOptions) => void; }; /** The same handlers, looked up when a frame arrives, so the caller's latest are the ones called. */ @@ -311,9 +320,17 @@ export function useFollowChat({ ended = undefined; host.current.beside?.onSettled?.(final); }; + /** When the reader was last told a reply is being written, so they are not told too often. */ + let announced = 0; + const announce = (options: AnnounceOptions): void => host.current.beside?.announce?.(options); const applier = createEventApplier({ schedule: perFrame, - onEffect: (effect) => host.current.onEffect?.(effect), + onEffect: (effect) => { + if (effect.type === 'announce') { + announce({ message: effect.message, isStatus: true }); + } + host.current.onEffect?.(effect); + }, ...forwardBeside(() => host.current.beside), /** Not when the server answered the start with a generation that is not this turn. */ turn: () => (attachment?.foreign === true ? undefined : sent?.submission), @@ -359,6 +376,14 @@ export function useFollowChat({ } } else if (event.type === 'final') { ended = event.data; + announce({ message: 'end', isStatus: true }); + announce({ message: getAllContentText(event.data.responseMessage as TMessage | null) }); + } else if (event.type === 'created') { + announced = Date.now(); + announce({ message: 'start', isStatus: true }); + } else if (event.type === 'step' && Date.now() - announced > MESSAGE_UPDATE_INTERVAL) { + announced = Date.now(); + announce({ message: 'composing', isStatus: true }); } host.current.onEvent?.(event); },