mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-20 06:51:59 +00:00
🔈 fix: Announce Side-Chat Replies to Screen Readers (#18467)
This commit is contained in:
parent
8e1ec2c650
commit
cd1188e174
5 changed files with 164 additions and 6 deletions
|
|
@ -20,6 +20,7 @@ import { retryTransientQuery } from '~/data-provider/retry';
|
|||
import useScaledMaxWidth from '~/hooks/useScaledMaxWidth';
|
||||
import { useChatTransport } from './ChatTransportContext';
|
||||
import { useChatSettings } from './ChatSettingsContext';
|
||||
import { useLiveAnnouncer } from './AnnouncerContext';
|
||||
import { fetchStreamStatus } from '~/data-provider';
|
||||
|
||||
/**
|
||||
|
|
@ -104,7 +105,12 @@ function useChat(
|
|||
},
|
||||
[queryClient, id],
|
||||
);
|
||||
const beside = useMemo(() => ({ onAttachment, onSettled }), [onAttachment, onSettled]);
|
||||
/** A turn here is said to a screen reader as main chat's is: started, still going, done. */
|
||||
const { announcePolite: announce } = useLiveAnnouncer();
|
||||
const beside = useMemo(
|
||||
() => ({ onAttachment, onSettled, announce }),
|
||||
[onAttachment, onSettled, announce],
|
||||
);
|
||||
return useFollowChat({
|
||||
id,
|
||||
transport: generation,
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import type {
|
|||
import type { Transport } from '~/hooks/Chat/contract';
|
||||
import { findConvoInAllQueries, hasServerFetchedReply, hasLocallyCommittedReply } from '~/utils';
|
||||
import { ChatTransportContext, defaultChatTransport } from '../ChatTransportContext';
|
||||
import AnnouncerContext from '../AnnouncerContext';
|
||||
|
||||
const mockMessages = jest.fn();
|
||||
const mockAgent = jest.fn();
|
||||
|
|
@ -105,17 +106,21 @@ const setup = (
|
|||
client = new QueryClient({ defaultOptions: { queries: { retry: false } } }),
|
||||
) => {
|
||||
const harness = createTransport();
|
||||
const announcePolite = jest.fn();
|
||||
const announcer = { announcePolite, announceAssertive: announcePolite };
|
||||
const wrapper = ({ children }: { children: React.ReactNode }) => (
|
||||
<RecoilRoot>
|
||||
<QueryClientProvider client={client}>
|
||||
<ChatTransportContext.Provider value={harness.transport}>
|
||||
{children}
|
||||
</ChatTransportContext.Provider>
|
||||
<AnnouncerContext.Provider value={announcer}>
|
||||
<ChatTransportContext.Provider value={harness.transport}>
|
||||
{children}
|
||||
</ChatTransportContext.Provider>
|
||||
</AnnouncerContext.Provider>
|
||||
</QueryClientProvider>
|
||||
</RecoilRoot>
|
||||
);
|
||||
const view = renderHook(() => appSideChatHost.useChat(target), { wrapper });
|
||||
return { ...view, ...harness, client };
|
||||
return { ...view, ...harness, client, announcePolite };
|
||||
};
|
||||
|
||||
describe("the app's side chat host", () => {
|
||||
|
|
@ -322,6 +327,43 @@ describe("the app's side chat host", () => {
|
|||
expect(hasLocallyCommittedReply(client, 'side-conversation', repliedAt)).toBe(true);
|
||||
});
|
||||
|
||||
it("says a turn through the app's announcer, as main chat's is said", async () => {
|
||||
mockStreamStatus.mockResolvedValue({
|
||||
active: true,
|
||||
streamId: 'side-conversation',
|
||||
createdAt: 2000,
|
||||
generationProtocolVersion: 2,
|
||||
});
|
||||
mockMessages.mockResolvedValue([asked]);
|
||||
const { attachments, announcePolite } = setup();
|
||||
await waitFor(() => expect(attachments).toHaveLength(1));
|
||||
const emit = (event: ChatEvent) => act(() => attachments[0].options.onEvent(event));
|
||||
|
||||
emit({ type: 'open' });
|
||||
emit({
|
||||
type: 'created',
|
||||
data: { created: true, message: asked, turn: { responseMessageId: 'response-1' } },
|
||||
} as ChatEvent);
|
||||
await waitFor(() =>
|
||||
expect(announcePolite).toHaveBeenCalledWith({ message: 'start', isStatus: true }),
|
||||
);
|
||||
|
||||
mockStreamStatus.mockResolvedValue({ active: false });
|
||||
emit({
|
||||
type: 'final',
|
||||
data: {
|
||||
final: true,
|
||||
conversation: { conversationId: 'side-conversation' },
|
||||
requestMessage: asked,
|
||||
responseMessage: answered,
|
||||
},
|
||||
} as ChatEvent);
|
||||
await waitFor(() =>
|
||||
expect(announcePolite).toHaveBeenLastCalledWith({ message: 'Here it is.' }),
|
||||
);
|
||||
expect(announcePolite).toHaveBeenCalledWith({ message: 'end', isStatus: true });
|
||||
});
|
||||
|
||||
it('stops a generation through the app, by its conversation and the stream it names', async () => {
|
||||
mockStreamStatus.mockResolvedValue({
|
||||
active: true,
|
||||
|
|
|
|||
|
|
@ -173,6 +173,34 @@ test('a reply read in a side chat is acknowledged @scenario:side-chat-reply-is-m
|
|||
await expect(row.getByRole('button').first()).not.toHaveAccessibleName(/Unread/);
|
||||
});
|
||||
|
||||
test('a reply in a side chat is read out when it ends @scenario:side-chat-reply-is-announced', async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(120_000);
|
||||
const earlier = uniqueLabel('earlier-said');
|
||||
const followUp = uniqueLabel('said-beside');
|
||||
const earlierUrl = await startChat(page, earlier);
|
||||
await startChat(page, uniqueLabel('current-said'));
|
||||
|
||||
await openChatHistory(page);
|
||||
const row = conversationRow(page, earlierUrl);
|
||||
await row.hover();
|
||||
await row.getByRole('button', { name: 'Conversation Menu Options' }).click();
|
||||
await page.getByRole('menuitem', { name: 'Open beside chat' }).click();
|
||||
const sideChat = page.getByRole('region', { name: 'Side chat' });
|
||||
await expect(sideChat).toContainText(replyText(earlier), { timeout: 30_000 });
|
||||
|
||||
const composer = sideChat.getByRole('textbox', { name: 'Message input' });
|
||||
await composer.click();
|
||||
await page.keyboard.type(replyPrompt(followUp));
|
||||
await page.keyboard.press('Enter');
|
||||
await expect(sideChat).toContainText(replyText(followUp), { timeout: 30_000 });
|
||||
|
||||
/** The page's polite live region, which is how a screen reader hears main chat's replies. */
|
||||
const spoken = page.locator('.sr-only > [aria-live="polite"]');
|
||||
await expect(spoken.filter({ hasText: replyText(followUp) })).toHaveCount(1);
|
||||
});
|
||||
|
||||
test('an unread conversation opened beside the chat is caught up with @scenario:side-chat-catches-up-an-unread-conversation', async ({
|
||||
page,
|
||||
}) => {
|
||||
|
|
|
|||
|
|
@ -244,6 +244,63 @@ describe('useFollowChat', () => {
|
|||
expect(held[1]).toEqual(['task-1:user', 'task-1:assistant']);
|
||||
});
|
||||
|
||||
it('says a turn to a screen reader: started, still being written, done, and what it says', async () => {
|
||||
const clock = jest.spyOn(Date, 'now').mockReturnValue(1_000_000);
|
||||
const announce = jest.fn();
|
||||
const { transport, streams } = createTransport(running);
|
||||
setup({ transport, beside: { announce } });
|
||||
await waitFor(() => expect(streams).toHaveLength(1));
|
||||
const step = (): ChatEvent =>
|
||||
({
|
||||
type: 'step',
|
||||
data: {
|
||||
event: StepEvents.ON_RUN_STEP,
|
||||
data: {
|
||||
id: 's0',
|
||||
runId: 'task-1:assistant',
|
||||
index: 0,
|
||||
type: StepTypes.MESSAGE_CREATION,
|
||||
stepDetails: {
|
||||
type: StepTypes.MESSAGE_CREATION,
|
||||
message_creation: { message_id: 's0' },
|
||||
},
|
||||
usage: null,
|
||||
},
|
||||
},
|
||||
}) as ChatEvent;
|
||||
|
||||
act(() =>
|
||||
streams[0].push({
|
||||
type: 'created',
|
||||
data: {
|
||||
created: true,
|
||||
message: brief,
|
||||
turn: { responseMessageId: 'task-1:assistant' },
|
||||
},
|
||||
} as ChatEvent),
|
||||
);
|
||||
await settle();
|
||||
expect(announce.mock.calls).toEqual([[{ message: 'start', isStatus: true }]]);
|
||||
|
||||
/** Soon after the start nothing more is said; later, that the reply is still being written. */
|
||||
act(() => streams[0].push(step()));
|
||||
await settle();
|
||||
expect(announce).toHaveBeenCalledTimes(1);
|
||||
clock.mockReturnValue(1_008_000);
|
||||
act(() => streams[0].push(token('Hel')));
|
||||
act(() => streams[0].push(token('lo')));
|
||||
await settle();
|
||||
expect(announce.mock.calls.slice(1)).toEqual([[{ message: 'composing', isStatus: true }]]);
|
||||
|
||||
act(() => streams[0].push(final('Hello')));
|
||||
await settle();
|
||||
expect(announce.mock.calls.slice(2)).toEqual([
|
||||
[{ message: 'end', isStatus: true }],
|
||||
[{ message: 'Hello' }],
|
||||
]);
|
||||
clock.mockRestore();
|
||||
});
|
||||
|
||||
it('reads again when the generation ended between the status answer and the attach', async () => {
|
||||
const { transport } = createTransport(running);
|
||||
(transport.reconnectToStream as jest.Mock).mockResolvedValue(null);
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import type {
|
|||
ChatTransport,
|
||||
SubmittedTurn,
|
||||
ChatConnection,
|
||||
AnnounceOptions,
|
||||
ChatFinishResult,
|
||||
EventApplierInit,
|
||||
EventApplierEffect,
|
||||
|
|
@ -13,6 +14,8 @@ import type {
|
|||
import {
|
||||
Chat,
|
||||
MAX_RETRIES,
|
||||
getAllContentText,
|
||||
MESSAGE_UPDATE_INTERVAL,
|
||||
getReconnectDelay,
|
||||
rerunTurnMessages,
|
||||
createEventApplier,
|
||||
|
|
@ -87,6 +90,12 @@ export type FollowBeside = AppliedBeside & {
|
|||
* of its history proves the reply it holds, so that reading the reply can be acknowledged.
|
||||
*/
|
||||
onSettled?: (final?: ChatFinalFrame) => void;
|
||||
/**
|
||||
* What main chat says to a screen reader over a turn, said for this conversation's: that a
|
||||
* reply has started, that it is still being written, and, at its end, that it is done and
|
||||
* what it says.
|
||||
*/
|
||||
announce?: (options: AnnounceOptions) => void;
|
||||
};
|
||||
|
||||
/** The same handlers, looked up when a frame arrives, so the caller's latest are the ones called. */
|
||||
|
|
@ -311,9 +320,17 @@ export function useFollowChat({
|
|||
ended = undefined;
|
||||
host.current.beside?.onSettled?.(final);
|
||||
};
|
||||
/** When the reader was last told a reply is being written, so they are not told too often. */
|
||||
let announced = 0;
|
||||
const announce = (options: AnnounceOptions): void => host.current.beside?.announce?.(options);
|
||||
const applier = createEventApplier({
|
||||
schedule: perFrame,
|
||||
onEffect: (effect) => host.current.onEffect?.(effect),
|
||||
onEffect: (effect) => {
|
||||
if (effect.type === 'announce') {
|
||||
announce({ message: effect.message, isStatus: true });
|
||||
}
|
||||
host.current.onEffect?.(effect);
|
||||
},
|
||||
...forwardBeside(() => host.current.beside),
|
||||
/** Not when the server answered the start with a generation that is not this turn. */
|
||||
turn: () => (attachment?.foreign === true ? undefined : sent?.submission),
|
||||
|
|
@ -359,6 +376,14 @@ export function useFollowChat({
|
|||
}
|
||||
} else if (event.type === 'final') {
|
||||
ended = event.data;
|
||||
announce({ message: 'end', isStatus: true });
|
||||
announce({ message: getAllContentText(event.data.responseMessage as TMessage | null) });
|
||||
} else if (event.type === 'created') {
|
||||
announced = Date.now();
|
||||
announce({ message: 'start', isStatus: true });
|
||||
} else if (event.type === 'step' && Date.now() - announced > MESSAGE_UPDATE_INTERVAL) {
|
||||
announced = Date.now();
|
||||
announce({ message: 'composing', isStatus: true });
|
||||
}
|
||||
host.current.onEvent?.(event);
|
||||
},
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue