🔊 refactor: Move Browser Audio Playback Into the Chat Package (#18182)

- refactor: Move the Browser Audio Driver Into the Chat Package
- fix: Preserve Browser Audio Route Cancellation
- test: Cover Browser Audio Playback in the App
- test: Use Compatible Browser Speech Utterances
- test: Scope Browser Audio Replies to Messages
- refactor: Defer Primary Browser Audio Until Route Adoption
This commit is contained in:
Marco Beretta 2026-11-15 13:35:30 +01:00 • committed by GitHub
parent dd753672f7
commit 28f26cd7fd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 835 additions and 38 deletions

View file

@ -1,7 +1,8 @@
import { useEffect } from 'react';
import { useCallback } from 'react';
import { useSetRecoilState } from 'recoil';
import { useParams } from 'react-router-dom';
import { getSpeechText } from 'librechat-data-provider';
import { Constants } from 'librechat-data-provider';
import { BrowserAudio as BrowserAudioDriver } from '@librechat/chat/components';
import useTextToSpeechBrowser from '~/hooks/Input/useTextToSpeechBrowser';
import useAutoplayTrigger from '~/hooks/Audio/useAutoplayTrigger';
import { logger } from '~/utils';
@ -20,35 +21,30 @@ export default function BrowserAudio({ index = 0 }) {
const { shouldPlay, activeRunId, latestMessage } = useAutoplayTrigger(index);
const { generateSpeechLocal, cancelSpeechLocal } = useTextToSpeechBrowser({ setIsSpeaking });
/**
* Leaving a conversation (or unmounting) stops whatever is still being spoken. React runs
* every cleanup before any effect body, so the utterance the effect below is about to start
* survives the `/c/new` -> `/c/:id` navigation that lands in the same commit as finalization.
*/
useEffect(() => {
return () => cancelSpeechLocal();
// We only want the effect to run when the paramId changes
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [paramId]);
const messageId = latestMessage?.messageId;
// The primary route must adopt a persisted reply before its cleanup can cancel playback.
const awaitingRoute =
index === 0 &&
paramId === Constants.NEW_CONVO &&
!!latestMessage?.conversationId &&
latestMessage.conversationId !== paramId;
const speak = useCallback(
(text: string) => {
logger.log('BrowserAudio.tsx - speaking message:', messageId);
return generateSpeechLocal(text);
},
[generateSpeechLocal, messageId],
);
useEffect(() => {
if (!shouldPlay || activeRunId == null || latestMessage == null) {
return;
}
const text = getSpeechText(latestMessage);
if (!text) {
return;
}
logger.log('BrowserAudio.tsx - speaking message:', latestMessage.messageId);
/** Only claim the run once an utterance was queued, so a not-yet-loaded voice list
* retries on the next render instead of silently swallowing the playback. */
if (generateSpeechLocal(text)) {
setAudioRunId(activeRunId);
}
}, [shouldPlay, activeRunId, latestMessage, setAudioRunId, generateSpeechLocal]);
return null;
return (
<BrowserAudioDriver
routeKey={paramId}
shouldPlay={shouldPlay && !awaitingRoute}
activeRunId={activeRunId}
latestMessage={latestMessage}
speak={speak}
cancel={cancelSpeechLocal}
onStarted={setAudioRunId}
/>
);
}

View file

@ -0,0 +1,243 @@
import React, { useLayoutEffect } from 'react';
import { flushSync } from 'react-dom';
import { getDefaultStore } from 'jotai';
import { RecoilRoot, useRecoilValue } from 'recoil';
import { Constants, QueryKeys } from 'librechat-data-provider';
import { act, render, screen, fireEvent } from '@testing-library/react';
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
import { Link, RouterProvider, useParams, createMemoryRouter } from 'react-router-dom';
import { isSubmittingFamily, resetSubmitting, useIsSubmitting } from '@librechat/chat/react';
import type { TMessage, TConversation } from 'librechat-data-provider';
import { activeRunFamily } from '~/hooks/SSE/atoms';
import BrowserAudio from '../BrowserAudio';
import store from '~/store';
const savedId = 'matrix-chat';
const runId = 'matrix-run';
const voiceName = 'Matrix Voice';
const adoptLabel = 'Adopt saved route';
const exitLabel = 'Different conversation';
const message = {
messageId: 'matrix-reply',
conversationId: savedId,
parentMessageId: Constants.NO_PARENT,
isCreatedByUser: false,
text: 'A finalized matrix answer.',
} as TMessage;
class Utterance extends EventTarget {
voice: SpeechSynthesisVoice | null = null;
onend: ((event: Event) => void) | null = null;
onerror: ((event: Event) => void) | null = null;
constructor(public text: string) {
super();
this.addEventListener('end', (event) => this.onend?.(event));
this.addEventListener('error', (event) => this.onerror?.(event));
}
}
let active: Utterance | null = null;
let committedRoute = '';
let acknowledged: string | null = null;
let voicesAvailable = true;
let voicesChanged = () => {};
const events: string[] = [];
beforeAll(() => {
Object.defineProperty(window, 'speechSynthesis', {
configurable: true,
value: {
getVoices: () => (voicesAvailable ? [{ name: voiceName, localService: true }] : []),
addEventListener: (_event: string, listener: () => void) => {
voicesChanged = listener;
},
speak: (utterance: Utterance) => {
active = utterance;
events.push(`speak:${committedRoute}`);
},
cancel: () => {
const previous = active;
active = null;
events.push(`cancel:${committedRoute}`);
if (previous) {
const event = new Event('error');
Object.defineProperty(event, 'error', { value: 'canceled' });
queueMicrotask(() => previous.dispatchEvent(event));
}
},
},
});
Object.defineProperty(window, 'SpeechSynthesisUtterance', {
configurable: true,
value: Utterance,
});
});
beforeEach(() => {
active = null;
committedRoute = '';
acknowledged = null;
events.length = 0;
voicesAvailable = true;
act(() => voicesChanged());
});
afterEach(() => resetSubmitting(getDefaultStore()));
function Probe({ index }: { index: number }) {
const { conversationId = '' } = useParams();
const audioRun = useRecoilValue(store.audioRunFamily(index));
const submitting = useIsSubmitting(index);
useLayoutEffect(() => {
committedRoute = conversationId;
acknowledged = audioRun;
events.push(`commit:${conversationId}:submitting=${submitting}:ack=${audioRun ?? 'none'}`);
}, [conversationId, audioRun, submitting]);
return <output data-testid="matrix-route">{conversationId}</output>;
}
function setup(index = 0) {
const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } });
queryClient.setQueryData([QueryKeys.messages, 'new'], [message]);
queryClient.setQueryData([QueryKeys.messages, savedId], [message]);
const router = createMemoryRouter(
[
{
path: '/c/:conversationId',
element: (
<>
<Link to={`/c/${savedId}`}>{adoptLabel}</Link>
<Link to="/c/other-chat">{exitLabel}</Link>
<Probe index={index} />
<BrowserAudio index={index} />
</>
),
},
],
{ initialEntries: ['/c/new'] },
);
const view = render(
<QueryClientProvider client={queryClient}>
<RecoilRoot
initializeState={({ set }) => {
set(store.voice, voiceName);
set(store.conversationByIndex(index), { conversationId: savedId } as TConversation);
set(store.audioRunFamily(index), null);
getDefaultStore().set(activeRunFamily(index), runId);
getDefaultStore().set(isSubmittingFamily(index), true);
}}
>
<RouterProvider router={router} />
</RecoilRoot>
</QueryClientProvider>,
);
return { ...view, router };
}
const finalize = (index = 0) => getDefaultStore().set(isSubmittingFamily(index), false);
const adopt = () => fireEvent.click(screen.getByRole('link', { name: adoptLabel }));
const spoken = () => events.filter((event) => event.startsWith('speak:'));
it.each([0, 1])(
'retains speech with a combined eligibility/route commit in pane %i',
async (index) => {
const { router } = setup(index);
await act(async () => {
flushSync(() => {
finalize(index);
void router.navigate(`/c/${savedId}`, { flushSync: true });
});
});
expect(committedRoute).toBe(savedId);
expect(active?.text).toBe(message.text);
expect(acknowledged).toBe(runId);
expect(spoken()).toEqual([`speak:${savedId}`]);
expect(events.some((event) => event.startsWith('commit:new:submitting=false'))).toBe(false);
},
);
it('defers primary-pane speech and acknowledgement until saved-route adoption', async () => {
setup();
await act(async () => finalize());
expect(committedRoute).toBe('new');
expect(active).toBeNull();
expect(acknowledged).toBeNull();
expect(spoken()).toEqual([]);
await act(async () => adopt());
expect(committedRoute).toBe(savedId);
expect(active?.text).toBe(message.text);
expect(acknowledged).toBe(runId);
expect(spoken()).toEqual([`speak:${savedId}`]);
expect(events.filter((event) => !event.startsWith('commit:'))).toEqual([
`cancel:${savedId}`,
`cancel:${savedId}`,
`speak:${savedId}`,
]);
expect(events.at(-1)).toBe(`commit:${savedId}:submitting=false:ack=${runId}`);
});
it('allows an additional pane to speak its own conversation before primary-route adoption', async () => {
setup(1);
await act(async () => finalize(1));
expect(committedRoute).toBe('new');
expect(active?.text).toBe(message.text);
expect(acknowledged).toBe(runId);
expect(spoken()).toEqual(['speak:new']);
});
it.each(['before-adoption', 'after-adoption'])(
'retries delayed voices %s without premature acknowledgement or replay',
async (arrival) => {
const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
voicesAvailable = false;
act(() => voicesChanged());
setup();
await act(async () => finalize());
const loadVoices = () => {
voicesAvailable = true;
voicesChanged();
};
if (arrival === 'before-adoption') {
await act(async () => loadVoices());
expect(active).toBeNull();
expect(acknowledged).toBeNull();
await act(async () => adopt());
} else {
await act(async () => adopt());
expect(active).toBeNull();
expect(acknowledged).toBeNull();
await act(async () => loadVoices());
}
expect(active?.text).toBe(message.text);
expect(acknowledged).toBe(runId);
expect(spoken()).toEqual([`speak:${savedId}`]);
await act(async () => voicesChanged());
expect(active?.text).toBe(message.text);
expect(spoken()).toHaveLength(1);
warn.mockRestore();
},
);
it.each([0, 1])('cancels accepted speech on a different conversation in pane %i', async (index) => {
setup(index);
await act(async () => adopt());
await act(async () => finalize(index));
expect(active?.text).toBe(message.text);
await act(async () => fireEvent.click(screen.getByRole('link', { name: exitLabel })));
expect(committedRoute).toBe('other-chat');
expect(active).toBeNull();
expect(acknowledged).toBe(runId);
expect(spoken()).toHaveLength(1);
});
it.each([0, 1])('cancels accepted speech on unmount in pane %i', async (index) => {
const view = setup(index);
await act(async () => adopt());
await act(async () => finalize(index));
expect(active?.text).toBe(message.text);
await act(async () => view.unmount());
expect(active).toBeNull();
expect(acknowledged).toBe(runId);
expect(spoken()).toHaveLength(1);
});

View file

@ -1,11 +1,11 @@
import React from 'react';
import { RecoilRoot } from 'recoil';
import { getDefaultStore } from 'jotai';
import { render, waitFor } from '@testing-library/react';
import { MemoryRouter, Routes, Route } from 'react-router-dom';
import { RecoilRoot, useRecoilValue } from 'recoil';
import { Link, MemoryRouter, Routes, Route } from 'react-router-dom';
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
import { isSubmittingFamily, resetSubmitting } from '@librechat/chat/react';
import { Constants, QueryKeys, ContentTypes } from 'librechat-data-provider';
import { act, render, screen, waitFor, fireEvent } from '@testing-library/react';
import type { TMessage, TConversation } from 'librechat-data-provider';
import { activeRunFamily } from '~/hooks/SSE/atoms';
import BrowserAudio from '../BrowserAudio';
@ -14,8 +14,10 @@ import store from '~/store';
const conversationId = 'convo-1';
const voiceName = 'Test Voice';
const responseText = 'The capital of Germany is Berlin.';
const nextChatLabel = 'Next chat';
const spoken: string[] = [];
const speechEvents: string[] = [];
let cancelCount = 0;
class FakeSpeechSynthesisUtterance {
@ -26,14 +28,20 @@ class FakeSpeechSynthesisUtterance {
}
const fakeVoice = { name: voiceName, localService: true } as SpeechSynthesisVoice;
let availableVoices = [fakeVoice];
let voicesChanged = () => {};
const speechSynthesis = {
getVoices: () => [fakeVoice],
addEventListener: () => undefined,
getVoices: () => availableVoices,
addEventListener: (_event: string, listener: () => void) => {
voicesChanged = listener;
},
speak: (utterance: FakeSpeechSynthesisUtterance) => {
speechEvents.push('speak');
spoken.push(utterance.text);
},
cancel: () => {
speechEvents.push('cancel');
cancelCount += 1;
},
};
@ -46,16 +54,25 @@ const assistantMessage = {
text: responseText,
} as TMessage;
function AudioRunProbe() {
const audioRunId = useRecoilValue(store.audioRunFamily(0));
return <output data-testid="played-run">{audioRunId ?? 'none'}</output>;
}
const renderBrowserAudio = ({
activeRunId = 'run-1',
audioRunId = null,
isSubmitting = false,
messages = [assistantMessage],
initialPath = `/c/${conversationId}`,
nextPath = '/c/other-chat',
}: {
activeRunId?: string | null;
audioRunId?: string | null;
isSubmitting?: boolean;
messages?: TMessage[];
initialPath?: string;
nextPath?: string;
} = {}) => {
const queryClient = new QueryClient({
defaultOptions: { queries: { retry: false } },
@ -64,7 +81,7 @@ const renderBrowserAudio = ({
return render(
<QueryClientProvider client={queryClient}>
<MemoryRouter initialEntries={[`/c/${conversationId}`]}>
<MemoryRouter initialEntries={[initialPath]}>
<RecoilRoot
initializeState={({ set }) => {
set(store.voice, voiceName);
@ -74,6 +91,8 @@ const renderBrowserAudio = ({
getDefaultStore().set(isSubmittingFamily(0), isSubmitting);
}}
>
<AudioRunProbe />
<Link to={nextPath}>{nextChatLabel}</Link>
<Routes>
<Route path="/c/:conversationId" element={<BrowserAudio index={0} />} />
</Routes>
@ -106,13 +125,37 @@ describe('BrowserAudio autoplay', () => {
beforeEach(() => {
spoken.length = 0;
speechEvents.length = 0;
cancelCount = 0;
availableVoices = [fakeVoice];
act(() => voicesChanged());
});
it('speaks the finalized assistant message through speech synthesis', async () => {
renderBrowserAudio();
await waitFor(() => expect(spoken).toEqual([responseText]));
expect(screen.getByTestId('played-run')).toHaveTextContent('run-1');
});
it('keeps an unqueued run eligible until the voices arrive', async () => {
const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
availableVoices = [];
act(() => voicesChanged());
renderBrowserAudio();
await settle();
expect(spoken).toEqual([]);
expect(screen.getByTestId('played-run')).toHaveTextContent('none');
act(() => {
availableVoices = [fakeVoice];
voicesChanged();
});
await waitFor(() => expect(spoken).toEqual([responseText]));
expect(screen.getByTestId('played-run')).toHaveTextContent('run-1');
warn.mockRestore();
});
it('speaks the answer without the reasoning that preceded it', async () => {
@ -186,4 +229,68 @@ describe('BrowserAudio autoplay', () => {
expect(cancelCount).toBeGreaterThan(0);
});
it('cancels on route navigation while the adapter remains mounted', async () => {
renderBrowserAudio();
await waitFor(() => expect(spoken).toEqual([responseText]));
const cancelsBeforeNavigation = cancelCount;
fireEvent.click(screen.getByRole('link', { name: 'Next chat' }));
expect(cancelCount).toBe(cancelsBeforeNavigation + 1);
expect(spoken).toEqual([responseText]);
});
it('starts the finalized reply after the new-chat route cleans up', async () => {
renderBrowserAudio({
initialPath: '/c/new',
nextPath: `/c/${conversationId}`,
isSubmitting: true,
activeRunId: null,
});
await settle();
expect(spoken).toEqual([]);
act(() => {
getDefaultStore().set(activeRunFamily(0), 'run-1');
getDefaultStore().set(isSubmittingFamily(0), false);
fireEvent.click(screen.getByRole('link', { name: nextChatLabel }));
});
await waitFor(() => expect(spoken).toEqual([responseText]));
expect(screen.getByTestId('played-run')).toHaveTextContent('run-1');
// Route cleanup and the speech service's pre-utterance cancellation both precede speaking.
expect(cancelCount).toBe(2);
expect(speechEvents).toEqual(['cancel', 'cancel', 'speak']);
});
it('does not cancel or replay an accepted utterance when voices update', async () => {
renderBrowserAudio();
await waitFor(() => expect(spoken).toEqual([responseText]));
const cancelsAfterSpeaking = cancelCount;
act(() => voicesChanged());
expect(cancelCount).toBe(cancelsAfterSpeaking);
expect(spoken).toEqual([responseText]);
expect(screen.getByTestId('played-run')).toHaveTextContent('run-1');
});
it('still cancels accepted speech if a later voice refresh fails', async () => {
const { unmount } = renderBrowserAudio();
await waitFor(() => expect(spoken).toEqual([responseText]));
const cancelsAfterSpeaking = cancelCount;
const error = jest.spyOn(console, 'error').mockImplementation(() => {});
const getVoices = jest.spyOn(speechSynthesis, 'getVoices').mockImplementationOnce(() => {
throw new Error('Voice service unavailable');
});
act(() => voicesChanged());
expect(cancelCount).toBe(cancelsAfterSpeaking);
unmount();
expect(cancelCount).toBe(cancelsAfterSpeaking + 1);
getVoices.mockRestore();
error.mockRestore();
});
});

View file

@ -0,0 +1,265 @@
import { randomUUID } from 'node:crypto';
import { expect, test } from '@playwright/test';
import type { Page } from '@playwright/test';
import {
MOCK_ENDPOINTS,
NEW_CHAT_PATH,
replyPrompt,
replyText,
thinkPrompt,
messagesView,
selectMockEndpoint,
sendMessageAndWaitForCompletion,
} from '../helpers';
import { deleteConversations, deleteMessagesByConversation } from '../db';
type SpeechEvent = { type: 'cancel' | 'speak' | 'end' | 'voices' | 'unavailable'; path: string };
type SpeechBoundary = {
events: SpeechEvent[];
spoken: string[];
activeText: string | null;
setVoices: (ready: boolean, fail?: boolean) => void;
finish: () => void;
};
declare global {
interface Window {
browserAudio: SpeechBoundary;
}
}
const createdConversationIds: string[] = [];
test.describe.configure({ timeout: 120_000 });
test.afterAll(async () => {
await deleteMessagesByConversation(createdConversationIds);
await deleteConversations(createdConversationIds);
});
/** Replace the Web Speech boundary; native utterances reject service-fake voice objects. */
async function openChat(page: Page, voicesReady = true) {
await page.addInitScript((ready) => {
for (const [key, value] of Object.entries({
engineSTT: 'browser',
engineTTS: 'browser',
textToSpeech: true,
automaticPlayback: true,
voice: 'E2E local voice',
cloudBrowserVoices: false,
})) {
localStorage.setItem(key, JSON.stringify(value));
}
const target = new EventTarget();
const voice: SpeechSynthesisVoice = {
name: 'E2E local voice',
voiceURI: 'e2e-local',
lang: 'en-US',
localService: true,
default: true,
};
let available = ready;
let fail = false;
class Utterance extends EventTarget {
voice: SpeechSynthesisVoice | null = null;
onend: ((event: Event) => void) | null = null;
onerror: ((event: Event) => void) | null = null;
constructor(public text: string) {
super();
this.addEventListener('end', (event) => this.onend?.(event));
this.addEventListener('error', (event) => this.onerror?.(event));
}
}
Object.defineProperty(window, 'SpeechSynthesisUtterance', {
configurable: true,
value: Utterance,
});
let current: Utterance | null = null;
const record = (type: SpeechEvent['type']) => {
window.browserAudio.events.push({ type, path: location.pathname });
};
window.browserAudio = {
events: [],
spoken: [],
activeText: null,
setVoices: (nextReady, nextFail = false) => {
available = nextReady;
fail = nextFail;
record('voices');
target.dispatchEvent(new Event('voiceschanged'));
},
finish: () => {
const utterance = current;
current = null;
window.browserAudio.activeText = null;
record('end');
utterance?.dispatchEvent(new Event('end'));
},
};
Object.defineProperty(window, 'speechSynthesis', {
configurable: true,
value: {
getVoices: () => {
if (fail) {
record('unavailable');
throw new Error('E2E voice service unavailable');
}
return available ? [voice] : [];
},
addEventListener: target.addEventListener.bind(target),
removeEventListener: target.removeEventListener.bind(target),
speak: (utterance: Utterance) => {
current = utterance;
window.browserAudio.activeText = utterance.text;
window.browserAudio.spoken.push(utterance.text);
record('speak');
},
cancel: () => {
const utterance = current;
current = null;
window.browserAudio.activeText = null;
record('cancel');
if (utterance) {
const event = new Event('error');
Object.defineProperty(event, 'error', { value: 'canceled' });
queueMicrotask(() => utterance.dispatchEvent(event));
}
},
},
});
}, voicesReady);
await page.goto(NEW_CHAT_PATH);
await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
}
async function reply(page: Page, reasoning = false) {
const label = `browser-audio-${randomUUID().slice(0, 8)}`;
const response = await sendMessageAndWaitForCompletion(
page,
reasoning ? thinkPrompt(label) : replyPrompt(label),
);
const { conversationId } = (await response.json()) as { conversationId: string };
createdConversationIds.push(conversationId);
await expect(messagesView(page).getByText(replyText(label), { exact: true })).toBeVisible();
await expect(page.getByRole('button', { name: 'Stop generating' })).toHaveCount(0);
return replyText(label);
}
/** Let the voiceschanged subscription and its passive playback effect reach a browser paint. */
async function refreshVoices(page: Page, fail = false) {
await page.evaluate(async (unavailable) => {
window.browserAudio.setVoices(true, unavailable);
await new Promise<void>((resolve) =>
requestAnimationFrame(() => requestAnimationFrame(() => resolve())),
);
}, fail);
}
const speech = (page: Page) =>
page.evaluate(() => ({
events: window.browserAudio.events,
spoken: window.browserAudio.spoken,
activeText: window.browserAudio.activeText,
}));
async function newChat(page: Page) {
await page
.getByRole('link', { name: 'New chat', exact: true })
.or(page.getByRole('button', { name: 'New chat', exact: true }))
.filter({ visible: true })
.first()
.click();
await expect(page).toHaveURL(/\/c\/new$/);
}
test.describe('browser speech autoplay', () => {
test('speaks only the answer after new-chat cleanup and acknowledges the run without replay @scenario:browser-audio-finalized-answer', async ({
page,
}) => {
await openChat(page);
const answer = await reply(page, true);
await expect.poll(async () => (await speech(page)).spoken).toEqual([answer]);
const queued = await speech(page);
expect(queued.activeText).toBe(answer);
expect(queued.events.map((event) => event.type)).toEqual(['cancel', 'cancel', 'speak']);
expect(queued.events.every((event) => event.path === new URL(page.url()).pathname)).toBe(true);
await page.evaluate(() => window.browserAudio.finish());
await refreshVoices(page);
expect((await speech(page)).spoken).toEqual([answer]);
expect((await speech(page)).activeText).toBeNull();
const nextAnswer = await reply(page);
await expect.poll(async () => (await speech(page)).spoken).toEqual([answer, nextAnswer]);
});
test('leaves a refused run eligible until voices become available @scenario:browser-audio-late-voices', async ({
page,
}) => {
await openChat(page, false);
const answer = await reply(page);
expect((await speech(page)).spoken).toEqual([]);
await refreshVoices(page);
await expect.poll(async () => (await speech(page)).spoken).toEqual([answer]);
await refreshVoices(page);
expect((await speech(page)).spoken).toEqual([answer]);
expect((await speech(page)).activeText).toBe(answer);
});
test('cancels active speech on route navigation and when the chat unmounts @scenario:browser-audio-route-cancellation', async ({
page,
}) => {
await openChat(page);
const answer = await reply(page);
await expect.poll(async () => (await speech(page)).activeText).toBe(answer);
await newChat(page);
await expect.poll(async () => (await speech(page)).activeText).toBeNull();
expect((await speech(page)).events.at(-1)).toEqual({ type: 'cancel', path: NEW_CHAT_PATH });
await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
const nextAnswer = await reply(page);
await expect.poll(async () => (await speech(page)).activeText).toBe(nextAnswer);
await page.evaluate(() => {
history.pushState(history.state, '', '/search');
window.dispatchEvent(new PopStateEvent('popstate'));
});
await expect(page.getByRole('textbox', { name: 'Message input' })).toHaveCount(0);
await expect.poll(async () => (await speech(page)).activeText).toBeNull();
expect((await speech(page)).events.at(-1)).toEqual({ type: 'cancel', path: '/search' });
});
test('retains cancellation after a later voice refresh fails @scenario:browser-audio-failed-voice-refresh', async ({
page,
}) => {
await openChat(page);
const answer = await reply(page);
await expect.poll(async () => (await speech(page)).activeText).toBe(answer);
await refreshVoices(page, true);
expect((await speech(page)).events.at(-1)?.type).toBe('unavailable');
expect((await speech(page)).activeText).toBe(answer);
await newChat(page);
await expect.poll(async () => (await speech(page)).activeText).toBeNull();
expect((await speech(page)).events.at(-1)).toEqual({ type: 'cancel', path: NEW_CHAT_PATH });
expect((await speech(page)).spoken).toEqual([answer]);
});
test('does not speak an empty completed answer @scenario:browser-audio-ineligible-message', async ({
page,
}) => {
await openChat(page);
const response = await sendMessageAndWaitForCompletion(
page,
`E2E_EMPTY_REPLY:browser-audio-${randomUUID().slice(0, 8)}`,
);
const { conversationId } = (await response.json()) as { conversationId: string };
createdConversationIds.push(conversationId);
await expect(page.getByRole('button', { name: 'Stop generating' })).toHaveCount(0);
await refreshVoices(page);
expect((await speech(page)).spoken).toEqual([]);
expect((await speech(page)).activeText).toBeNull();
});
});

View file

@ -17,6 +17,8 @@ export type {
} from './components/Part';
/** Composer: the chat input, its controls and the steering queue. */
export { default as BrowserAudio } from './components/composer/Audio';
export type { BrowserAudioProps } from './components/composer/Audio';
export { default as StopButton } from './components/composer/StopButton';
export { default as MentionItem } from './components/composer/MentionItem';
export { default as MentionPopover } from './components/composer/popover/Mention';

View file

@ -0,0 +1,56 @@
import { useEffect, useRef } from 'react';
import { getSpeechText } from 'librechat-data-provider';
import type { TMessage } from 'librechat-data-provider';
export interface BrowserAudioProps {
/** The route lifetime that owns playback, including the new-chat route. */
routeKey?: string;
shouldPlay: boolean;
activeRunId: string | null;
latestMessage: TMessage | null;
/** Returns true only when an utterance was queued; may change when voices load. */
speak: (text: string) => boolean;
cancel: () => void;
onStarted: (runId: string) => void;
}
/** Browser speech playback, with route, voice configuration and run state supplied by the host. */
export default function BrowserAudio({
routeKey,
shouldPlay,
activeRunId,
latestMessage,
speak,
cancel,
onStarted,
}: BrowserAudioProps): null {
const cancelRef = useRef(cancel);
useEffect(() => {
cancelRef.current = cancel;
}, [cancel]);
/** Cleanup belongs only to the route lifetime, not voice availability or callback identity.
* React runs cleanups before effect bodies, so same-commit new-chat finalization cancels the
* previous utterance before starting the newly finalized reply. */
useEffect(() => {
// Preserve this route's cancellation even if a later voice refresh reports no support.
const cancelRoute = cancelRef.current;
return () => cancelRoute();
}, [routeKey]);
useEffect(() => {
if (!shouldPlay || activeRunId == null || latestMessage == null) {
return;
}
const text = getSpeechText(latestMessage);
if (!text) {
return;
}
// A voice list that has not loaded yet must leave the run eligible for a later retry.
if (speak(text)) {
onStarted(activeRunId);
}
}, [shouldPlay, activeRunId, latestMessage, onStarted, speak]);
return null;
}

View file

@ -0,0 +1,128 @@
/** @jest-environment jsdom */
import { render } from '@testing-library/react';
import { Constants, ContentTypes } from 'librechat-data-provider';
import type { TMessage } from 'librechat-data-provider';
import type { BrowserAudioProps } from '../Audio';
import BrowserAudio from '../Audio';
const message = {
messageId: 'reply-1',
conversationId: 'chat-1',
parentMessageId: Constants.NO_PARENT,
isCreatedByUser: false,
text: 'A spoken reply.',
} as TMessage;
function props(overrides: Partial<BrowserAudioProps> = {}): BrowserAudioProps {
return {
routeKey: 'chat-1',
shouldPlay: true,
activeRunId: 'run-1',
latestMessage: message,
speak: jest.fn(() => true),
cancel: jest.fn(),
onStarted: jest.fn(),
...overrides,
};
}
describe('BrowserAudio', () => {
it('renders no DOM and acknowledges a run only after speech is accepted', () => {
const input = props();
const { container } = render(<BrowserAudio {...input} />);
expect(container).toBeEmptyDOMElement();
expect(input.speak).toHaveBeenCalledWith(message.text);
expect(input.onStarted).toHaveBeenCalledWith('run-1');
expect(input.cancel).not.toHaveBeenCalled();
});
it.each<Partial<BrowserAudioProps>>([
{ shouldPlay: false },
{ activeRunId: null },
{ latestMessage: null },
{ latestMessage: { ...message, text: '' } },
{
latestMessage: {
...message,
text: 'Private reasoning.',
content: [{ type: ContentTypes.THINK, think: 'Private reasoning.' }],
},
},
])('does not request speech for an ineligible input: %j', (override) => {
const input = props(override);
render(<BrowserAudio {...input} />);
expect(input.speak).not.toHaveBeenCalled();
expect(input.onStarted).not.toHaveBeenCalled();
});
it('filters reasoning before speaking the answer', () => {
const input = props({
latestMessage: {
...message,
text: '',
content: [
{ type: ContentTypes.THINK, think: 'Private reasoning.' },
{ type: ContentTypes.TEXT, text: 'Public answer.' },
],
},
});
render(<BrowserAudio {...input} />);
expect(input.speak).toHaveBeenCalledWith('Public answer.');
});
it('retries refused speech when the voice service becomes ready', () => {
const input = props({ speak: jest.fn(() => false) });
const { rerender } = render(<BrowserAudio {...input} />);
expect(input.onStarted).not.toHaveBeenCalled();
const readySpeak = jest.fn(() => true);
rerender(<BrowserAudio {...input} speak={readySpeak} />);
expect(readySpeak).toHaveBeenCalledWith(message.text);
expect(input.onStarted).toHaveBeenCalledTimes(1);
expect(input.cancel).not.toHaveBeenCalled();
});
it('cancels the previous route before same-commit finalization starts speech', () => {
const events: string[] = [];
const input = props({
routeKey: 'new',
shouldPlay: false,
cancel: () => events.push('cancel'),
speak: () => {
events.push('speak');
return true;
},
onStarted: () => events.push('started'),
});
const { rerender, unmount } = render(<BrowserAudio {...input} />);
rerender(<BrowserAudio {...input} routeKey="saved-chat" shouldPlay />);
expect(events).toEqual(['cancel', 'speak', 'started']);
unmount();
expect(events).toEqual(['cancel', 'speak', 'started', 'cancel']);
});
it('preserves the route cancellation callback without cancelling or replaying on updates', () => {
const input = props();
const { rerender, unmount } = render(<BrowserAudio {...input} />);
const nextCancel = jest.fn();
rerender(<BrowserAudio {...input} cancel={nextCancel} />);
expect(input.speak).toHaveBeenCalledTimes(1);
expect(input.cancel).not.toHaveBeenCalled();
expect(nextCancel).not.toHaveBeenCalled();
rerender(<BrowserAudio {...input} routeKey="next-chat" cancel={nextCancel} />);
expect(input.cancel).toHaveBeenCalledTimes(1);
expect(nextCancel).not.toHaveBeenCalled();
unmount();
expect(nextCancel).toHaveBeenCalledTimes(1);
expect(input.cancel).toHaveBeenCalledTimes(1);
});
});