From 28f26cd7fd90103b1f95b76872b85f2ba9b20c7b Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Sun, 15 Nov 2026 13:35:30 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=8A=20refactor:=20Move=20Browser=20Aud?= =?UTF-8?q?io=20Playback=20Into=20the=20Chat=20Package=20(#18182)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - refactor: Move the Browser Audio Driver Into the Chat Package - fix: Preserve Browser Audio Route Cancellation - test: Cover Browser Audio Playback in the App - test: Use Compatible Browser Speech Utterances - test: Scope Browser Audio Replies to Messages - refactor: Defer Primary Browser Audio Until Route Adoption --- .../components/Chat/Input/BrowserAudio.tsx | 60 ++-- .../__tests__/BrowserAudio.matrix.spec.tsx | 243 ++++++++++++++++ .../Input/__tests__/BrowserAudio.spec.tsx | 119 +++++++- .../mock/scenarios/browser-audio.spec.ts | 265 ++++++++++++++++++ packages/chat/src/components.ts | 2 + .../chat/src/components/composer/Audio.tsx | 56 ++++ .../composer/__tests__/Audio.spec.tsx | 128 +++++++++ 7 files changed, 835 insertions(+), 38 deletions(-) create mode 100644 client/src/components/Chat/Input/__tests__/BrowserAudio.matrix.spec.tsx create mode 100644 e2e/specs/mock/scenarios/browser-audio.spec.ts create mode 100644 packages/chat/src/components/composer/Audio.tsx create mode 100644 packages/chat/src/components/composer/__tests__/Audio.spec.tsx diff --git a/client/src/components/Chat/Input/BrowserAudio.tsx b/client/src/components/Chat/Input/BrowserAudio.tsx index 1da7af5073..1ab79f28ec 100644 --- a/client/src/components/Chat/Input/BrowserAudio.tsx +++ b/client/src/components/Chat/Input/BrowserAudio.tsx @@ -1,7 +1,8 @@ -import { useEffect } from 'react'; +import { useCallback } from 'react'; import { useSetRecoilState } from 'recoil'; import { useParams } from 'react-router-dom'; -import { getSpeechText } from 'librechat-data-provider'; +import { Constants } from 'librechat-data-provider'; +import { BrowserAudio as BrowserAudioDriver } from '@librechat/chat/components'; import useTextToSpeechBrowser from '~/hooks/Input/useTextToSpeechBrowser'; import useAutoplayTrigger from '~/hooks/Audio/useAutoplayTrigger'; import { logger } from '~/utils'; @@ -20,35 +21,30 @@ export default function BrowserAudio({ index = 0 }) { const { shouldPlay, activeRunId, latestMessage } = useAutoplayTrigger(index); const { generateSpeechLocal, cancelSpeechLocal } = useTextToSpeechBrowser({ setIsSpeaking }); - /** - * Leaving a conversation (or unmounting) stops whatever is still being spoken. React runs - * every cleanup before any effect body, so the utterance the effect below is about to start - * survives the `/c/new` -> `/c/:id` navigation that lands in the same commit as finalization. - */ - useEffect(() => { - return () => cancelSpeechLocal(); - // We only want the effect to run when the paramId changes - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [paramId]); + const messageId = latestMessage?.messageId; + // The primary route must adopt a persisted reply before its cleanup can cancel playback. + const awaitingRoute = + index === 0 && + paramId === Constants.NEW_CONVO && + !!latestMessage?.conversationId && + latestMessage.conversationId !== paramId; + const speak = useCallback( + (text: string) => { + logger.log('BrowserAudio.tsx - speaking message:', messageId); + return generateSpeechLocal(text); + }, + [generateSpeechLocal, messageId], + ); - useEffect(() => { - if (!shouldPlay || activeRunId == null || latestMessage == null) { - return; - } - - const text = getSpeechText(latestMessage); - - if (!text) { - return; - } - - logger.log('BrowserAudio.tsx - speaking message:', latestMessage.messageId); - /** Only claim the run once an utterance was queued, so a not-yet-loaded voice list - * retries on the next render instead of silently swallowing the playback. */ - if (generateSpeechLocal(text)) { - setAudioRunId(activeRunId); - } - }, [shouldPlay, activeRunId, latestMessage, setAudioRunId, generateSpeechLocal]); - - return null; + return ( + + ); } diff --git a/client/src/components/Chat/Input/__tests__/BrowserAudio.matrix.spec.tsx b/client/src/components/Chat/Input/__tests__/BrowserAudio.matrix.spec.tsx new file mode 100644 index 0000000000..cadd2a6687 --- /dev/null +++ b/client/src/components/Chat/Input/__tests__/BrowserAudio.matrix.spec.tsx @@ -0,0 +1,243 @@ +import React, { useLayoutEffect } from 'react'; +import { flushSync } from 'react-dom'; +import { getDefaultStore } from 'jotai'; +import { RecoilRoot, useRecoilValue } from 'recoil'; +import { Constants, QueryKeys } from 'librechat-data-provider'; +import { act, render, screen, fireEvent } from '@testing-library/react'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { Link, RouterProvider, useParams, createMemoryRouter } from 'react-router-dom'; +import { isSubmittingFamily, resetSubmitting, useIsSubmitting } from '@librechat/chat/react'; +import type { TMessage, TConversation } from 'librechat-data-provider'; +import { activeRunFamily } from '~/hooks/SSE/atoms'; +import BrowserAudio from '../BrowserAudio'; +import store from '~/store'; + +const savedId = 'matrix-chat'; +const runId = 'matrix-run'; +const voiceName = 'Matrix Voice'; +const adoptLabel = 'Adopt saved route'; +const exitLabel = 'Different conversation'; +const message = { + messageId: 'matrix-reply', + conversationId: savedId, + parentMessageId: Constants.NO_PARENT, + isCreatedByUser: false, + text: 'A finalized matrix answer.', +} as TMessage; + +class Utterance extends EventTarget { + voice: SpeechSynthesisVoice | null = null; + onend: ((event: Event) => void) | null = null; + onerror: ((event: Event) => void) | null = null; + constructor(public text: string) { + super(); + this.addEventListener('end', (event) => this.onend?.(event)); + this.addEventListener('error', (event) => this.onerror?.(event)); + } +} + +let active: Utterance | null = null; +let committedRoute = ''; +let acknowledged: string | null = null; +let voicesAvailable = true; +let voicesChanged = () => {}; +const events: string[] = []; + +beforeAll(() => { + Object.defineProperty(window, 'speechSynthesis', { + configurable: true, + value: { + getVoices: () => (voicesAvailable ? [{ name: voiceName, localService: true }] : []), + addEventListener: (_event: string, listener: () => void) => { + voicesChanged = listener; + }, + speak: (utterance: Utterance) => { + active = utterance; + events.push(`speak:${committedRoute}`); + }, + cancel: () => { + const previous = active; + active = null; + events.push(`cancel:${committedRoute}`); + if (previous) { + const event = new Event('error'); + Object.defineProperty(event, 'error', { value: 'canceled' }); + queueMicrotask(() => previous.dispatchEvent(event)); + } + }, + }, + }); + Object.defineProperty(window, 'SpeechSynthesisUtterance', { + configurable: true, + value: Utterance, + }); +}); + +beforeEach(() => { + active = null; + committedRoute = ''; + acknowledged = null; + events.length = 0; + voicesAvailable = true; + act(() => voicesChanged()); +}); + +afterEach(() => resetSubmitting(getDefaultStore())); + +function Probe({ index }: { index: number }) { + const { conversationId = '' } = useParams(); + const audioRun = useRecoilValue(store.audioRunFamily(index)); + const submitting = useIsSubmitting(index); + useLayoutEffect(() => { + committedRoute = conversationId; + acknowledged = audioRun; + events.push(`commit:${conversationId}:submitting=${submitting}:ack=${audioRun ?? 'none'}`); + }, [conversationId, audioRun, submitting]); + return {conversationId}; +} + +function setup(index = 0) { + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + queryClient.setQueryData([QueryKeys.messages, 'new'], [message]); + queryClient.setQueryData([QueryKeys.messages, savedId], [message]); + const router = createMemoryRouter( + [ + { + path: '/c/:conversationId', + element: ( + <> + {adoptLabel} + {exitLabel} + + + + ), + }, + ], + { initialEntries: ['/c/new'] }, + ); + const view = render( + + { + set(store.voice, voiceName); + set(store.conversationByIndex(index), { conversationId: savedId } as TConversation); + set(store.audioRunFamily(index), null); + getDefaultStore().set(activeRunFamily(index), runId); + getDefaultStore().set(isSubmittingFamily(index), true); + }} + > + + + , + ); + return { ...view, router }; +} + +const finalize = (index = 0) => getDefaultStore().set(isSubmittingFamily(index), false); +const adopt = () => fireEvent.click(screen.getByRole('link', { name: adoptLabel })); +const spoken = () => events.filter((event) => event.startsWith('speak:')); + +it.each([0, 1])( + 'retains speech with a combined eligibility/route commit in pane %i', + async (index) => { + const { router } = setup(index); + await act(async () => { + flushSync(() => { + finalize(index); + void router.navigate(`/c/${savedId}`, { flushSync: true }); + }); + }); + expect(committedRoute).toBe(savedId); + expect(active?.text).toBe(message.text); + expect(acknowledged).toBe(runId); + expect(spoken()).toEqual([`speak:${savedId}`]); + expect(events.some((event) => event.startsWith('commit:new:submitting=false'))).toBe(false); + }, +); + +it('defers primary-pane speech and acknowledgement until saved-route adoption', async () => { + setup(); + await act(async () => finalize()); + expect(committedRoute).toBe('new'); + expect(active).toBeNull(); + expect(acknowledged).toBeNull(); + expect(spoken()).toEqual([]); + + await act(async () => adopt()); + expect(committedRoute).toBe(savedId); + expect(active?.text).toBe(message.text); + expect(acknowledged).toBe(runId); + expect(spoken()).toEqual([`speak:${savedId}`]); + expect(events.filter((event) => !event.startsWith('commit:'))).toEqual([ + `cancel:${savedId}`, + `cancel:${savedId}`, + `speak:${savedId}`, + ]); + expect(events.at(-1)).toBe(`commit:${savedId}:submitting=false:ack=${runId}`); +}); + +it('allows an additional pane to speak its own conversation before primary-route adoption', async () => { + setup(1); + await act(async () => finalize(1)); + expect(committedRoute).toBe('new'); + expect(active?.text).toBe(message.text); + expect(acknowledged).toBe(runId); + expect(spoken()).toEqual(['speak:new']); +}); + +it.each(['before-adoption', 'after-adoption'])( + 'retries delayed voices %s without premature acknowledgement or replay', + async (arrival) => { + const warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + voicesAvailable = false; + act(() => voicesChanged()); + setup(); + await act(async () => finalize()); + const loadVoices = () => { + voicesAvailable = true; + voicesChanged(); + }; + if (arrival === 'before-adoption') { + await act(async () => loadVoices()); + expect(active).toBeNull(); + expect(acknowledged).toBeNull(); + await act(async () => adopt()); + } else { + await act(async () => adopt()); + expect(active).toBeNull(); + expect(acknowledged).toBeNull(); + await act(async () => loadVoices()); + } + expect(active?.text).toBe(message.text); + expect(acknowledged).toBe(runId); + expect(spoken()).toEqual([`speak:${savedId}`]); + await act(async () => voicesChanged()); + expect(active?.text).toBe(message.text); + expect(spoken()).toHaveLength(1); + warn.mockRestore(); + }, +); + +it.each([0, 1])('cancels accepted speech on a different conversation in pane %i', async (index) => { + setup(index); + await act(async () => adopt()); + await act(async () => finalize(index)); + expect(active?.text).toBe(message.text); + await act(async () => fireEvent.click(screen.getByRole('link', { name: exitLabel }))); + expect(committedRoute).toBe('other-chat'); + expect(active).toBeNull(); + expect(acknowledged).toBe(runId); + expect(spoken()).toHaveLength(1); +}); + +it.each([0, 1])('cancels accepted speech on unmount in pane %i', async (index) => { + const view = setup(index); + await act(async () => adopt()); + await act(async () => finalize(index)); + expect(active?.text).toBe(message.text); + await act(async () => view.unmount()); + expect(active).toBeNull(); + expect(acknowledged).toBe(runId); + expect(spoken()).toHaveLength(1); +}); diff --git a/client/src/components/Chat/Input/__tests__/BrowserAudio.spec.tsx b/client/src/components/Chat/Input/__tests__/BrowserAudio.spec.tsx index 1185e5eced..aab1f4420b 100644 --- a/client/src/components/Chat/Input/__tests__/BrowserAudio.spec.tsx +++ b/client/src/components/Chat/Input/__tests__/BrowserAudio.spec.tsx @@ -1,11 +1,11 @@ import React from 'react'; -import { RecoilRoot } from 'recoil'; import { getDefaultStore } from 'jotai'; -import { render, waitFor } from '@testing-library/react'; -import { MemoryRouter, Routes, Route } from 'react-router-dom'; +import { RecoilRoot, useRecoilValue } from 'recoil'; +import { Link, MemoryRouter, Routes, Route } from 'react-router-dom'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { isSubmittingFamily, resetSubmitting } from '@librechat/chat/react'; import { Constants, QueryKeys, ContentTypes } from 'librechat-data-provider'; +import { act, render, screen, waitFor, fireEvent } from '@testing-library/react'; import type { TMessage, TConversation } from 'librechat-data-provider'; import { activeRunFamily } from '~/hooks/SSE/atoms'; import BrowserAudio from '../BrowserAudio'; @@ -14,8 +14,10 @@ import store from '~/store'; const conversationId = 'convo-1'; const voiceName = 'Test Voice'; const responseText = 'The capital of Germany is Berlin.'; +const nextChatLabel = 'Next chat'; const spoken: string[] = []; +const speechEvents: string[] = []; let cancelCount = 0; class FakeSpeechSynthesisUtterance { @@ -26,14 +28,20 @@ class FakeSpeechSynthesisUtterance { } const fakeVoice = { name: voiceName, localService: true } as SpeechSynthesisVoice; +let availableVoices = [fakeVoice]; +let voicesChanged = () => {}; const speechSynthesis = { - getVoices: () => [fakeVoice], - addEventListener: () => undefined, + getVoices: () => availableVoices, + addEventListener: (_event: string, listener: () => void) => { + voicesChanged = listener; + }, speak: (utterance: FakeSpeechSynthesisUtterance) => { + speechEvents.push('speak'); spoken.push(utterance.text); }, cancel: () => { + speechEvents.push('cancel'); cancelCount += 1; }, }; @@ -46,16 +54,25 @@ const assistantMessage = { text: responseText, } as TMessage; +function AudioRunProbe() { + const audioRunId = useRecoilValue(store.audioRunFamily(0)); + return {audioRunId ?? 'none'}; +} + const renderBrowserAudio = ({ activeRunId = 'run-1', audioRunId = null, isSubmitting = false, messages = [assistantMessage], + initialPath = `/c/${conversationId}`, + nextPath = '/c/other-chat', }: { activeRunId?: string | null; audioRunId?: string | null; isSubmitting?: boolean; messages?: TMessage[]; + initialPath?: string; + nextPath?: string; } = {}) => { const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } }, @@ -64,7 +81,7 @@ const renderBrowserAudio = ({ return render( - + { set(store.voice, voiceName); @@ -74,6 +91,8 @@ const renderBrowserAudio = ({ getDefaultStore().set(isSubmittingFamily(0), isSubmitting); }} > + + {nextChatLabel} } /> @@ -106,13 +125,37 @@ describe('BrowserAudio autoplay', () => { beforeEach(() => { spoken.length = 0; + speechEvents.length = 0; cancelCount = 0; + availableVoices = [fakeVoice]; + act(() => voicesChanged()); }); it('speaks the finalized assistant message through speech synthesis', async () => { renderBrowserAudio(); await waitFor(() => expect(spoken).toEqual([responseText])); + expect(screen.getByTestId('played-run')).toHaveTextContent('run-1'); + }); + + it('keeps an unqueued run eligible until the voices arrive', async () => { + const warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + availableVoices = []; + act(() => voicesChanged()); + renderBrowserAudio(); + await settle(); + + expect(spoken).toEqual([]); + expect(screen.getByTestId('played-run')).toHaveTextContent('none'); + + act(() => { + availableVoices = [fakeVoice]; + voicesChanged(); + }); + + await waitFor(() => expect(spoken).toEqual([responseText])); + expect(screen.getByTestId('played-run')).toHaveTextContent('run-1'); + warn.mockRestore(); }); it('speaks the answer without the reasoning that preceded it', async () => { @@ -186,4 +229,68 @@ describe('BrowserAudio autoplay', () => { expect(cancelCount).toBeGreaterThan(0); }); + + it('cancels on route navigation while the adapter remains mounted', async () => { + renderBrowserAudio(); + await waitFor(() => expect(spoken).toEqual([responseText])); + const cancelsBeforeNavigation = cancelCount; + + fireEvent.click(screen.getByRole('link', { name: 'Next chat' })); + + expect(cancelCount).toBe(cancelsBeforeNavigation + 1); + expect(spoken).toEqual([responseText]); + }); + + it('starts the finalized reply after the new-chat route cleans up', async () => { + renderBrowserAudio({ + initialPath: '/c/new', + nextPath: `/c/${conversationId}`, + isSubmitting: true, + activeRunId: null, + }); + await settle(); + expect(spoken).toEqual([]); + + act(() => { + getDefaultStore().set(activeRunFamily(0), 'run-1'); + getDefaultStore().set(isSubmittingFamily(0), false); + fireEvent.click(screen.getByRole('link', { name: nextChatLabel })); + }); + + await waitFor(() => expect(spoken).toEqual([responseText])); + expect(screen.getByTestId('played-run')).toHaveTextContent('run-1'); + // Route cleanup and the speech service's pre-utterance cancellation both precede speaking. + expect(cancelCount).toBe(2); + expect(speechEvents).toEqual(['cancel', 'cancel', 'speak']); + }); + + it('does not cancel or replay an accepted utterance when voices update', async () => { + renderBrowserAudio(); + await waitFor(() => expect(spoken).toEqual([responseText])); + const cancelsAfterSpeaking = cancelCount; + + act(() => voicesChanged()); + + expect(cancelCount).toBe(cancelsAfterSpeaking); + expect(spoken).toEqual([responseText]); + expect(screen.getByTestId('played-run')).toHaveTextContent('run-1'); + }); + + it('still cancels accepted speech if a later voice refresh fails', async () => { + const { unmount } = renderBrowserAudio(); + await waitFor(() => expect(spoken).toEqual([responseText])); + const cancelsAfterSpeaking = cancelCount; + const error = jest.spyOn(console, 'error').mockImplementation(() => {}); + const getVoices = jest.spyOn(speechSynthesis, 'getVoices').mockImplementationOnce(() => { + throw new Error('Voice service unavailable'); + }); + + act(() => voicesChanged()); + expect(cancelCount).toBe(cancelsAfterSpeaking); + unmount(); + + expect(cancelCount).toBe(cancelsAfterSpeaking + 1); + getVoices.mockRestore(); + error.mockRestore(); + }); }); diff --git a/e2e/specs/mock/scenarios/browser-audio.spec.ts b/e2e/specs/mock/scenarios/browser-audio.spec.ts new file mode 100644 index 0000000000..2698935262 --- /dev/null +++ b/e2e/specs/mock/scenarios/browser-audio.spec.ts @@ -0,0 +1,265 @@ +import { randomUUID } from 'node:crypto'; +import { expect, test } from '@playwright/test'; +import type { Page } from '@playwright/test'; +import { + MOCK_ENDPOINTS, + NEW_CHAT_PATH, + replyPrompt, + replyText, + thinkPrompt, + messagesView, + selectMockEndpoint, + sendMessageAndWaitForCompletion, +} from '../helpers'; +import { deleteConversations, deleteMessagesByConversation } from '../db'; + +type SpeechEvent = { type: 'cancel' | 'speak' | 'end' | 'voices' | 'unavailable'; path: string }; +type SpeechBoundary = { + events: SpeechEvent[]; + spoken: string[]; + activeText: string | null; + setVoices: (ready: boolean, fail?: boolean) => void; + finish: () => void; +}; + +declare global { + interface Window { + browserAudio: SpeechBoundary; + } +} + +const createdConversationIds: string[] = []; + +test.describe.configure({ timeout: 120_000 }); + +test.afterAll(async () => { + await deleteMessagesByConversation(createdConversationIds); + await deleteConversations(createdConversationIds); +}); + +/** Replace the Web Speech boundary; native utterances reject service-fake voice objects. */ +async function openChat(page: Page, voicesReady = true) { + await page.addInitScript((ready) => { + for (const [key, value] of Object.entries({ + engineSTT: 'browser', + engineTTS: 'browser', + textToSpeech: true, + automaticPlayback: true, + voice: 'E2E local voice', + cloudBrowserVoices: false, + })) { + localStorage.setItem(key, JSON.stringify(value)); + } + + const target = new EventTarget(); + const voice: SpeechSynthesisVoice = { + name: 'E2E local voice', + voiceURI: 'e2e-local', + lang: 'en-US', + localService: true, + default: true, + }; + let available = ready; + let fail = false; + class Utterance extends EventTarget { + voice: SpeechSynthesisVoice | null = null; + onend: ((event: Event) => void) | null = null; + onerror: ((event: Event) => void) | null = null; + + constructor(public text: string) { + super(); + this.addEventListener('end', (event) => this.onend?.(event)); + this.addEventListener('error', (event) => this.onerror?.(event)); + } + } + Object.defineProperty(window, 'SpeechSynthesisUtterance', { + configurable: true, + value: Utterance, + }); + let current: Utterance | null = null; + const record = (type: SpeechEvent['type']) => { + window.browserAudio.events.push({ type, path: location.pathname }); + }; + window.browserAudio = { + events: [], + spoken: [], + activeText: null, + setVoices: (nextReady, nextFail = false) => { + available = nextReady; + fail = nextFail; + record('voices'); + target.dispatchEvent(new Event('voiceschanged')); + }, + finish: () => { + const utterance = current; + current = null; + window.browserAudio.activeText = null; + record('end'); + utterance?.dispatchEvent(new Event('end')); + }, + }; + Object.defineProperty(window, 'speechSynthesis', { + configurable: true, + value: { + getVoices: () => { + if (fail) { + record('unavailable'); + throw new Error('E2E voice service unavailable'); + } + return available ? [voice] : []; + }, + addEventListener: target.addEventListener.bind(target), + removeEventListener: target.removeEventListener.bind(target), + speak: (utterance: Utterance) => { + current = utterance; + window.browserAudio.activeText = utterance.text; + window.browserAudio.spoken.push(utterance.text); + record('speak'); + }, + cancel: () => { + const utterance = current; + current = null; + window.browserAudio.activeText = null; + record('cancel'); + if (utterance) { + const event = new Event('error'); + Object.defineProperty(event, 'error', { value: 'canceled' }); + queueMicrotask(() => utterance.dispatchEvent(event)); + } + }, + }, + }); + }, voicesReady); + await page.goto(NEW_CHAT_PATH); + await selectMockEndpoint(page, MOCK_ENDPOINTS[0]); +} + +async function reply(page: Page, reasoning = false) { + const label = `browser-audio-${randomUUID().slice(0, 8)}`; + const response = await sendMessageAndWaitForCompletion( + page, + reasoning ? thinkPrompt(label) : replyPrompt(label), + ); + const { conversationId } = (await response.json()) as { conversationId: string }; + createdConversationIds.push(conversationId); + await expect(messagesView(page).getByText(replyText(label), { exact: true })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Stop generating' })).toHaveCount(0); + return replyText(label); +} + +/** Let the voiceschanged subscription and its passive playback effect reach a browser paint. */ +async function refreshVoices(page: Page, fail = false) { + await page.evaluate(async (unavailable) => { + window.browserAudio.setVoices(true, unavailable); + await new Promise((resolve) => + requestAnimationFrame(() => requestAnimationFrame(() => resolve())), + ); + }, fail); +} + +const speech = (page: Page) => + page.evaluate(() => ({ + events: window.browserAudio.events, + spoken: window.browserAudio.spoken, + activeText: window.browserAudio.activeText, + })); + +async function newChat(page: Page) { + await page + .getByRole('link', { name: 'New chat', exact: true }) + .or(page.getByRole('button', { name: 'New chat', exact: true })) + .filter({ visible: true }) + .first() + .click(); + await expect(page).toHaveURL(/\/c\/new$/); +} + +test.describe('browser speech autoplay', () => { + test('speaks only the answer after new-chat cleanup and acknowledges the run without replay @scenario:browser-audio-finalized-answer', async ({ + page, + }) => { + await openChat(page); + const answer = await reply(page, true); + await expect.poll(async () => (await speech(page)).spoken).toEqual([answer]); + const queued = await speech(page); + expect(queued.activeText).toBe(answer); + expect(queued.events.map((event) => event.type)).toEqual(['cancel', 'cancel', 'speak']); + expect(queued.events.every((event) => event.path === new URL(page.url()).pathname)).toBe(true); + + await page.evaluate(() => window.browserAudio.finish()); + await refreshVoices(page); + expect((await speech(page)).spoken).toEqual([answer]); + expect((await speech(page)).activeText).toBeNull(); + + const nextAnswer = await reply(page); + await expect.poll(async () => (await speech(page)).spoken).toEqual([answer, nextAnswer]); + }); + + test('leaves a refused run eligible until voices become available @scenario:browser-audio-late-voices', async ({ + page, + }) => { + await openChat(page, false); + const answer = await reply(page); + expect((await speech(page)).spoken).toEqual([]); + + await refreshVoices(page); + await expect.poll(async () => (await speech(page)).spoken).toEqual([answer]); + await refreshVoices(page); + expect((await speech(page)).spoken).toEqual([answer]); + expect((await speech(page)).activeText).toBe(answer); + }); + + test('cancels active speech on route navigation and when the chat unmounts @scenario:browser-audio-route-cancellation', async ({ + page, + }) => { + await openChat(page); + const answer = await reply(page); + await expect.poll(async () => (await speech(page)).activeText).toBe(answer); + await newChat(page); + await expect.poll(async () => (await speech(page)).activeText).toBeNull(); + expect((await speech(page)).events.at(-1)).toEqual({ type: 'cancel', path: NEW_CHAT_PATH }); + + await selectMockEndpoint(page, MOCK_ENDPOINTS[0]); + const nextAnswer = await reply(page); + await expect.poll(async () => (await speech(page)).activeText).toBe(nextAnswer); + await page.evaluate(() => { + history.pushState(history.state, '', '/search'); + window.dispatchEvent(new PopStateEvent('popstate')); + }); + await expect(page.getByRole('textbox', { name: 'Message input' })).toHaveCount(0); + await expect.poll(async () => (await speech(page)).activeText).toBeNull(); + expect((await speech(page)).events.at(-1)).toEqual({ type: 'cancel', path: '/search' }); + }); + + test('retains cancellation after a later voice refresh fails @scenario:browser-audio-failed-voice-refresh', async ({ + page, + }) => { + await openChat(page); + const answer = await reply(page); + await expect.poll(async () => (await speech(page)).activeText).toBe(answer); + await refreshVoices(page, true); + expect((await speech(page)).events.at(-1)?.type).toBe('unavailable'); + expect((await speech(page)).activeText).toBe(answer); + + await newChat(page); + await expect.poll(async () => (await speech(page)).activeText).toBeNull(); + expect((await speech(page)).events.at(-1)).toEqual({ type: 'cancel', path: NEW_CHAT_PATH }); + expect((await speech(page)).spoken).toEqual([answer]); + }); + + test('does not speak an empty completed answer @scenario:browser-audio-ineligible-message', async ({ + page, + }) => { + await openChat(page); + const response = await sendMessageAndWaitForCompletion( + page, + `E2E_EMPTY_REPLY:browser-audio-${randomUUID().slice(0, 8)}`, + ); + const { conversationId } = (await response.json()) as { conversationId: string }; + createdConversationIds.push(conversationId); + await expect(page.getByRole('button', { name: 'Stop generating' })).toHaveCount(0); + await refreshVoices(page); + expect((await speech(page)).spoken).toEqual([]); + expect((await speech(page)).activeText).toBeNull(); + }); +}); diff --git a/packages/chat/src/components.ts b/packages/chat/src/components.ts index 34a823ca4a..8d3ae85058 100644 --- a/packages/chat/src/components.ts +++ b/packages/chat/src/components.ts @@ -17,6 +17,8 @@ export type { } from './components/Part'; /** Composer: the chat input, its controls and the steering queue. */ +export { default as BrowserAudio } from './components/composer/Audio'; +export type { BrowserAudioProps } from './components/composer/Audio'; export { default as StopButton } from './components/composer/StopButton'; export { default as MentionItem } from './components/composer/MentionItem'; export { default as MentionPopover } from './components/composer/popover/Mention'; diff --git a/packages/chat/src/components/composer/Audio.tsx b/packages/chat/src/components/composer/Audio.tsx new file mode 100644 index 0000000000..8e0cbacfa0 --- /dev/null +++ b/packages/chat/src/components/composer/Audio.tsx @@ -0,0 +1,56 @@ +import { useEffect, useRef } from 'react'; +import { getSpeechText } from 'librechat-data-provider'; +import type { TMessage } from 'librechat-data-provider'; + +export interface BrowserAudioProps { + /** The route lifetime that owns playback, including the new-chat route. */ + routeKey?: string; + shouldPlay: boolean; + activeRunId: string | null; + latestMessage: TMessage | null; + /** Returns true only when an utterance was queued; may change when voices load. */ + speak: (text: string) => boolean; + cancel: () => void; + onStarted: (runId: string) => void; +} + +/** Browser speech playback, with route, voice configuration and run state supplied by the host. */ +export default function BrowserAudio({ + routeKey, + shouldPlay, + activeRunId, + latestMessage, + speak, + cancel, + onStarted, +}: BrowserAudioProps): null { + const cancelRef = useRef(cancel); + useEffect(() => { + cancelRef.current = cancel; + }, [cancel]); + + /** Cleanup belongs only to the route lifetime, not voice availability or callback identity. + * React runs cleanups before effect bodies, so same-commit new-chat finalization cancels the + * previous utterance before starting the newly finalized reply. */ + useEffect(() => { + // Preserve this route's cancellation even if a later voice refresh reports no support. + const cancelRoute = cancelRef.current; + return () => cancelRoute(); + }, [routeKey]); + + useEffect(() => { + if (!shouldPlay || activeRunId == null || latestMessage == null) { + return; + } + const text = getSpeechText(latestMessage); + if (!text) { + return; + } + // A voice list that has not loaded yet must leave the run eligible for a later retry. + if (speak(text)) { + onStarted(activeRunId); + } + }, [shouldPlay, activeRunId, latestMessage, onStarted, speak]); + + return null; +} diff --git a/packages/chat/src/components/composer/__tests__/Audio.spec.tsx b/packages/chat/src/components/composer/__tests__/Audio.spec.tsx new file mode 100644 index 0000000000..90081d3695 --- /dev/null +++ b/packages/chat/src/components/composer/__tests__/Audio.spec.tsx @@ -0,0 +1,128 @@ +/** @jest-environment jsdom */ +import { render } from '@testing-library/react'; +import { Constants, ContentTypes } from 'librechat-data-provider'; +import type { TMessage } from 'librechat-data-provider'; +import type { BrowserAudioProps } from '../Audio'; +import BrowserAudio from '../Audio'; + +const message = { + messageId: 'reply-1', + conversationId: 'chat-1', + parentMessageId: Constants.NO_PARENT, + isCreatedByUser: false, + text: 'A spoken reply.', +} as TMessage; + +function props(overrides: Partial = {}): BrowserAudioProps { + return { + routeKey: 'chat-1', + shouldPlay: true, + activeRunId: 'run-1', + latestMessage: message, + speak: jest.fn(() => true), + cancel: jest.fn(), + onStarted: jest.fn(), + ...overrides, + }; +} + +describe('BrowserAudio', () => { + it('renders no DOM and acknowledges a run only after speech is accepted', () => { + const input = props(); + const { container } = render(); + + expect(container).toBeEmptyDOMElement(); + expect(input.speak).toHaveBeenCalledWith(message.text); + expect(input.onStarted).toHaveBeenCalledWith('run-1'); + expect(input.cancel).not.toHaveBeenCalled(); + }); + + it.each>([ + { shouldPlay: false }, + { activeRunId: null }, + { latestMessage: null }, + { latestMessage: { ...message, text: '' } }, + { + latestMessage: { + ...message, + text: 'Private reasoning.', + content: [{ type: ContentTypes.THINK, think: 'Private reasoning.' }], + }, + }, + ])('does not request speech for an ineligible input: %j', (override) => { + const input = props(override); + render(); + + expect(input.speak).not.toHaveBeenCalled(); + expect(input.onStarted).not.toHaveBeenCalled(); + }); + + it('filters reasoning before speaking the answer', () => { + const input = props({ + latestMessage: { + ...message, + text: '', + content: [ + { type: ContentTypes.THINK, think: 'Private reasoning.' }, + { type: ContentTypes.TEXT, text: 'Public answer.' }, + ], + }, + }); + render(); + + expect(input.speak).toHaveBeenCalledWith('Public answer.'); + }); + + it('retries refused speech when the voice service becomes ready', () => { + const input = props({ speak: jest.fn(() => false) }); + const { rerender } = render(); + expect(input.onStarted).not.toHaveBeenCalled(); + + const readySpeak = jest.fn(() => true); + rerender(); + + expect(readySpeak).toHaveBeenCalledWith(message.text); + expect(input.onStarted).toHaveBeenCalledTimes(1); + expect(input.cancel).not.toHaveBeenCalled(); + }); + + it('cancels the previous route before same-commit finalization starts speech', () => { + const events: string[] = []; + const input = props({ + routeKey: 'new', + shouldPlay: false, + cancel: () => events.push('cancel'), + speak: () => { + events.push('speak'); + return true; + }, + onStarted: () => events.push('started'), + }); + const { rerender, unmount } = render(); + + rerender(); + expect(events).toEqual(['cancel', 'speak', 'started']); + + unmount(); + expect(events).toEqual(['cancel', 'speak', 'started', 'cancel']); + }); + + it('preserves the route cancellation callback without cancelling or replaying on updates', () => { + const input = props(); + const { rerender, unmount } = render(); + const nextCancel = jest.fn(); + + rerender(); + expect(input.speak).toHaveBeenCalledTimes(1); + expect(input.cancel).not.toHaveBeenCalled(); + expect(nextCancel).not.toHaveBeenCalled(); + + rerender(); + expect(input.cancel).toHaveBeenCalledTimes(1); + expect(nextCancel).not.toHaveBeenCalled(); + + unmount(); + expect(nextCancel).toHaveBeenCalledTimes(1); + expect(input.cancel).toHaveBeenCalledTimes(1); + }); +});