mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-20 23:13:35 +00:00
🔊 refactor: Move Browser Audio Playback Into the Chat Package (#18182)
- refactor: Move the Browser Audio Driver Into the Chat Package - fix: Preserve Browser Audio Route Cancellation - test: Cover Browser Audio Playback in the App - test: Use Compatible Browser Speech Utterances - test: Scope Browser Audio Replies to Messages - refactor: Defer Primary Browser Audio Until Route Adoption
This commit is contained in:
parent
dd753672f7
commit
28f26cd7fd
7 changed files with 835 additions and 38 deletions
|
|
@ -1,7 +1,8 @@
|
|||
import { useEffect } from 'react';
|
||||
import { useCallback } from 'react';
|
||||
import { useSetRecoilState } from 'recoil';
|
||||
import { useParams } from 'react-router-dom';
|
||||
import { getSpeechText } from 'librechat-data-provider';
|
||||
import { Constants } from 'librechat-data-provider';
|
||||
import { BrowserAudio as BrowserAudioDriver } from '@librechat/chat/components';
|
||||
import useTextToSpeechBrowser from '~/hooks/Input/useTextToSpeechBrowser';
|
||||
import useAutoplayTrigger from '~/hooks/Audio/useAutoplayTrigger';
|
||||
import { logger } from '~/utils';
|
||||
|
|
@ -20,35 +21,30 @@ export default function BrowserAudio({ index = 0 }) {
|
|||
const { shouldPlay, activeRunId, latestMessage } = useAutoplayTrigger(index);
|
||||
const { generateSpeechLocal, cancelSpeechLocal } = useTextToSpeechBrowser({ setIsSpeaking });
|
||||
|
||||
/**
|
||||
* Leaving a conversation (or unmounting) stops whatever is still being spoken. React runs
|
||||
* every cleanup before any effect body, so the utterance the effect below is about to start
|
||||
* survives the `/c/new` -> `/c/:id` navigation that lands in the same commit as finalization.
|
||||
*/
|
||||
useEffect(() => {
|
||||
return () => cancelSpeechLocal();
|
||||
// We only want the effect to run when the paramId changes
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [paramId]);
|
||||
const messageId = latestMessage?.messageId;
|
||||
// The primary route must adopt a persisted reply before its cleanup can cancel playback.
|
||||
const awaitingRoute =
|
||||
index === 0 &&
|
||||
paramId === Constants.NEW_CONVO &&
|
||||
!!latestMessage?.conversationId &&
|
||||
latestMessage.conversationId !== paramId;
|
||||
const speak = useCallback(
|
||||
(text: string) => {
|
||||
logger.log('BrowserAudio.tsx - speaking message:', messageId);
|
||||
return generateSpeechLocal(text);
|
||||
},
|
||||
[generateSpeechLocal, messageId],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (!shouldPlay || activeRunId == null || latestMessage == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const text = getSpeechText(latestMessage);
|
||||
|
||||
if (!text) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger.log('BrowserAudio.tsx - speaking message:', latestMessage.messageId);
|
||||
/** Only claim the run once an utterance was queued, so a not-yet-loaded voice list
|
||||
* retries on the next render instead of silently swallowing the playback. */
|
||||
if (generateSpeechLocal(text)) {
|
||||
setAudioRunId(activeRunId);
|
||||
}
|
||||
}, [shouldPlay, activeRunId, latestMessage, setAudioRunId, generateSpeechLocal]);
|
||||
|
||||
return null;
|
||||
return (
|
||||
<BrowserAudioDriver
|
||||
routeKey={paramId}
|
||||
shouldPlay={shouldPlay && !awaitingRoute}
|
||||
activeRunId={activeRunId}
|
||||
latestMessage={latestMessage}
|
||||
speak={speak}
|
||||
cancel={cancelSpeechLocal}
|
||||
onStarted={setAudioRunId}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,243 @@
|
|||
import React, { useLayoutEffect } from 'react';
|
||||
import { flushSync } from 'react-dom';
|
||||
import { getDefaultStore } from 'jotai';
|
||||
import { RecoilRoot, useRecoilValue } from 'recoil';
|
||||
import { Constants, QueryKeys } from 'librechat-data-provider';
|
||||
import { act, render, screen, fireEvent } from '@testing-library/react';
|
||||
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
|
||||
import { Link, RouterProvider, useParams, createMemoryRouter } from 'react-router-dom';
|
||||
import { isSubmittingFamily, resetSubmitting, useIsSubmitting } from '@librechat/chat/react';
|
||||
import type { TMessage, TConversation } from 'librechat-data-provider';
|
||||
import { activeRunFamily } from '~/hooks/SSE/atoms';
|
||||
import BrowserAudio from '../BrowserAudio';
|
||||
import store from '~/store';
|
||||
|
||||
const savedId = 'matrix-chat';
|
||||
const runId = 'matrix-run';
|
||||
const voiceName = 'Matrix Voice';
|
||||
const adoptLabel = 'Adopt saved route';
|
||||
const exitLabel = 'Different conversation';
|
||||
const message = {
|
||||
messageId: 'matrix-reply',
|
||||
conversationId: savedId,
|
||||
parentMessageId: Constants.NO_PARENT,
|
||||
isCreatedByUser: false,
|
||||
text: 'A finalized matrix answer.',
|
||||
} as TMessage;
|
||||
|
||||
class Utterance extends EventTarget {
|
||||
voice: SpeechSynthesisVoice | null = null;
|
||||
onend: ((event: Event) => void) | null = null;
|
||||
onerror: ((event: Event) => void) | null = null;
|
||||
constructor(public text: string) {
|
||||
super();
|
||||
this.addEventListener('end', (event) => this.onend?.(event));
|
||||
this.addEventListener('error', (event) => this.onerror?.(event));
|
||||
}
|
||||
}
|
||||
|
||||
let active: Utterance | null = null;
|
||||
let committedRoute = '';
|
||||
let acknowledged: string | null = null;
|
||||
let voicesAvailable = true;
|
||||
let voicesChanged = () => {};
|
||||
const events: string[] = [];
|
||||
|
||||
beforeAll(() => {
|
||||
Object.defineProperty(window, 'speechSynthesis', {
|
||||
configurable: true,
|
||||
value: {
|
||||
getVoices: () => (voicesAvailable ? [{ name: voiceName, localService: true }] : []),
|
||||
addEventListener: (_event: string, listener: () => void) => {
|
||||
voicesChanged = listener;
|
||||
},
|
||||
speak: (utterance: Utterance) => {
|
||||
active = utterance;
|
||||
events.push(`speak:${committedRoute}`);
|
||||
},
|
||||
cancel: () => {
|
||||
const previous = active;
|
||||
active = null;
|
||||
events.push(`cancel:${committedRoute}`);
|
||||
if (previous) {
|
||||
const event = new Event('error');
|
||||
Object.defineProperty(event, 'error', { value: 'canceled' });
|
||||
queueMicrotask(() => previous.dispatchEvent(event));
|
||||
}
|
||||
},
|
||||
},
|
||||
});
|
||||
Object.defineProperty(window, 'SpeechSynthesisUtterance', {
|
||||
configurable: true,
|
||||
value: Utterance,
|
||||
});
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
active = null;
|
||||
committedRoute = '';
|
||||
acknowledged = null;
|
||||
events.length = 0;
|
||||
voicesAvailable = true;
|
||||
act(() => voicesChanged());
|
||||
});
|
||||
|
||||
afterEach(() => resetSubmitting(getDefaultStore()));
|
||||
|
||||
function Probe({ index }: { index: number }) {
|
||||
const { conversationId = '' } = useParams();
|
||||
const audioRun = useRecoilValue(store.audioRunFamily(index));
|
||||
const submitting = useIsSubmitting(index);
|
||||
useLayoutEffect(() => {
|
||||
committedRoute = conversationId;
|
||||
acknowledged = audioRun;
|
||||
events.push(`commit:${conversationId}:submitting=${submitting}:ack=${audioRun ?? 'none'}`);
|
||||
}, [conversationId, audioRun, submitting]);
|
||||
return <output data-testid="matrix-route">{conversationId}</output>;
|
||||
}
|
||||
|
||||
function setup(index = 0) {
|
||||
const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } });
|
||||
queryClient.setQueryData([QueryKeys.messages, 'new'], [message]);
|
||||
queryClient.setQueryData([QueryKeys.messages, savedId], [message]);
|
||||
const router = createMemoryRouter(
|
||||
[
|
||||
{
|
||||
path: '/c/:conversationId',
|
||||
element: (
|
||||
<>
|
||||
<Link to={`/c/${savedId}`}>{adoptLabel}</Link>
|
||||
<Link to="/c/other-chat">{exitLabel}</Link>
|
||||
<Probe index={index} />
|
||||
<BrowserAudio index={index} />
|
||||
</>
|
||||
),
|
||||
},
|
||||
],
|
||||
{ initialEntries: ['/c/new'] },
|
||||
);
|
||||
const view = render(
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<RecoilRoot
|
||||
initializeState={({ set }) => {
|
||||
set(store.voice, voiceName);
|
||||
set(store.conversationByIndex(index), { conversationId: savedId } as TConversation);
|
||||
set(store.audioRunFamily(index), null);
|
||||
getDefaultStore().set(activeRunFamily(index), runId);
|
||||
getDefaultStore().set(isSubmittingFamily(index), true);
|
||||
}}
|
||||
>
|
||||
<RouterProvider router={router} />
|
||||
</RecoilRoot>
|
||||
</QueryClientProvider>,
|
||||
);
|
||||
return { ...view, router };
|
||||
}
|
||||
|
||||
const finalize = (index = 0) => getDefaultStore().set(isSubmittingFamily(index), false);
|
||||
const adopt = () => fireEvent.click(screen.getByRole('link', { name: adoptLabel }));
|
||||
const spoken = () => events.filter((event) => event.startsWith('speak:'));
|
||||
|
||||
it.each([0, 1])(
|
||||
'retains speech with a combined eligibility/route commit in pane %i',
|
||||
async (index) => {
|
||||
const { router } = setup(index);
|
||||
await act(async () => {
|
||||
flushSync(() => {
|
||||
finalize(index);
|
||||
void router.navigate(`/c/${savedId}`, { flushSync: true });
|
||||
});
|
||||
});
|
||||
expect(committedRoute).toBe(savedId);
|
||||
expect(active?.text).toBe(message.text);
|
||||
expect(acknowledged).toBe(runId);
|
||||
expect(spoken()).toEqual([`speak:${savedId}`]);
|
||||
expect(events.some((event) => event.startsWith('commit:new:submitting=false'))).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it('defers primary-pane speech and acknowledgement until saved-route adoption', async () => {
|
||||
setup();
|
||||
await act(async () => finalize());
|
||||
expect(committedRoute).toBe('new');
|
||||
expect(active).toBeNull();
|
||||
expect(acknowledged).toBeNull();
|
||||
expect(spoken()).toEqual([]);
|
||||
|
||||
await act(async () => adopt());
|
||||
expect(committedRoute).toBe(savedId);
|
||||
expect(active?.text).toBe(message.text);
|
||||
expect(acknowledged).toBe(runId);
|
||||
expect(spoken()).toEqual([`speak:${savedId}`]);
|
||||
expect(events.filter((event) => !event.startsWith('commit:'))).toEqual([
|
||||
`cancel:${savedId}`,
|
||||
`cancel:${savedId}`,
|
||||
`speak:${savedId}`,
|
||||
]);
|
||||
expect(events.at(-1)).toBe(`commit:${savedId}:submitting=false:ack=${runId}`);
|
||||
});
|
||||
|
||||
it('allows an additional pane to speak its own conversation before primary-route adoption', async () => {
|
||||
setup(1);
|
||||
await act(async () => finalize(1));
|
||||
expect(committedRoute).toBe('new');
|
||||
expect(active?.text).toBe(message.text);
|
||||
expect(acknowledged).toBe(runId);
|
||||
expect(spoken()).toEqual(['speak:new']);
|
||||
});
|
||||
|
||||
it.each(['before-adoption', 'after-adoption'])(
|
||||
'retries delayed voices %s without premature acknowledgement or replay',
|
||||
async (arrival) => {
|
||||
const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
voicesAvailable = false;
|
||||
act(() => voicesChanged());
|
||||
setup();
|
||||
await act(async () => finalize());
|
||||
const loadVoices = () => {
|
||||
voicesAvailable = true;
|
||||
voicesChanged();
|
||||
};
|
||||
if (arrival === 'before-adoption') {
|
||||
await act(async () => loadVoices());
|
||||
expect(active).toBeNull();
|
||||
expect(acknowledged).toBeNull();
|
||||
await act(async () => adopt());
|
||||
} else {
|
||||
await act(async () => adopt());
|
||||
expect(active).toBeNull();
|
||||
expect(acknowledged).toBeNull();
|
||||
await act(async () => loadVoices());
|
||||
}
|
||||
expect(active?.text).toBe(message.text);
|
||||
expect(acknowledged).toBe(runId);
|
||||
expect(spoken()).toEqual([`speak:${savedId}`]);
|
||||
await act(async () => voicesChanged());
|
||||
expect(active?.text).toBe(message.text);
|
||||
expect(spoken()).toHaveLength(1);
|
||||
warn.mockRestore();
|
||||
},
|
||||
);
|
||||
|
||||
it.each([0, 1])('cancels accepted speech on a different conversation in pane %i', async (index) => {
|
||||
setup(index);
|
||||
await act(async () => adopt());
|
||||
await act(async () => finalize(index));
|
||||
expect(active?.text).toBe(message.text);
|
||||
await act(async () => fireEvent.click(screen.getByRole('link', { name: exitLabel })));
|
||||
expect(committedRoute).toBe('other-chat');
|
||||
expect(active).toBeNull();
|
||||
expect(acknowledged).toBe(runId);
|
||||
expect(spoken()).toHaveLength(1);
|
||||
});
|
||||
|
||||
it.each([0, 1])('cancels accepted speech on unmount in pane %i', async (index) => {
|
||||
const view = setup(index);
|
||||
await act(async () => adopt());
|
||||
await act(async () => finalize(index));
|
||||
expect(active?.text).toBe(message.text);
|
||||
await act(async () => view.unmount());
|
||||
expect(active).toBeNull();
|
||||
expect(acknowledged).toBe(runId);
|
||||
expect(spoken()).toHaveLength(1);
|
||||
});
|
||||
|
|
@ -1,11 +1,11 @@
|
|||
import React from 'react';
|
||||
import { RecoilRoot } from 'recoil';
|
||||
import { getDefaultStore } from 'jotai';
|
||||
import { render, waitFor } from '@testing-library/react';
|
||||
import { MemoryRouter, Routes, Route } from 'react-router-dom';
|
||||
import { RecoilRoot, useRecoilValue } from 'recoil';
|
||||
import { Link, MemoryRouter, Routes, Route } from 'react-router-dom';
|
||||
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
|
||||
import { isSubmittingFamily, resetSubmitting } from '@librechat/chat/react';
|
||||
import { Constants, QueryKeys, ContentTypes } from 'librechat-data-provider';
|
||||
import { act, render, screen, waitFor, fireEvent } from '@testing-library/react';
|
||||
import type { TMessage, TConversation } from 'librechat-data-provider';
|
||||
import { activeRunFamily } from '~/hooks/SSE/atoms';
|
||||
import BrowserAudio from '../BrowserAudio';
|
||||
|
|
@ -14,8 +14,10 @@ import store from '~/store';
|
|||
const conversationId = 'convo-1';
|
||||
const voiceName = 'Test Voice';
|
||||
const responseText = 'The capital of Germany is Berlin.';
|
||||
const nextChatLabel = 'Next chat';
|
||||
|
||||
const spoken: string[] = [];
|
||||
const speechEvents: string[] = [];
|
||||
let cancelCount = 0;
|
||||
|
||||
class FakeSpeechSynthesisUtterance {
|
||||
|
|
@ -26,14 +28,20 @@ class FakeSpeechSynthesisUtterance {
|
|||
}
|
||||
|
||||
const fakeVoice = { name: voiceName, localService: true } as SpeechSynthesisVoice;
|
||||
let availableVoices = [fakeVoice];
|
||||
let voicesChanged = () => {};
|
||||
|
||||
const speechSynthesis = {
|
||||
getVoices: () => [fakeVoice],
|
||||
addEventListener: () => undefined,
|
||||
getVoices: () => availableVoices,
|
||||
addEventListener: (_event: string, listener: () => void) => {
|
||||
voicesChanged = listener;
|
||||
},
|
||||
speak: (utterance: FakeSpeechSynthesisUtterance) => {
|
||||
speechEvents.push('speak');
|
||||
spoken.push(utterance.text);
|
||||
},
|
||||
cancel: () => {
|
||||
speechEvents.push('cancel');
|
||||
cancelCount += 1;
|
||||
},
|
||||
};
|
||||
|
|
@ -46,16 +54,25 @@ const assistantMessage = {
|
|||
text: responseText,
|
||||
} as TMessage;
|
||||
|
||||
function AudioRunProbe() {
|
||||
const audioRunId = useRecoilValue(store.audioRunFamily(0));
|
||||
return <output data-testid="played-run">{audioRunId ?? 'none'}</output>;
|
||||
}
|
||||
|
||||
const renderBrowserAudio = ({
|
||||
activeRunId = 'run-1',
|
||||
audioRunId = null,
|
||||
isSubmitting = false,
|
||||
messages = [assistantMessage],
|
||||
initialPath = `/c/${conversationId}`,
|
||||
nextPath = '/c/other-chat',
|
||||
}: {
|
||||
activeRunId?: string | null;
|
||||
audioRunId?: string | null;
|
||||
isSubmitting?: boolean;
|
||||
messages?: TMessage[];
|
||||
initialPath?: string;
|
||||
nextPath?: string;
|
||||
} = {}) => {
|
||||
const queryClient = new QueryClient({
|
||||
defaultOptions: { queries: { retry: false } },
|
||||
|
|
@ -64,7 +81,7 @@ const renderBrowserAudio = ({
|
|||
|
||||
return render(
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<MemoryRouter initialEntries={[`/c/${conversationId}`]}>
|
||||
<MemoryRouter initialEntries={[initialPath]}>
|
||||
<RecoilRoot
|
||||
initializeState={({ set }) => {
|
||||
set(store.voice, voiceName);
|
||||
|
|
@ -74,6 +91,8 @@ const renderBrowserAudio = ({
|
|||
getDefaultStore().set(isSubmittingFamily(0), isSubmitting);
|
||||
}}
|
||||
>
|
||||
<AudioRunProbe />
|
||||
<Link to={nextPath}>{nextChatLabel}</Link>
|
||||
<Routes>
|
||||
<Route path="/c/:conversationId" element={<BrowserAudio index={0} />} />
|
||||
</Routes>
|
||||
|
|
@ -106,13 +125,37 @@ describe('BrowserAudio autoplay', () => {
|
|||
|
||||
beforeEach(() => {
|
||||
spoken.length = 0;
|
||||
speechEvents.length = 0;
|
||||
cancelCount = 0;
|
||||
availableVoices = [fakeVoice];
|
||||
act(() => voicesChanged());
|
||||
});
|
||||
|
||||
it('speaks the finalized assistant message through speech synthesis', async () => {
|
||||
renderBrowserAudio();
|
||||
|
||||
await waitFor(() => expect(spoken).toEqual([responseText]));
|
||||
expect(screen.getByTestId('played-run')).toHaveTextContent('run-1');
|
||||
});
|
||||
|
||||
it('keeps an unqueued run eligible until the voices arrive', async () => {
|
||||
const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
availableVoices = [];
|
||||
act(() => voicesChanged());
|
||||
renderBrowserAudio();
|
||||
await settle();
|
||||
|
||||
expect(spoken).toEqual([]);
|
||||
expect(screen.getByTestId('played-run')).toHaveTextContent('none');
|
||||
|
||||
act(() => {
|
||||
availableVoices = [fakeVoice];
|
||||
voicesChanged();
|
||||
});
|
||||
|
||||
await waitFor(() => expect(spoken).toEqual([responseText]));
|
||||
expect(screen.getByTestId('played-run')).toHaveTextContent('run-1');
|
||||
warn.mockRestore();
|
||||
});
|
||||
|
||||
it('speaks the answer without the reasoning that preceded it', async () => {
|
||||
|
|
@ -186,4 +229,68 @@ describe('BrowserAudio autoplay', () => {
|
|||
|
||||
expect(cancelCount).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('cancels on route navigation while the adapter remains mounted', async () => {
|
||||
renderBrowserAudio();
|
||||
await waitFor(() => expect(spoken).toEqual([responseText]));
|
||||
const cancelsBeforeNavigation = cancelCount;
|
||||
|
||||
fireEvent.click(screen.getByRole('link', { name: 'Next chat' }));
|
||||
|
||||
expect(cancelCount).toBe(cancelsBeforeNavigation + 1);
|
||||
expect(spoken).toEqual([responseText]);
|
||||
});
|
||||
|
||||
it('starts the finalized reply after the new-chat route cleans up', async () => {
|
||||
renderBrowserAudio({
|
||||
initialPath: '/c/new',
|
||||
nextPath: `/c/${conversationId}`,
|
||||
isSubmitting: true,
|
||||
activeRunId: null,
|
||||
});
|
||||
await settle();
|
||||
expect(spoken).toEqual([]);
|
||||
|
||||
act(() => {
|
||||
getDefaultStore().set(activeRunFamily(0), 'run-1');
|
||||
getDefaultStore().set(isSubmittingFamily(0), false);
|
||||
fireEvent.click(screen.getByRole('link', { name: nextChatLabel }));
|
||||
});
|
||||
|
||||
await waitFor(() => expect(spoken).toEqual([responseText]));
|
||||
expect(screen.getByTestId('played-run')).toHaveTextContent('run-1');
|
||||
// Route cleanup and the speech service's pre-utterance cancellation both precede speaking.
|
||||
expect(cancelCount).toBe(2);
|
||||
expect(speechEvents).toEqual(['cancel', 'cancel', 'speak']);
|
||||
});
|
||||
|
||||
it('does not cancel or replay an accepted utterance when voices update', async () => {
|
||||
renderBrowserAudio();
|
||||
await waitFor(() => expect(spoken).toEqual([responseText]));
|
||||
const cancelsAfterSpeaking = cancelCount;
|
||||
|
||||
act(() => voicesChanged());
|
||||
|
||||
expect(cancelCount).toBe(cancelsAfterSpeaking);
|
||||
expect(spoken).toEqual([responseText]);
|
||||
expect(screen.getByTestId('played-run')).toHaveTextContent('run-1');
|
||||
});
|
||||
|
||||
it('still cancels accepted speech if a later voice refresh fails', async () => {
|
||||
const { unmount } = renderBrowserAudio();
|
||||
await waitFor(() => expect(spoken).toEqual([responseText]));
|
||||
const cancelsAfterSpeaking = cancelCount;
|
||||
const error = jest.spyOn(console, 'error').mockImplementation(() => {});
|
||||
const getVoices = jest.spyOn(speechSynthesis, 'getVoices').mockImplementationOnce(() => {
|
||||
throw new Error('Voice service unavailable');
|
||||
});
|
||||
|
||||
act(() => voicesChanged());
|
||||
expect(cancelCount).toBe(cancelsAfterSpeaking);
|
||||
unmount();
|
||||
|
||||
expect(cancelCount).toBe(cancelsAfterSpeaking + 1);
|
||||
getVoices.mockRestore();
|
||||
error.mockRestore();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
265
e2e/specs/mock/scenarios/browser-audio.spec.ts
Normal file
265
e2e/specs/mock/scenarios/browser-audio.spec.ts
Normal file
|
|
@ -0,0 +1,265 @@
|
|||
import { randomUUID } from 'node:crypto';
|
||||
import { expect, test } from '@playwright/test';
|
||||
import type { Page } from '@playwright/test';
|
||||
import {
|
||||
MOCK_ENDPOINTS,
|
||||
NEW_CHAT_PATH,
|
||||
replyPrompt,
|
||||
replyText,
|
||||
thinkPrompt,
|
||||
messagesView,
|
||||
selectMockEndpoint,
|
||||
sendMessageAndWaitForCompletion,
|
||||
} from '../helpers';
|
||||
import { deleteConversations, deleteMessagesByConversation } from '../db';
|
||||
|
||||
type SpeechEvent = { type: 'cancel' | 'speak' | 'end' | 'voices' | 'unavailable'; path: string };
|
||||
type SpeechBoundary = {
|
||||
events: SpeechEvent[];
|
||||
spoken: string[];
|
||||
activeText: string | null;
|
||||
setVoices: (ready: boolean, fail?: boolean) => void;
|
||||
finish: () => void;
|
||||
};
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
browserAudio: SpeechBoundary;
|
||||
}
|
||||
}
|
||||
|
||||
const createdConversationIds: string[] = [];
|
||||
|
||||
test.describe.configure({ timeout: 120_000 });
|
||||
|
||||
test.afterAll(async () => {
|
||||
await deleteMessagesByConversation(createdConversationIds);
|
||||
await deleteConversations(createdConversationIds);
|
||||
});
|
||||
|
||||
/** Replace the Web Speech boundary; native utterances reject service-fake voice objects. */
|
||||
async function openChat(page: Page, voicesReady = true) {
|
||||
await page.addInitScript((ready) => {
|
||||
for (const [key, value] of Object.entries({
|
||||
engineSTT: 'browser',
|
||||
engineTTS: 'browser',
|
||||
textToSpeech: true,
|
||||
automaticPlayback: true,
|
||||
voice: 'E2E local voice',
|
||||
cloudBrowserVoices: false,
|
||||
})) {
|
||||
localStorage.setItem(key, JSON.stringify(value));
|
||||
}
|
||||
|
||||
const target = new EventTarget();
|
||||
const voice: SpeechSynthesisVoice = {
|
||||
name: 'E2E local voice',
|
||||
voiceURI: 'e2e-local',
|
||||
lang: 'en-US',
|
||||
localService: true,
|
||||
default: true,
|
||||
};
|
||||
let available = ready;
|
||||
let fail = false;
|
||||
class Utterance extends EventTarget {
|
||||
voice: SpeechSynthesisVoice | null = null;
|
||||
onend: ((event: Event) => void) | null = null;
|
||||
onerror: ((event: Event) => void) | null = null;
|
||||
|
||||
constructor(public text: string) {
|
||||
super();
|
||||
this.addEventListener('end', (event) => this.onend?.(event));
|
||||
this.addEventListener('error', (event) => this.onerror?.(event));
|
||||
}
|
||||
}
|
||||
Object.defineProperty(window, 'SpeechSynthesisUtterance', {
|
||||
configurable: true,
|
||||
value: Utterance,
|
||||
});
|
||||
let current: Utterance | null = null;
|
||||
const record = (type: SpeechEvent['type']) => {
|
||||
window.browserAudio.events.push({ type, path: location.pathname });
|
||||
};
|
||||
window.browserAudio = {
|
||||
events: [],
|
||||
spoken: [],
|
||||
activeText: null,
|
||||
setVoices: (nextReady, nextFail = false) => {
|
||||
available = nextReady;
|
||||
fail = nextFail;
|
||||
record('voices');
|
||||
target.dispatchEvent(new Event('voiceschanged'));
|
||||
},
|
||||
finish: () => {
|
||||
const utterance = current;
|
||||
current = null;
|
||||
window.browserAudio.activeText = null;
|
||||
record('end');
|
||||
utterance?.dispatchEvent(new Event('end'));
|
||||
},
|
||||
};
|
||||
Object.defineProperty(window, 'speechSynthesis', {
|
||||
configurable: true,
|
||||
value: {
|
||||
getVoices: () => {
|
||||
if (fail) {
|
||||
record('unavailable');
|
||||
throw new Error('E2E voice service unavailable');
|
||||
}
|
||||
return available ? [voice] : [];
|
||||
},
|
||||
addEventListener: target.addEventListener.bind(target),
|
||||
removeEventListener: target.removeEventListener.bind(target),
|
||||
speak: (utterance: Utterance) => {
|
||||
current = utterance;
|
||||
window.browserAudio.activeText = utterance.text;
|
||||
window.browserAudio.spoken.push(utterance.text);
|
||||
record('speak');
|
||||
},
|
||||
cancel: () => {
|
||||
const utterance = current;
|
||||
current = null;
|
||||
window.browserAudio.activeText = null;
|
||||
record('cancel');
|
||||
if (utterance) {
|
||||
const event = new Event('error');
|
||||
Object.defineProperty(event, 'error', { value: 'canceled' });
|
||||
queueMicrotask(() => utterance.dispatchEvent(event));
|
||||
}
|
||||
},
|
||||
},
|
||||
});
|
||||
}, voicesReady);
|
||||
await page.goto(NEW_CHAT_PATH);
|
||||
await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
|
||||
}
|
||||
|
||||
async function reply(page: Page, reasoning = false) {
|
||||
const label = `browser-audio-${randomUUID().slice(0, 8)}`;
|
||||
const response = await sendMessageAndWaitForCompletion(
|
||||
page,
|
||||
reasoning ? thinkPrompt(label) : replyPrompt(label),
|
||||
);
|
||||
const { conversationId } = (await response.json()) as { conversationId: string };
|
||||
createdConversationIds.push(conversationId);
|
||||
await expect(messagesView(page).getByText(replyText(label), { exact: true })).toBeVisible();
|
||||
await expect(page.getByRole('button', { name: 'Stop generating' })).toHaveCount(0);
|
||||
return replyText(label);
|
||||
}
|
||||
|
||||
/** Let the voiceschanged subscription and its passive playback effect reach a browser paint. */
|
||||
async function refreshVoices(page: Page, fail = false) {
|
||||
await page.evaluate(async (unavailable) => {
|
||||
window.browserAudio.setVoices(true, unavailable);
|
||||
await new Promise<void>((resolve) =>
|
||||
requestAnimationFrame(() => requestAnimationFrame(() => resolve())),
|
||||
);
|
||||
}, fail);
|
||||
}
|
||||
|
||||
const speech = (page: Page) =>
|
||||
page.evaluate(() => ({
|
||||
events: window.browserAudio.events,
|
||||
spoken: window.browserAudio.spoken,
|
||||
activeText: window.browserAudio.activeText,
|
||||
}));
|
||||
|
||||
async function newChat(page: Page) {
|
||||
await page
|
||||
.getByRole('link', { name: 'New chat', exact: true })
|
||||
.or(page.getByRole('button', { name: 'New chat', exact: true }))
|
||||
.filter({ visible: true })
|
||||
.first()
|
||||
.click();
|
||||
await expect(page).toHaveURL(/\/c\/new$/);
|
||||
}
|
||||
|
||||
test.describe('browser speech autoplay', () => {
|
||||
test('speaks only the answer after new-chat cleanup and acknowledges the run without replay @scenario:browser-audio-finalized-answer', async ({
|
||||
page,
|
||||
}) => {
|
||||
await openChat(page);
|
||||
const answer = await reply(page, true);
|
||||
await expect.poll(async () => (await speech(page)).spoken).toEqual([answer]);
|
||||
const queued = await speech(page);
|
||||
expect(queued.activeText).toBe(answer);
|
||||
expect(queued.events.map((event) => event.type)).toEqual(['cancel', 'cancel', 'speak']);
|
||||
expect(queued.events.every((event) => event.path === new URL(page.url()).pathname)).toBe(true);
|
||||
|
||||
await page.evaluate(() => window.browserAudio.finish());
|
||||
await refreshVoices(page);
|
||||
expect((await speech(page)).spoken).toEqual([answer]);
|
||||
expect((await speech(page)).activeText).toBeNull();
|
||||
|
||||
const nextAnswer = await reply(page);
|
||||
await expect.poll(async () => (await speech(page)).spoken).toEqual([answer, nextAnswer]);
|
||||
});
|
||||
|
||||
test('leaves a refused run eligible until voices become available @scenario:browser-audio-late-voices', async ({
|
||||
page,
|
||||
}) => {
|
||||
await openChat(page, false);
|
||||
const answer = await reply(page);
|
||||
expect((await speech(page)).spoken).toEqual([]);
|
||||
|
||||
await refreshVoices(page);
|
||||
await expect.poll(async () => (await speech(page)).spoken).toEqual([answer]);
|
||||
await refreshVoices(page);
|
||||
expect((await speech(page)).spoken).toEqual([answer]);
|
||||
expect((await speech(page)).activeText).toBe(answer);
|
||||
});
|
||||
|
||||
test('cancels active speech on route navigation and when the chat unmounts @scenario:browser-audio-route-cancellation', async ({
|
||||
page,
|
||||
}) => {
|
||||
await openChat(page);
|
||||
const answer = await reply(page);
|
||||
await expect.poll(async () => (await speech(page)).activeText).toBe(answer);
|
||||
await newChat(page);
|
||||
await expect.poll(async () => (await speech(page)).activeText).toBeNull();
|
||||
expect((await speech(page)).events.at(-1)).toEqual({ type: 'cancel', path: NEW_CHAT_PATH });
|
||||
|
||||
await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
|
||||
const nextAnswer = await reply(page);
|
||||
await expect.poll(async () => (await speech(page)).activeText).toBe(nextAnswer);
|
||||
await page.evaluate(() => {
|
||||
history.pushState(history.state, '', '/search');
|
||||
window.dispatchEvent(new PopStateEvent('popstate'));
|
||||
});
|
||||
await expect(page.getByRole('textbox', { name: 'Message input' })).toHaveCount(0);
|
||||
await expect.poll(async () => (await speech(page)).activeText).toBeNull();
|
||||
expect((await speech(page)).events.at(-1)).toEqual({ type: 'cancel', path: '/search' });
|
||||
});
|
||||
|
||||
test('retains cancellation after a later voice refresh fails @scenario:browser-audio-failed-voice-refresh', async ({
|
||||
page,
|
||||
}) => {
|
||||
await openChat(page);
|
||||
const answer = await reply(page);
|
||||
await expect.poll(async () => (await speech(page)).activeText).toBe(answer);
|
||||
await refreshVoices(page, true);
|
||||
expect((await speech(page)).events.at(-1)?.type).toBe('unavailable');
|
||||
expect((await speech(page)).activeText).toBe(answer);
|
||||
|
||||
await newChat(page);
|
||||
await expect.poll(async () => (await speech(page)).activeText).toBeNull();
|
||||
expect((await speech(page)).events.at(-1)).toEqual({ type: 'cancel', path: NEW_CHAT_PATH });
|
||||
expect((await speech(page)).spoken).toEqual([answer]);
|
||||
});
|
||||
|
||||
test('does not speak an empty completed answer @scenario:browser-audio-ineligible-message', async ({
|
||||
page,
|
||||
}) => {
|
||||
await openChat(page);
|
||||
const response = await sendMessageAndWaitForCompletion(
|
||||
page,
|
||||
`E2E_EMPTY_REPLY:browser-audio-${randomUUID().slice(0, 8)}`,
|
||||
);
|
||||
const { conversationId } = (await response.json()) as { conversationId: string };
|
||||
createdConversationIds.push(conversationId);
|
||||
await expect(page.getByRole('button', { name: 'Stop generating' })).toHaveCount(0);
|
||||
await refreshVoices(page);
|
||||
expect((await speech(page)).spoken).toEqual([]);
|
||||
expect((await speech(page)).activeText).toBeNull();
|
||||
});
|
||||
});
|
||||
|
|
@ -17,6 +17,8 @@ export type {
|
|||
} from './components/Part';
|
||||
|
||||
/** Composer: the chat input, its controls and the steering queue. */
|
||||
export { default as BrowserAudio } from './components/composer/Audio';
|
||||
export type { BrowserAudioProps } from './components/composer/Audio';
|
||||
export { default as StopButton } from './components/composer/StopButton';
|
||||
export { default as MentionItem } from './components/composer/MentionItem';
|
||||
export { default as MentionPopover } from './components/composer/popover/Mention';
|
||||
|
|
|
|||
56
packages/chat/src/components/composer/Audio.tsx
Normal file
56
packages/chat/src/components/composer/Audio.tsx
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
import { useEffect, useRef } from 'react';
|
||||
import { getSpeechText } from 'librechat-data-provider';
|
||||
import type { TMessage } from 'librechat-data-provider';
|
||||
|
||||
export interface BrowserAudioProps {
|
||||
/** The route lifetime that owns playback, including the new-chat route. */
|
||||
routeKey?: string;
|
||||
shouldPlay: boolean;
|
||||
activeRunId: string | null;
|
||||
latestMessage: TMessage | null;
|
||||
/** Returns true only when an utterance was queued; may change when voices load. */
|
||||
speak: (text: string) => boolean;
|
||||
cancel: () => void;
|
||||
onStarted: (runId: string) => void;
|
||||
}
|
||||
|
||||
/** Browser speech playback, with route, voice configuration and run state supplied by the host. */
|
||||
export default function BrowserAudio({
|
||||
routeKey,
|
||||
shouldPlay,
|
||||
activeRunId,
|
||||
latestMessage,
|
||||
speak,
|
||||
cancel,
|
||||
onStarted,
|
||||
}: BrowserAudioProps): null {
|
||||
const cancelRef = useRef(cancel);
|
||||
useEffect(() => {
|
||||
cancelRef.current = cancel;
|
||||
}, [cancel]);
|
||||
|
||||
/** Cleanup belongs only to the route lifetime, not voice availability or callback identity.
|
||||
* React runs cleanups before effect bodies, so same-commit new-chat finalization cancels the
|
||||
* previous utterance before starting the newly finalized reply. */
|
||||
useEffect(() => {
|
||||
// Preserve this route's cancellation even if a later voice refresh reports no support.
|
||||
const cancelRoute = cancelRef.current;
|
||||
return () => cancelRoute();
|
||||
}, [routeKey]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!shouldPlay || activeRunId == null || latestMessage == null) {
|
||||
return;
|
||||
}
|
||||
const text = getSpeechText(latestMessage);
|
||||
if (!text) {
|
||||
return;
|
||||
}
|
||||
// A voice list that has not loaded yet must leave the run eligible for a later retry.
|
||||
if (speak(text)) {
|
||||
onStarted(activeRunId);
|
||||
}
|
||||
}, [shouldPlay, activeRunId, latestMessage, onStarted, speak]);
|
||||
|
||||
return null;
|
||||
}
|
||||
128
packages/chat/src/components/composer/__tests__/Audio.spec.tsx
Normal file
128
packages/chat/src/components/composer/__tests__/Audio.spec.tsx
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
/** @jest-environment jsdom */
|
||||
import { render } from '@testing-library/react';
|
||||
import { Constants, ContentTypes } from 'librechat-data-provider';
|
||||
import type { TMessage } from 'librechat-data-provider';
|
||||
import type { BrowserAudioProps } from '../Audio';
|
||||
import BrowserAudio from '../Audio';
|
||||
|
||||
const message = {
|
||||
messageId: 'reply-1',
|
||||
conversationId: 'chat-1',
|
||||
parentMessageId: Constants.NO_PARENT,
|
||||
isCreatedByUser: false,
|
||||
text: 'A spoken reply.',
|
||||
} as TMessage;
|
||||
|
||||
function props(overrides: Partial<BrowserAudioProps> = {}): BrowserAudioProps {
|
||||
return {
|
||||
routeKey: 'chat-1',
|
||||
shouldPlay: true,
|
||||
activeRunId: 'run-1',
|
||||
latestMessage: message,
|
||||
speak: jest.fn(() => true),
|
||||
cancel: jest.fn(),
|
||||
onStarted: jest.fn(),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('BrowserAudio', () => {
|
||||
it('renders no DOM and acknowledges a run only after speech is accepted', () => {
|
||||
const input = props();
|
||||
const { container } = render(<BrowserAudio {...input} />);
|
||||
|
||||
expect(container).toBeEmptyDOMElement();
|
||||
expect(input.speak).toHaveBeenCalledWith(message.text);
|
||||
expect(input.onStarted).toHaveBeenCalledWith('run-1');
|
||||
expect(input.cancel).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each<Partial<BrowserAudioProps>>([
|
||||
{ shouldPlay: false },
|
||||
{ activeRunId: null },
|
||||
{ latestMessage: null },
|
||||
{ latestMessage: { ...message, text: '' } },
|
||||
{
|
||||
latestMessage: {
|
||||
...message,
|
||||
text: 'Private reasoning.',
|
||||
content: [{ type: ContentTypes.THINK, think: 'Private reasoning.' }],
|
||||
},
|
||||
},
|
||||
])('does not request speech for an ineligible input: %j', (override) => {
|
||||
const input = props(override);
|
||||
render(<BrowserAudio {...input} />);
|
||||
|
||||
expect(input.speak).not.toHaveBeenCalled();
|
||||
expect(input.onStarted).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('filters reasoning before speaking the answer', () => {
|
||||
const input = props({
|
||||
latestMessage: {
|
||||
...message,
|
||||
text: '',
|
||||
content: [
|
||||
{ type: ContentTypes.THINK, think: 'Private reasoning.' },
|
||||
{ type: ContentTypes.TEXT, text: 'Public answer.' },
|
||||
],
|
||||
},
|
||||
});
|
||||
render(<BrowserAudio {...input} />);
|
||||
|
||||
expect(input.speak).toHaveBeenCalledWith('Public answer.');
|
||||
});
|
||||
|
||||
it('retries refused speech when the voice service becomes ready', () => {
|
||||
const input = props({ speak: jest.fn(() => false) });
|
||||
const { rerender } = render(<BrowserAudio {...input} />);
|
||||
expect(input.onStarted).not.toHaveBeenCalled();
|
||||
|
||||
const readySpeak = jest.fn(() => true);
|
||||
rerender(<BrowserAudio {...input} speak={readySpeak} />);
|
||||
|
||||
expect(readySpeak).toHaveBeenCalledWith(message.text);
|
||||
expect(input.onStarted).toHaveBeenCalledTimes(1);
|
||||
expect(input.cancel).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('cancels the previous route before same-commit finalization starts speech', () => {
|
||||
const events: string[] = [];
|
||||
const input = props({
|
||||
routeKey: 'new',
|
||||
shouldPlay: false,
|
||||
cancel: () => events.push('cancel'),
|
||||
speak: () => {
|
||||
events.push('speak');
|
||||
return true;
|
||||
},
|
||||
onStarted: () => events.push('started'),
|
||||
});
|
||||
const { rerender, unmount } = render(<BrowserAudio {...input} />);
|
||||
|
||||
rerender(<BrowserAudio {...input} routeKey="saved-chat" shouldPlay />);
|
||||
expect(events).toEqual(['cancel', 'speak', 'started']);
|
||||
|
||||
unmount();
|
||||
expect(events).toEqual(['cancel', 'speak', 'started', 'cancel']);
|
||||
});
|
||||
|
||||
it('preserves the route cancellation callback without cancelling or replaying on updates', () => {
|
||||
const input = props();
|
||||
const { rerender, unmount } = render(<BrowserAudio {...input} />);
|
||||
const nextCancel = jest.fn();
|
||||
|
||||
rerender(<BrowserAudio {...input} cancel={nextCancel} />);
|
||||
expect(input.speak).toHaveBeenCalledTimes(1);
|
||||
expect(input.cancel).not.toHaveBeenCalled();
|
||||
expect(nextCancel).not.toHaveBeenCalled();
|
||||
|
||||
rerender(<BrowserAudio {...input} routeKey="next-chat" cancel={nextCancel} />);
|
||||
expect(input.cancel).toHaveBeenCalledTimes(1);
|
||||
expect(nextCancel).not.toHaveBeenCalled();
|
||||
|
||||
unmount();
|
||||
expect(nextCancel).toHaveBeenCalledTimes(1);
|
||||
expect(input.cancel).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue