🎙️ fix: Align Speech Engine Configuration With Runtime (#14736)

* fix: align speech engine configuration with runtime

* fix: guard speech recording shortcuts

* fix: reconcile speech engine availability

---------

Co-authored-by: Danny Avila <danny@librechat.ai>
This commit is contained in:
Marco Beretta 2026-08-12 19:22:28 +02:00 committed by GitHub
parent 92a8058f02
commit 5ff282f900
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 858 additions and 84 deletions

View file

@ -1,6 +1,21 @@
const { logger } = require('@librechat/data-schemas');
const { getAppConfig } = require('~/server/services/Config');
const LEGACY_EXTERNAL_STT_ENGINES = new Set(['openai', 'azureOpenAI']);
const LEGACY_EXTERNAL_TTS_ENGINES = new Set(['openai', 'azureOpenAI', 'elevenlabs', 'localai']);
function normalizeSpeechEngine(key, value) {
if (key === 'engineSTT' && LEGACY_EXTERNAL_STT_ENGINES.has(value)) {
return 'external';
}
if (key === 'engineTTS' && LEGACY_EXTERNAL_TTS_ENGINES.has(value)) {
return 'external';
}
return value;
}
/**
* This function retrieves the speechTab settings from the custom configuration
* It first fetches the custom configuration
@ -52,7 +67,8 @@ async function getCustomConfigSpeech(req, res) {
} else {
for (const key in speechTab.speechToText) {
if (speechTab.speechToText[key] !== undefined) {
settings[key] = speechTab.speechToText[key];
const value = speechTab.speechToText[key];
settings[key] = normalizeSpeechEngine(key, value);
}
}
}
@ -64,7 +80,8 @@ async function getCustomConfigSpeech(req, res) {
} else {
for (const key in speechTab.textToSpeech) {
if (speechTab.textToSpeech[key] !== undefined) {
settings[key] = speechTab.textToSpeech[key];
const value = speechTab.textToSpeech[key];
settings[key] = normalizeSpeechEngine(key, value);
}
}
}

View file

@ -0,0 +1,95 @@
jest.mock('@librechat/data-schemas', () => ({ logger: { error: jest.fn() } }));
jest.mock('~/server/services/Config', () => ({ getAppConfig: jest.fn() }));
const getCustomConfigSpeech = require('./getCustomConfigSpeech');
const createResponse = () => {
const res = {
send: jest.fn(),
status: jest.fn(),
};
res.status.mockReturnValue(res);
return res;
};
describe('getCustomConfigSpeech', () => {
it.each(['openai', 'azureOpenAI'])(
'normalizes the legacy STT provider "%s" to the external engine',
async (engineSTT) => {
const req = {
config: {
speech: {
stt: { openai: {} },
speechTab: { speechToText: { engineSTT } },
},
},
};
const res = createResponse();
await getCustomConfigSpeech(req, res);
expect(res.status).toHaveBeenCalledWith(200);
expect(res.send).toHaveBeenCalledWith({
sttExternal: true,
ttsExternal: false,
engineSTT: 'external',
});
},
);
it.each(['browser', 'external'])('preserves the runtime STT engine "%s"', async (engineSTT) => {
const req = {
config: {
speech: {
stt: { openai: {} },
speechTab: { speechToText: { engineSTT } },
},
},
};
const res = createResponse();
await getCustomConfigSpeech(req, res);
expect(res.send).toHaveBeenCalledWith(expect.objectContaining({ engineSTT }));
});
it.each(['openai', 'azureOpenAI', 'elevenlabs', 'localai'])(
'normalizes the legacy TTS provider "%s" to the external engine',
async (engineTTS) => {
const req = {
config: {
speech: {
tts: { openai: {} },
speechTab: { textToSpeech: { engineTTS } },
},
},
};
const res = createResponse();
await getCustomConfigSpeech(req, res);
expect(res.status).toHaveBeenCalledWith(200);
expect(res.send).toHaveBeenCalledWith({
sttExternal: false,
ttsExternal: true,
engineTTS: 'external',
});
},
);
it.each(['browser', 'external'])('preserves the runtime TTS engine "%s"', async (engineTTS) => {
const req = {
config: {
speech: {
tts: { openai: {} },
speechTab: { textToSpeech: { engineTTS } },
},
},
};
const res = createResponse();
await getCustomConfigSpeech(req, res);
expect(res.send).toHaveBeenCalledWith(expect.objectContaining({ engineTTS }));
});
});

View file

@ -1,5 +1,6 @@
import { memo, useCallback, useRef } from 'react';
import { memo, useCallback, useEffect, useRef } from 'react';
import { MicOff } from 'lucide-react';
import { useRecoilValue } from 'recoil';
import {
IconButton,
useToastContext,
@ -10,6 +11,7 @@ import {
import { useLocalize, useSpeechToText, useGetAudioSettings } from '~/hooks';
import { globalAudioId, type TAskFunction } from '~/common';
import { useChatFormContext } from '~/Providers';
import store from '~/store';
const isExternalSTT = (speechToTextEndpoint: string) => speechToTextEndpoint === 'external';
export default memo(function AudioRecorder({
@ -27,6 +29,8 @@ export default memo(function AudioRecorder({
const localize = useLocalize();
const { showToast } = useToastContext();
const { speechToTextEndpoint } = useGetAudioSettings();
const speechSettingsInitialized = useRecoilValue(store.speechSettingsInitialized);
const recorderDisabled = disabled || !speechSettingsInitialized;
const existingTextRef = useRef<string>('');
const isSubmittingRef = useRef(isSubmitting);
@ -85,18 +89,37 @@ export default memo(function AudioRecorder({
onTranscriptionComplete,
);
const handleStartRecording = async () => {
const handleStartRecording = useCallback(() => {
existingTextRef.current = getValues('text') || '';
startRecording();
};
}, [getValues, startRecording]);
const handleStopRecording = async () => {
const handleStopRecording = useCallback(() => {
stopRecording();
/** For browser STT, clear the reference since text was already being updated */
if (!isExternalSTT(speechToTextEndpoint)) {
existingTextRef.current = '';
}
};
}, [speechToTextEndpoint, stopRecording]);
useEffect(() => {
const handleKeyDown = (event: KeyboardEvent) => {
if (!event.shiftKey || !event.altKey || event.code !== 'KeyL' || recorderDisabled) {
return;
}
event.preventDefault();
if (isListening === true) {
handleStopRecording();
return;
}
handleStartRecording();
};
window.addEventListener('keydown', handleKeyDown);
return () => window.removeEventListener('keydown', handleKeyDown);
}, [handleStartRecording, handleStopRecording, isListening, recorderDisabled]);
const renderIcon = () => {
if (isListening === true) {
@ -120,7 +143,7 @@ export default memo(function AudioRecorder({
shape="theme"
label={localize('com_ui_use_micrphone')}
onClick={isListening === true ? handleStopRecording : handleStartRecording}
disabled={disabled}
disabled={recorderDisabled}
className="p-1 hover:bg-surface-composer-hover"
aria-pressed={isListening}
>

View file

@ -0,0 +1,181 @@
import React from 'react';
import { RecoilRoot } from 'recoil';
import { act, render, screen } from '@testing-library/react';
import AudioRecorder from '../AudioRecorder';
import store from '~/store';
let mockSpeechToTextEndpoint = 'browser';
let mockBrowserIsListening = false;
let mockExternalIsListening = false;
let mockSetText: ((text: string) => void) | undefined;
const mockStartSpeechRecordingBrowser = jest.fn();
const mockStopSpeechRecordingBrowser = jest.fn();
const mockStartSpeechRecordingExternal = jest.fn();
const mockStopSpeechRecordingExternal = jest.fn();
const mockSetValue = jest.fn();
const mockReset = jest.fn();
const mockGetValues = jest.fn(() => 'existing draft');
const mockAsk = jest.fn(() => true);
type MockButtonProps = React.ComponentProps<'button'> & {
label?: string;
variant?: string;
size?: string;
shape?: string;
};
jest.mock('@librechat/client', () => ({
IconButton: ({
children,
label,
variant: _variant,
size: _size,
shape: _shape,
...props
}: MockButtonProps) => (
<button aria-label={label} {...props}>
{children}
</button>
),
TooltipAnchor: ({ render }: { render: React.ReactElement }) => render,
ListeningIcon: () => <span />,
Spinner: () => <span />,
useToastContext: () => ({ showToast: jest.fn() }),
}));
jest.mock('~/hooks/Input/useSpeechToTextBrowser', () => ({
__esModule: true,
default: (setText: (text: string) => void) => {
mockSetText = setText;
return {
isListening: mockBrowserIsListening,
isLoading: false,
startRecording: mockStartSpeechRecordingBrowser,
stopRecording: mockStopSpeechRecordingBrowser,
};
},
}));
jest.mock('~/hooks/Input/useSpeechToTextExternal', () => ({
__esModule: true,
default: (setText: (text: string) => void) => {
mockSetText = setText;
return {
isListening: mockExternalIsListening,
isLoading: false,
externalStartRecording: mockStartSpeechRecordingExternal,
externalStopRecording: mockStopSpeechRecordingExternal,
};
},
}));
jest.mock('~/hooks/Input/useGetAudioSettings', () => ({
__esModule: true,
default: () => ({ speechToTextEndpoint: mockSpeechToTextEndpoint }),
}));
jest.mock('~/hooks', () => ({
useLocalize: () => (key: string) => key,
useGetAudioSettings: () => ({ speechToTextEndpoint: mockSpeechToTextEndpoint }),
useSpeechToText: jest.requireActual('~/hooks/Input/useSpeechToText').default,
}));
jest.mock('~/common', () => ({
...jest.requireActual('~/common'),
globalAudioId: 'global-audio',
}));
const dispatchSpeechShortcut = () => {
const event = new KeyboardEvent('keydown', {
shiftKey: true,
altKey: true,
code: 'KeyL',
bubbles: true,
cancelable: true,
});
act(() => window.dispatchEvent(event));
return event;
};
const renderRecorder = ({ disabled = false, initialized = true } = {}) =>
render(
<RecoilRoot initializeState={({ set }) => set(store.speechSettingsInitialized, initialized)}>
<AudioRecorder
disabled={disabled}
ask={mockAsk as never}
methods={
{
setValue: mockSetValue,
reset: mockReset,
getValues: mockGetValues,
} as never
}
isSubmitting={false}
/>
</RecoilRoot>,
);
describe('AudioRecorder speech shortcut', () => {
beforeEach(() => {
jest.clearAllMocks();
mockSpeechToTextEndpoint = 'browser';
mockBrowserIsListening = false;
mockExternalIsListening = false;
mockSetText = undefined;
});
it('preserves the existing draft when browser recording starts from the shortcut', () => {
renderRecorder();
const event = dispatchSpeechShortcut();
act(() => mockSetText?.('transcript'));
expect(mockStartSpeechRecordingBrowser).toHaveBeenCalledTimes(1);
expect(mockSetValue).toHaveBeenCalledWith('text', 'existing draft transcript', {
shouldValidate: true,
});
expect(event.defaultPrevented).toBe(true);
});
it('does not invoke either recorder while the control is disabled', () => {
renderRecorder({ disabled: true });
dispatchSpeechShortcut();
expect(mockStartSpeechRecordingBrowser).not.toHaveBeenCalled();
expect(mockStopSpeechRecordingBrowser).not.toHaveBeenCalled();
});
it('does not start browser recording before speech settings initialization completes', () => {
renderRecorder({ initialized: false });
dispatchSpeechShortcut();
expect(mockStartSpeechRecordingBrowser).not.toHaveBeenCalled();
expect(mockStartSpeechRecordingExternal).not.toHaveBeenCalled();
expect(screen.getByRole('button', { name: 'com_ui_use_micrphone' })).toBeDisabled();
});
it('stops the selected engine from the shortcut', () => {
mockSpeechToTextEndpoint = 'external';
mockExternalIsListening = true;
renderRecorder();
dispatchSpeechShortcut();
expect(mockStopSpeechRecordingExternal).toHaveBeenCalledTimes(1);
expect(mockStopSpeechRecordingBrowser).not.toHaveBeenCalled();
});
it('does not stop a recorder while the control is disabled', () => {
mockSpeechToTextEndpoint = 'external';
mockExternalIsListening = true;
renderRecorder({ disabled: true });
dispatchSpeechShortcut();
expect(mockStopSpeechRecordingExternal).not.toHaveBeenCalled();
});
});

View file

@ -7,6 +7,11 @@ import store from '~/store';
function MessageAudio(props: TMessageAudio) {
const engineTTS = useRecoilValue<string>(store.engineTTS);
const speechSettingsInitialized = useRecoilValue(store.speechSettingsInitialized);
if (!speechSettingsInitialized) {
return null;
}
const TTSComponents = {
[TTSEndpoints.browser]: BrowserTTS,

View file

@ -0,0 +1,44 @@
import React from 'react';
import { RecoilRoot } from 'recoil';
import { render, screen } from '@testing-library/react';
import MessageAudio from '../MessageAudio';
import store from '~/store';
jest.mock('~/components/Audio/TTS', () => ({
BrowserTTS: () => <div data-testid="browser-tts" />,
ExternalTTS: () => <div data-testid="external-tts" />,
}));
const renderMessageAudio = ({
engineTTS,
initialized,
}: {
engineTTS: 'browser' | 'external';
initialized: boolean;
}) =>
render(
<RecoilRoot
initializeState={({ set }) => {
set(store.engineTTS, engineTTS);
set(store.speechSettingsInitialized, initialized);
}}
>
<MessageAudio index={0} messageId="message-1" content="Read this" />
</RecoilRoot>,
);
describe('MessageAudio speech settings initialization', () => {
it('does not mount browser TTS before speech settings initialize', () => {
renderMessageAudio({ engineTTS: 'browser', initialized: false });
expect(screen.queryByTestId('browser-tts')).not.toBeInTheDocument();
expect(screen.queryByTestId('external-tts')).not.toBeInTheDocument();
});
it('mounts external TTS after its configured engine initializes', () => {
renderMessageAudio({ engineTTS: 'external', initialized: true });
expect(screen.getByTestId('external-tts')).toBeInTheDocument();
expect(screen.queryByTestId('browser-tts')).not.toBeInTheDocument();
});
});

View file

@ -0,0 +1,252 @@
import React from 'react';
import { RecoilRoot, useRecoilValue } from 'recoil';
import { renderHook, waitFor } from '@testing-library/react';
const mockUseGetCustomConfigSpeechQuery = jest.fn();
jest.mock('librechat-data-provider/react-query', () => ({
useGetCustomConfigSpeechQuery: () => mockUseGetCustomConfigSpeechQuery(),
}));
jest.mock('~/utils', () => ({
logger: { log: jest.fn() },
}));
import useSpeechSettingsInit from '../useSpeechSettingsInit';
import store from '~/store';
const wrapper = ({ children }: { children: React.ReactNode }) => (
<RecoilRoot>{children}</RecoilRoot>
);
const useSpeechSettingsHarness = (isAuthenticated = true) => {
useSpeechSettingsInit(isAuthenticated);
return {
engineSTT: useRecoilValue(store.engineSTT),
engineTTS: useRecoilValue(store.engineTTS),
speechSettingsInitialized: useRecoilValue(store.speechSettingsInitialized),
};
};
describe('useSpeechSettingsInit', () => {
beforeEach(() => {
localStorage.clear();
mockUseGetCustomConfigSpeechQuery.mockReturnValue({
data: { sttExternal: true, ttsExternal: true },
isFetched: true,
});
});
it.each(['openai', 'azureOpenAI'])(
'migrates the persisted STT provider "%s" to the external engine',
async (engineSTT) => {
localStorage.setItem('engineSTT', JSON.stringify(engineSTT));
const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper });
await waitFor(() => expect(result.current.engineSTT).toBe('external'));
expect(localStorage.getItem('engineSTT')).toBe(JSON.stringify('external'));
},
);
it.each(['openai', 'azureOpenAI', 'elevenlabs', 'localai'])(
'migrates the persisted TTS provider "%s" to the external engine',
async (engineTTS) => {
localStorage.setItem('engineTTS', JSON.stringify(engineTTS));
const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper });
await waitFor(() => expect(result.current.engineTTS).toBe('external'));
expect(localStorage.getItem('engineTTS')).toBe(JSON.stringify('external'));
},
);
it('falls back migrated providers when no external provider is reported', async () => {
localStorage.setItem('engineSTT', JSON.stringify('openai'));
localStorage.setItem('engineTTS', JSON.stringify('elevenlabs'));
mockUseGetCustomConfigSpeechQuery.mockReturnValue({
data: { sttExternal: false, ttsExternal: false },
isFetched: true,
});
const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper });
await waitFor(() => {
expect(result.current).toEqual({
engineSTT: 'browser',
engineTTS: 'browser',
speechSettingsInitialized: true,
});
});
expect(localStorage.getItem('engineSTT')).toBe(JSON.stringify('browser'));
expect(localStorage.getItem('engineTTS')).toBe(JSON.stringify('browser'));
});
it('falls back saved external engines when their providers are unavailable', async () => {
localStorage.setItem('engineSTT', JSON.stringify('external'));
localStorage.setItem('engineTTS', JSON.stringify('external'));
mockUseGetCustomConfigSpeechQuery.mockReturnValue({
data: { sttExternal: false, ttsExternal: false },
isFetched: true,
});
const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper });
await waitFor(() => {
expect(result.current).toEqual({
engineSTT: 'browser',
engineTTS: 'browser',
speechSettingsInitialized: true,
});
});
expect(localStorage.getItem('engineSTT')).toBe(JSON.stringify('browser'));
expect(localStorage.getItem('engineTTS')).toBe(JSON.stringify('browser'));
});
it.each(['browser', 'external'])('preserves the valid engine "%s"', async (engine) => {
localStorage.setItem('engineSTT', JSON.stringify(engine));
localStorage.setItem('engineTTS', JSON.stringify(engine));
const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper });
await waitFor(() => {
expect(result.current).toEqual({
engineSTT: engine,
engineTTS: engine,
speechSettingsInitialized: true,
});
});
});
it.each([
['loading', { data: undefined, isFetched: false, isError: false }, false],
['error', { data: undefined, isFetched: true, isError: true }, true],
])(
'normalizes legacy providers while the speech configuration query is in the %s state',
async (_state, queryResult, speechSettingsInitialized) => {
localStorage.setItem('engineSTT', JSON.stringify('openai'));
localStorage.setItem('engineTTS', JSON.stringify('localai'));
mockUseGetCustomConfigSpeechQuery.mockReturnValue(queryResult);
const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper });
await waitFor(() => {
expect(result.current).toEqual({
engineSTT: 'external',
engineTTS: 'external',
speechSettingsInitialized,
});
});
expect(localStorage.getItem('engineSTT')).toBe(JSON.stringify('external'));
expect(localStorage.getItem('engineTTS')).toBe(JSON.stringify('external'));
},
);
it('normalizes unknown saved engines to browser', async () => {
localStorage.setItem('engineSTT', JSON.stringify('unknown-stt'));
localStorage.setItem('engineTTS', JSON.stringify('unknown-tts'));
const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper });
await waitFor(() => {
expect(result.current).toEqual({
engineSTT: 'browser',
engineTTS: 'browser',
speechSettingsInitialized: true,
});
});
expect(localStorage.getItem('engineSTT')).toBe(JSON.stringify('browser'));
expect(localStorage.getItem('engineTTS')).toBe(JSON.stringify('browser'));
});
it('seeds external engines from the server configuration when storage is empty', async () => {
mockUseGetCustomConfigSpeechQuery.mockReturnValue({
data: {
sttExternal: true,
ttsExternal: true,
engineSTT: 'external',
engineTTS: 'external',
},
isFetched: true,
});
const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper });
await waitFor(() => {
expect(result.current).toEqual({
engineSTT: 'external',
engineTTS: 'external',
speechSettingsInitialized: true,
});
});
expect(localStorage.getItem('engineSTT')).toBe(JSON.stringify('external'));
expect(localStorage.getItem('engineTTS')).toBe(JSON.stringify('external'));
});
it('does not write browser defaults before a server default is provided', () => {
mockUseGetCustomConfigSpeechQuery.mockReturnValue({
data: { sttExternal: false, ttsExternal: false },
isFetched: true,
});
const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper });
expect(result.current).toEqual({
engineSTT: 'browser',
engineTTS: 'browser',
speechSettingsInitialized: true,
});
expect(localStorage.getItem('engineSTT')).toBeNull();
expect(localStorage.getItem('engineTTS')).toBeNull();
});
it('keeps speech controls disabled until configuration initialization settles', () => {
mockUseGetCustomConfigSpeechQuery.mockReturnValue({
data: undefined,
isFetched: false,
});
const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper });
expect(result.current.speechSettingsInitialized).toBe(false);
});
it('enables speech controls after a missing configuration response', async () => {
mockUseGetCustomConfigSpeechQuery.mockReturnValue({
data: { message: 'not_found' },
isFetched: true,
});
const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper });
await waitFor(() => expect(result.current.speechSettingsInitialized).toBe(true));
});
it('keeps speech controls disabled after a configuration error on fresh storage', () => {
mockUseGetCustomConfigSpeechQuery.mockReturnValue({
data: undefined,
isFetched: true,
isError: true,
});
const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper });
expect(result.current.speechSettingsInitialized).toBe(false);
});
it.each(['engineSTT', 'engineTTS'])(
'keeps speech controls disabled after an error when only %s is saved',
(savedEngine) => {
localStorage.setItem(savedEngine, JSON.stringify('browser'));
mockUseGetCustomConfigSpeechQuery.mockReturnValue({
data: undefined,
isFetched: true,
isError: true,
});
const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper });
expect(result.current.speechSettingsInitialized).toBe(false);
},
);
});

View file

@ -1,7 +1,7 @@
import { useEffect, useRef } from 'react';
import { useRecoilState, useSetRecoilState } from 'recoil';
import { useGetCustomConfigSpeechQuery } from 'librechat-data-provider/react-query';
import { TTSEndpoints } from '~/common';
import { STTEndpoints, TTSEndpoints } from '~/common';
import { logger } from '~/utils';
import store from '~/store';
@ -12,8 +12,10 @@ const VALID_TTS_ENGINES: string[] = [TTSEndpoints.browser, TTSEndpoints.external
* configuration on first load (only when the user is authenticated)
*/
export default function useSpeechSettingsInit(isAuthenticated: boolean) {
const { data } = useGetCustomConfigSpeechQuery({ enabled: isAuthenticated });
const { data, isError, isFetched } = useGetCustomConfigSpeechQuery({ enabled: isAuthenticated });
const [engineSTT, setEngineSTT] = useRecoilState<string>(store.engineSTT);
const [engineTTS, setEngineTTS] = useRecoilState<string>(store.engineTTS);
const setSpeechSettingsInitialized = useSetRecoilState(store.speechSettingsInitialized);
const setters = useRef({
conversationMode: useSetRecoilState(store.conversationMode),
@ -21,7 +23,7 @@ export default function useSpeechSettingsInit(isAuthenticated: boolean) {
speechToText: useSetRecoilState(store.speechToText),
textToSpeech: useSetRecoilState(store.textToSpeech),
cacheTTS: useSetRecoilState(store.cacheTTS),
engineSTT: useSetRecoilState(store.engineSTT),
engineSTT: setEngineSTT,
languageSTT: useSetRecoilState(store.languageSTT),
autoTranscribeAudio: useSetRecoilState(store.autoTranscribeAudio),
decibelValue: useSetRecoilState(store.decibelValue),
@ -35,22 +37,65 @@ export default function useSpeechSettingsInit(isAuthenticated: boolean) {
}).current;
useEffect(() => {
if (!isAuthenticated || !data || data.message === 'not_found') return;
if (!isAuthenticated) {
setSpeechSettingsInitialized(false);
return;
}
logger.log('Initializing speech settings from config:', data);
if (!isFetched) return;
Object.entries(data).forEach(([key, value]) => {
if (key === 'sttExternal' || key === 'ttsExternal') return;
if (
isError &&
(localStorage.getItem('engineSTT') === null || localStorage.getItem('engineTTS') === null)
) {
setSpeechSettingsInitialized(false);
return;
}
if (localStorage.getItem(key) !== null) return;
const hasSavedEngineSTT = localStorage.getItem('engineSTT') !== null;
const hasSavedEngineTTS = localStorage.getItem('engineTTS') !== null;
const setter = setters[key as keyof typeof setters];
if (setter) {
logger.log(`Setting default speech setting: ${key} = ${value}`);
setter(value as any);
}
});
}, [isAuthenticated, data, setters]);
if (data && data.message !== 'not_found') {
logger.log('Initializing speech settings from config:', data);
Object.entries(data).forEach(([key, value]) => {
if (key === 'sttExternal' || key === 'ttsExternal') return;
if (localStorage.getItem(key) !== null) return;
const setter = setters[key as keyof typeof setters];
if (setter) {
logger.log(`Setting default speech setting: ${key} = ${value}`);
setter(value as any);
}
});
}
const configuredEngineSTT = hasSavedEngineSTT ? engineSTT : data?.engineSTT;
const configuredEngineTTS = hasSavedEngineTTS ? engineTTS : data?.engineTTS;
const sttExternalUnavailable = data?.sttExternal != null && !data.sttExternal;
const ttsExternalUnavailable = data?.ttsExternal != null && !data.ttsExternal;
if (sttExternalUnavailable && configuredEngineSTT === STTEndpoints.external) {
setEngineSTT(STTEndpoints.browser);
}
if (ttsExternalUnavailable && configuredEngineTTS === TTSEndpoints.external) {
setEngineTTS(TTSEndpoints.browser);
}
setSpeechSettingsInitialized(true);
}, [
data,
engineSTT,
engineTTS,
isAuthenticated,
isError,
isFetched,
setEngineSTT,
setEngineTTS,
setSpeechSettingsInitialized,
setters,
]);
useEffect(() => {
if (VALID_TTS_ENGINES.includes(engineTTS)) return;

View file

@ -0,0 +1,79 @@
import { act, renderHook } from '@testing-library/react';
import useSpeechToText from './useSpeechToText';
let mockSpeechToTextEndpoint = 'external';
let mockBrowserIsListening = false;
let mockExternalIsListening = false;
const mockStartSpeechRecordingBrowser = jest.fn();
const mockStopSpeechRecordingBrowser = jest.fn();
const mockStartSpeechRecordingExternal = jest.fn();
const mockStopSpeechRecordingExternal = jest.fn();
jest.mock('./useGetAudioSettings', () => ({
__esModule: true,
default: () => ({ speechToTextEndpoint: mockSpeechToTextEndpoint }),
}));
jest.mock('./useSpeechToTextBrowser', () => ({
__esModule: true,
default: () => ({
isListening: mockBrowserIsListening,
isLoading: false,
startRecording: mockStartSpeechRecordingBrowser,
stopRecording: mockStopSpeechRecordingBrowser,
}),
}));
jest.mock('./useSpeechToTextExternal', () => ({
__esModule: true,
default: () => ({
isListening: mockExternalIsListening,
isLoading: false,
externalStartRecording: mockStartSpeechRecordingExternal,
externalStopRecording: mockStopSpeechRecordingExternal,
}),
}));
describe('useSpeechToText', () => {
beforeEach(() => {
jest.clearAllMocks();
mockSpeechToTextEndpoint = 'external';
mockBrowserIsListening = false;
mockExternalIsListening = false;
});
it('selects the externally seeded engine after settings change', () => {
mockSpeechToTextEndpoint = 'browser';
const { result, rerender } = renderHook(() => useSpeechToText(jest.fn(), jest.fn()));
mockSpeechToTextEndpoint = 'external';
rerender();
act(() => result.current.startRecording());
expect(mockStartSpeechRecordingExternal).toHaveBeenCalledTimes(1);
expect(mockStartSpeechRecordingBrowser).not.toHaveBeenCalled();
});
it('selects the browser engine', () => {
mockSpeechToTextEndpoint = 'browser';
const { result } = renderHook(() => useSpeechToText(jest.fn(), jest.fn()));
act(() => result.current.startRecording());
expect(mockStartSpeechRecordingBrowser).toHaveBeenCalledTimes(1);
expect(mockStartSpeechRecordingExternal).not.toHaveBeenCalled();
});
it('selects the active engine stop handler', () => {
mockExternalIsListening = true;
const { result } = renderHook(() => useSpeechToText(jest.fn(), jest.fn()));
act(() => result.current.stopRecording());
expect(mockStopSpeechRecordingExternal).toHaveBeenCalledTimes(1);
expect(mockStartSpeechRecordingExternal).not.toHaveBeenCalled();
expect(mockStopSpeechRecordingBrowser).not.toHaveBeenCalled();
});
});

View file

@ -1,5 +1,5 @@
import useSpeechToTextBrowser from './useSpeechToTextBrowser';
import useSpeechToTextExternal from './useSpeechToTextExternal';
import useSpeechToTextBrowser from './useSpeechToTextBrowser';
import useGetAudioSettings from './useGetAudioSettings';
const useSpeechToText = (

View file

@ -3,7 +3,6 @@ import { useRecoilState } from 'recoil';
import { useToastContext } from '@librechat/client';
import { useGetCustomConfigSpeechQuery } from 'librechat-data-provider/react-query';
import SpeechRecognitionImport, { useSpeechRecognition } from 'react-speech-recognition';
import useGetAudioSettings from './useGetAudioSettings';
import { useLocalize } from '~/hooks';
import store from '~/store';
@ -32,8 +31,6 @@ const useSpeechToTextBrowser = (
) => {
const localize = useLocalize();
const { showToast } = useToastContext();
const { speechToTextEndpoint } = useGetAudioSettings();
const isBrowserSTTEnabled = speechToTextEndpoint === 'browser';
const { data: speechConfig } = useGetCustomConfigSpeechQuery({ enabled: true });
const sttExternal = Boolean(speechConfig?.sttExternal);
@ -140,17 +137,6 @@ const useSpeechToTextBrowser = (
sttExternal,
]);
useEffect(() => {
const handleKeyDown = (e: KeyboardEvent) => {
if (e.shiftKey && e.altKey && e.code === 'KeyL' && !isBrowserSTTEnabled) {
toggleListening();
}
};
window.addEventListener('keydown', handleKeyDown);
return () => window.removeEventListener('keydown', handleKeyDown);
}, [isBrowserSTTEnabled, toggleListening]);
return {
isListening,
isLoading: false,

View file

@ -1,8 +1,7 @@
import { useState, useEffect, useRef } from 'react';
import { useState, useRef } from 'react';
import { useRecoilState } from 'recoil';
import { useToastContext } from '@librechat/client';
import { useSpeechToTextMutation } from '~/data-provider';
import useGetAudioSettings from './useGetAudioSettings';
import store from '~/store';
const useSpeechToTextExternal = (
@ -10,15 +9,12 @@ const useSpeechToTextExternal = (
onTranscriptionComplete: (text: string) => void,
) => {
const { showToast } = useToastContext();
const { speechToTextEndpoint } = useGetAudioSettings();
const isExternalSTTEnabled = speechToTextEndpoint === 'external';
const audioStream = useRef<MediaStream | null>(null);
const animationFrameIdRef = useRef<number | null>(null);
const audioContextRef = useRef<AudioContext | null>(null);
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
const audioChunksRef = useRef<Blob[]>([]);
const [permission, setPermission] = useState(false);
const [isListening, setIsListening] = useState(false);
const [isRequestBeingMade, setIsRequestBeingMade] = useState(false);
const [audioMimeType, setAudioMimeType] = useState<string>(() => getBestSupportedMimeType());
@ -102,10 +98,9 @@ const useSpeechToTextExternal = (
audio: true,
video: false,
});
setPermission(true);
audioStream.current = streamData ?? null;
} catch {
setPermission(false);
audioStream.current = null;
}
};
@ -221,6 +216,11 @@ const useSpeechToTextExternal = (
};
const externalStartRecording = () => {
if (typeof MediaRecorder === 'undefined') {
showToast({ message: 'MediaRecorder is not supported in this browser', status: 'error' });
return;
}
if (isListening) {
showToast({ message: 'Already listening. Please stop recording first.', status: 'warning' });
return;
@ -241,36 +241,6 @@ const useSpeechToTextExternal = (
stopRecording();
};
const handleKeyDown = async (e: KeyboardEvent) => {
if (e.shiftKey && e.altKey && e.code === 'KeyL' && isExternalSTTEnabled) {
if (!window.MediaRecorder) {
showToast({ message: 'MediaRecorder is not supported in this browser', status: 'error' });
return;
}
if (permission === false) {
await getMicrophonePermission();
}
if (isListening) {
stopRecording();
} else {
startRecording();
}
e.preventDefault();
}
};
useEffect(() => {
window.addEventListener('keydown', handleKeyDown);
return () => {
window.removeEventListener('keydown', handleKeyDown);
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [isListening]);
return {
isListening,
externalStopRecording,

View file

@ -2,6 +2,24 @@ import { atom } from 'recoil';
import { SettingsViews, LocalStorageKeys } from 'librechat-data-provider';
import type { TOptionSettings } from '~/common';
import { atomWithLocalStorage } from '~/store/utils';
import { STTEndpoints } from '~/common';
const VALID_SPEECH_ENGINES = new Set<string>(Object.values(STTEndpoints));
const LEGACY_EXTERNAL_STT_ENGINES = new Set<string>(['openai', 'azureOpenAI']);
const LEGACY_EXTERNAL_TTS_ENGINES = new Set<string>([
'openai',
'azureOpenAI',
'elevenlabs',
'localai',
]);
const normalizeSavedSpeechEngine = (
engine: string,
legacyExternalEngines: ReadonlySet<string>,
): string => {
if (VALID_SPEECH_ENGINES.has(engine)) return engine;
return legacyExternalEngines.has(engine) ? STTEndpoints.external : STTEndpoints.browser;
};
// Static atoms without localStorage
const staticAtoms = {
@ -12,6 +30,7 @@ const staticAtoms = {
default: SettingsViews.default,
}),
showPopover: atom<boolean>({ key: 'showPopover', default: false }),
speechSettingsInitialized: atom<boolean>({ key: 'speechSettingsInitialized', default: false }),
};
const localStorageAtoms = {
@ -70,14 +89,18 @@ const localStorageAtoms = {
advancedMode: atomWithLocalStorage('advancedMode', false),
speechToText: atomWithLocalStorage('speechToText', true),
engineSTT: atomWithLocalStorage('engineSTT', 'browser'),
engineSTT: atomWithLocalStorage('engineSTT', 'browser', (engine) =>
normalizeSavedSpeechEngine(engine, LEGACY_EXTERNAL_STT_ENGINES),
),
languageSTT: atomWithLocalStorage('languageSTT', ''),
autoTranscribeAudio: atomWithLocalStorage('autoTranscribeAudio', false),
decibelValue: atomWithLocalStorage('decibelValue', -45),
autoSendText: atomWithLocalStorage('autoSendText', -1),
textToSpeech: atomWithLocalStorage('textToSpeech', true),
engineTTS: atomWithLocalStorage('engineTTS', 'browser'),
engineTTS: atomWithLocalStorage('engineTTS', 'browser', (engine) =>
normalizeSavedSpeechEngine(engine, LEGACY_EXTERNAL_TTS_ENGINES),
),
voice: atomWithLocalStorage<string | undefined>('voice', undefined),
cloudBrowserVoices: atomWithLocalStorage('cloudBrowserVoices', false),
languageTTS: atomWithLocalStorage('languageTTS', ''),

View file

@ -1,7 +1,11 @@
import { atom } from 'recoil';
// Improved helper function to create atoms with localStorage
export function atomWithLocalStorage<T>(key: string, defaultValue: T) {
export function atomWithLocalStorage<T>(
key: string,
defaultValue: T,
normalizeSavedValue: (value: T) => T = (value) => value,
) {
return atom<T>({
key,
default: defaultValue,
@ -10,8 +14,12 @@ export function atomWithLocalStorage<T>(key: string, defaultValue: T) {
const savedValue = localStorage.getItem(key);
if (savedValue !== null) {
try {
const parsedValue = JSON.parse(savedValue);
setSelf(parsedValue);
const parsedValue = JSON.parse(savedValue) as T;
const normalizedValue = normalizeSavedValue(parsedValue);
if (!Object.is(normalizedValue, parsedValue)) {
localStorage.setItem(key, JSON.stringify(normalizedValue));
}
setSelf(normalizedValue);
} catch (e) {
console.error(
`Error parsing localStorage key "${key}", \`savedValue\`: defaultValue, error:`,

View file

@ -58,6 +58,50 @@ describe('bedrockEndpointSchema', () => {
});
});
describe('speechTab schema', () => {
it.each(['browser', 'external', 'openai', 'azureOpenAI'])(
'accepts the speech-to-text engine "%s"',
(engineSTT) => {
const result = configSchema.safeParse({
version: '1.0',
speech: { speechTab: { speechToText: { engineSTT } } },
});
expect(result.success).toBe(true);
},
);
it('rejects an unknown speech-to-text engine', () => {
const result = configSchema.safeParse({
version: '1.0',
speech: { speechTab: { speechToText: { engineSTT: 'unknown' } } },
});
expect(result.success).toBe(false);
});
it.each(['browser', 'external', 'openai', 'azureOpenAI', 'elevenlabs', 'localai'])(
'accepts the text-to-speech engine "%s"',
(engineTTS) => {
const result = configSchema.safeParse({
version: '1.0',
speech: { speechTab: { textToSpeech: { engineTTS } } },
});
expect(result.success).toBe(true);
},
);
it('rejects an unknown text-to-speech engine', () => {
const result = configSchema.safeParse({
version: '1.0',
speech: { speechTab: { textToSpeech: { engineTTS: 'unknown' } } },
});
expect(result.success).toBe(false);
});
});
describe('resolveEndpointType', () => {
describe('non-agents endpoints', () => {
it('returns the config type for a custom endpoint', () => {

View file

@ -1291,8 +1291,8 @@ const speechTab = z
.optional()
.or(
z.object({
/** Keep in sync with STTProviders enum (defined below — cannot reference due to eval order) */
engineSTT: z.enum(['openai', 'azureOpenAI']).optional(),
/** Provider names remain valid for backward compatibility and are normalized for clients. */
engineSTT: z.enum(['browser', 'external', 'openai', 'azureOpenAI']).optional(),
languageSTT: z.string().optional(),
autoTranscribeAudio: z.boolean().optional(),
decibelValue: z.number().optional(),
@ -1305,8 +1305,10 @@ const speechTab = z
.optional()
.or(
z.object({
/** Keep in sync with TTSProviders enum (defined below — cannot reference due to eval order) */
engineTTS: z.enum(['openai', 'azureOpenAI', 'elevenlabs', 'localai']).optional(),
/** Provider names remain valid for backward compatibility and are normalized for clients. */
engineTTS: z
.enum(['browser', 'external', 'openai', 'azureOpenAI', 'elevenlabs', 'localai'])
.optional(),
voice: z.string().optional(),
languageTTS: z.string().optional(),
automaticPlayback: z.boolean().optional(),