diff --git a/api/server/services/Files/Audio/getCustomConfigSpeech.js b/api/server/services/Files/Audio/getCustomConfigSpeech.js index 1edca8e188..be3b181060 100644 --- a/api/server/services/Files/Audio/getCustomConfigSpeech.js +++ b/api/server/services/Files/Audio/getCustomConfigSpeech.js @@ -1,6 +1,21 @@ const { logger } = require('@librechat/data-schemas'); const { getAppConfig } = require('~/server/services/Config'); +const LEGACY_EXTERNAL_STT_ENGINES = new Set(['openai', 'azureOpenAI']); +const LEGACY_EXTERNAL_TTS_ENGINES = new Set(['openai', 'azureOpenAI', 'elevenlabs', 'localai']); + +function normalizeSpeechEngine(key, value) { + if (key === 'engineSTT' && LEGACY_EXTERNAL_STT_ENGINES.has(value)) { + return 'external'; + } + + if (key === 'engineTTS' && LEGACY_EXTERNAL_TTS_ENGINES.has(value)) { + return 'external'; + } + + return value; +} + /** * This function retrieves the speechTab settings from the custom configuration * It first fetches the custom configuration @@ -52,7 +67,8 @@ async function getCustomConfigSpeech(req, res) { } else { for (const key in speechTab.speechToText) { if (speechTab.speechToText[key] !== undefined) { - settings[key] = speechTab.speechToText[key]; + const value = speechTab.speechToText[key]; + settings[key] = normalizeSpeechEngine(key, value); } } } @@ -64,7 +80,8 @@ async function getCustomConfigSpeech(req, res) { } else { for (const key in speechTab.textToSpeech) { if (speechTab.textToSpeech[key] !== undefined) { - settings[key] = speechTab.textToSpeech[key]; + const value = speechTab.textToSpeech[key]; + settings[key] = normalizeSpeechEngine(key, value); } } } diff --git a/api/server/services/Files/Audio/getCustomConfigSpeech.spec.js b/api/server/services/Files/Audio/getCustomConfigSpeech.spec.js new file mode 100644 index 0000000000..c255839f66 --- /dev/null +++ b/api/server/services/Files/Audio/getCustomConfigSpeech.spec.js @@ -0,0 +1,95 @@ +jest.mock('@librechat/data-schemas', () => ({ logger: { error: jest.fn() } })); +jest.mock('~/server/services/Config', () => ({ getAppConfig: jest.fn() })); + +const getCustomConfigSpeech = require('./getCustomConfigSpeech'); + +const createResponse = () => { + const res = { + send: jest.fn(), + status: jest.fn(), + }; + res.status.mockReturnValue(res); + return res; +}; + +describe('getCustomConfigSpeech', () => { + it.each(['openai', 'azureOpenAI'])( + 'normalizes the legacy STT provider "%s" to the external engine', + async (engineSTT) => { + const req = { + config: { + speech: { + stt: { openai: {} }, + speechTab: { speechToText: { engineSTT } }, + }, + }, + }; + const res = createResponse(); + + await getCustomConfigSpeech(req, res); + + expect(res.status).toHaveBeenCalledWith(200); + expect(res.send).toHaveBeenCalledWith({ + sttExternal: true, + ttsExternal: false, + engineSTT: 'external', + }); + }, + ); + + it.each(['browser', 'external'])('preserves the runtime STT engine "%s"', async (engineSTT) => { + const req = { + config: { + speech: { + stt: { openai: {} }, + speechTab: { speechToText: { engineSTT } }, + }, + }, + }; + const res = createResponse(); + + await getCustomConfigSpeech(req, res); + + expect(res.send).toHaveBeenCalledWith(expect.objectContaining({ engineSTT })); + }); + + it.each(['openai', 'azureOpenAI', 'elevenlabs', 'localai'])( + 'normalizes the legacy TTS provider "%s" to the external engine', + async (engineTTS) => { + const req = { + config: { + speech: { + tts: { openai: {} }, + speechTab: { textToSpeech: { engineTTS } }, + }, + }, + }; + const res = createResponse(); + + await getCustomConfigSpeech(req, res); + + expect(res.status).toHaveBeenCalledWith(200); + expect(res.send).toHaveBeenCalledWith({ + sttExternal: false, + ttsExternal: true, + engineTTS: 'external', + }); + }, + ); + + it.each(['browser', 'external'])('preserves the runtime TTS engine "%s"', async (engineTTS) => { + const req = { + config: { + speech: { + tts: { openai: {} }, + speechTab: { textToSpeech: { engineTTS } }, + }, + }, + }; + const res = createResponse(); + + await getCustomConfigSpeech(req, res); + + expect(res.send).toHaveBeenCalledWith(expect.objectContaining({ engineTTS })); + }); +}); diff --git a/client/src/components/Chat/Input/AudioRecorder.tsx b/client/src/components/Chat/Input/AudioRecorder.tsx index fa3c2ad6f2..3e5774b46a 100644 --- a/client/src/components/Chat/Input/AudioRecorder.tsx +++ b/client/src/components/Chat/Input/AudioRecorder.tsx @@ -1,5 +1,6 @@ -import { memo, useCallback, useRef } from 'react'; +import { memo, useCallback, useEffect, useRef } from 'react'; import { MicOff } from 'lucide-react'; +import { useRecoilValue } from 'recoil'; import { IconButton, useToastContext, @@ -10,6 +11,7 @@ import { import { useLocalize, useSpeechToText, useGetAudioSettings } from '~/hooks'; import { globalAudioId, type TAskFunction } from '~/common'; import { useChatFormContext } from '~/Providers'; +import store from '~/store'; const isExternalSTT = (speechToTextEndpoint: string) => speechToTextEndpoint === 'external'; export default memo(function AudioRecorder({ @@ -27,6 +29,8 @@ export default memo(function AudioRecorder({ const localize = useLocalize(); const { showToast } = useToastContext(); const { speechToTextEndpoint } = useGetAudioSettings(); + const speechSettingsInitialized = useRecoilValue(store.speechSettingsInitialized); + const recorderDisabled = disabled || !speechSettingsInitialized; const existingTextRef = useRef(''); const isSubmittingRef = useRef(isSubmitting); @@ -85,18 +89,37 @@ export default memo(function AudioRecorder({ onTranscriptionComplete, ); - const handleStartRecording = async () => { + const handleStartRecording = useCallback(() => { existingTextRef.current = getValues('text') || ''; startRecording(); - }; + }, [getValues, startRecording]); - const handleStopRecording = async () => { + const handleStopRecording = useCallback(() => { stopRecording(); /** For browser STT, clear the reference since text was already being updated */ if (!isExternalSTT(speechToTextEndpoint)) { existingTextRef.current = ''; } - }; + }, [speechToTextEndpoint, stopRecording]); + + useEffect(() => { + const handleKeyDown = (event: KeyboardEvent) => { + if (!event.shiftKey || !event.altKey || event.code !== 'KeyL' || recorderDisabled) { + return; + } + + event.preventDefault(); + if (isListening === true) { + handleStopRecording(); + return; + } + + handleStartRecording(); + }; + + window.addEventListener('keydown', handleKeyDown); + return () => window.removeEventListener('keydown', handleKeyDown); + }, [handleStartRecording, handleStopRecording, isListening, recorderDisabled]); const renderIcon = () => { if (isListening === true) { @@ -120,7 +143,7 @@ export default memo(function AudioRecorder({ shape="theme" label={localize('com_ui_use_micrphone')} onClick={isListening === true ? handleStopRecording : handleStartRecording} - disabled={disabled} + disabled={recorderDisabled} className="p-1 hover:bg-surface-composer-hover" aria-pressed={isListening} > diff --git a/client/src/components/Chat/Input/__tests__/AudioRecorder.spec.tsx b/client/src/components/Chat/Input/__tests__/AudioRecorder.spec.tsx new file mode 100644 index 0000000000..96c86d2ff2 --- /dev/null +++ b/client/src/components/Chat/Input/__tests__/AudioRecorder.spec.tsx @@ -0,0 +1,181 @@ +import React from 'react'; +import { RecoilRoot } from 'recoil'; +import { act, render, screen } from '@testing-library/react'; +import AudioRecorder from '../AudioRecorder'; +import store from '~/store'; + +let mockSpeechToTextEndpoint = 'browser'; +let mockBrowserIsListening = false; +let mockExternalIsListening = false; +let mockSetText: ((text: string) => void) | undefined; + +const mockStartSpeechRecordingBrowser = jest.fn(); +const mockStopSpeechRecordingBrowser = jest.fn(); +const mockStartSpeechRecordingExternal = jest.fn(); +const mockStopSpeechRecordingExternal = jest.fn(); +const mockSetValue = jest.fn(); +const mockReset = jest.fn(); +const mockGetValues = jest.fn(() => 'existing draft'); +const mockAsk = jest.fn(() => true); + +type MockButtonProps = React.ComponentProps<'button'> & { + label?: string; + variant?: string; + size?: string; + shape?: string; +}; + +jest.mock('@librechat/client', () => ({ + IconButton: ({ + children, + label, + variant: _variant, + size: _size, + shape: _shape, + ...props + }: MockButtonProps) => ( + + ), + TooltipAnchor: ({ render }: { render: React.ReactElement }) => render, + ListeningIcon: () => , + Spinner: () => , + useToastContext: () => ({ showToast: jest.fn() }), +})); + +jest.mock('~/hooks/Input/useSpeechToTextBrowser', () => ({ + __esModule: true, + default: (setText: (text: string) => void) => { + mockSetText = setText; + return { + isListening: mockBrowserIsListening, + isLoading: false, + startRecording: mockStartSpeechRecordingBrowser, + stopRecording: mockStopSpeechRecordingBrowser, + }; + }, +})); + +jest.mock('~/hooks/Input/useSpeechToTextExternal', () => ({ + __esModule: true, + default: (setText: (text: string) => void) => { + mockSetText = setText; + return { + isListening: mockExternalIsListening, + isLoading: false, + externalStartRecording: mockStartSpeechRecordingExternal, + externalStopRecording: mockStopSpeechRecordingExternal, + }; + }, +})); + +jest.mock('~/hooks/Input/useGetAudioSettings', () => ({ + __esModule: true, + default: () => ({ speechToTextEndpoint: mockSpeechToTextEndpoint }), +})); + +jest.mock('~/hooks', () => ({ + useLocalize: () => (key: string) => key, + useGetAudioSettings: () => ({ speechToTextEndpoint: mockSpeechToTextEndpoint }), + useSpeechToText: jest.requireActual('~/hooks/Input/useSpeechToText').default, +})); + +jest.mock('~/common', () => ({ + ...jest.requireActual('~/common'), + globalAudioId: 'global-audio', +})); + +const dispatchSpeechShortcut = () => { + const event = new KeyboardEvent('keydown', { + shiftKey: true, + altKey: true, + code: 'KeyL', + bubbles: true, + cancelable: true, + }); + + act(() => window.dispatchEvent(event)); + return event; +}; + +const renderRecorder = ({ disabled = false, initialized = true } = {}) => + render( + set(store.speechSettingsInitialized, initialized)}> + + , + ); + +describe('AudioRecorder speech shortcut', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockSpeechToTextEndpoint = 'browser'; + mockBrowserIsListening = false; + mockExternalIsListening = false; + mockSetText = undefined; + }); + + it('preserves the existing draft when browser recording starts from the shortcut', () => { + renderRecorder(); + + const event = dispatchSpeechShortcut(); + act(() => mockSetText?.('transcript')); + + expect(mockStartSpeechRecordingBrowser).toHaveBeenCalledTimes(1); + expect(mockSetValue).toHaveBeenCalledWith('text', 'existing draft transcript', { + shouldValidate: true, + }); + expect(event.defaultPrevented).toBe(true); + }); + + it('does not invoke either recorder while the control is disabled', () => { + renderRecorder({ disabled: true }); + + dispatchSpeechShortcut(); + + expect(mockStartSpeechRecordingBrowser).not.toHaveBeenCalled(); + expect(mockStopSpeechRecordingBrowser).not.toHaveBeenCalled(); + }); + + it('does not start browser recording before speech settings initialization completes', () => { + renderRecorder({ initialized: false }); + + dispatchSpeechShortcut(); + + expect(mockStartSpeechRecordingBrowser).not.toHaveBeenCalled(); + expect(mockStartSpeechRecordingExternal).not.toHaveBeenCalled(); + expect(screen.getByRole('button', { name: 'com_ui_use_micrphone' })).toBeDisabled(); + }); + + it('stops the selected engine from the shortcut', () => { + mockSpeechToTextEndpoint = 'external'; + mockExternalIsListening = true; + renderRecorder(); + + dispatchSpeechShortcut(); + + expect(mockStopSpeechRecordingExternal).toHaveBeenCalledTimes(1); + expect(mockStopSpeechRecordingBrowser).not.toHaveBeenCalled(); + }); + + it('does not stop a recorder while the control is disabled', () => { + mockSpeechToTextEndpoint = 'external'; + mockExternalIsListening = true; + renderRecorder({ disabled: true }); + + dispatchSpeechShortcut(); + + expect(mockStopSpeechRecordingExternal).not.toHaveBeenCalled(); + }); +}); diff --git a/client/src/components/Chat/Messages/MessageAudio.tsx b/client/src/components/Chat/Messages/MessageAudio.tsx index eb4c52a407..d1a38f2e07 100644 --- a/client/src/components/Chat/Messages/MessageAudio.tsx +++ b/client/src/components/Chat/Messages/MessageAudio.tsx @@ -7,6 +7,11 @@ import store from '~/store'; function MessageAudio(props: TMessageAudio) { const engineTTS = useRecoilValue(store.engineTTS); + const speechSettingsInitialized = useRecoilValue(store.speechSettingsInitialized); + + if (!speechSettingsInitialized) { + return null; + } const TTSComponents = { [TTSEndpoints.browser]: BrowserTTS, diff --git a/client/src/components/Chat/Messages/__tests__/MessageAudio.spec.tsx b/client/src/components/Chat/Messages/__tests__/MessageAudio.spec.tsx new file mode 100644 index 0000000000..f791c76c83 --- /dev/null +++ b/client/src/components/Chat/Messages/__tests__/MessageAudio.spec.tsx @@ -0,0 +1,44 @@ +import React from 'react'; +import { RecoilRoot } from 'recoil'; +import { render, screen } from '@testing-library/react'; +import MessageAudio from '../MessageAudio'; +import store from '~/store'; + +jest.mock('~/components/Audio/TTS', () => ({ + BrowserTTS: () =>
, + ExternalTTS: () =>
, +})); + +const renderMessageAudio = ({ + engineTTS, + initialized, +}: { + engineTTS: 'browser' | 'external'; + initialized: boolean; +}) => + render( + { + set(store.engineTTS, engineTTS); + set(store.speechSettingsInitialized, initialized); + }} + > + + , + ); + +describe('MessageAudio speech settings initialization', () => { + it('does not mount browser TTS before speech settings initialize', () => { + renderMessageAudio({ engineTTS: 'browser', initialized: false }); + + expect(screen.queryByTestId('browser-tts')).not.toBeInTheDocument(); + expect(screen.queryByTestId('external-tts')).not.toBeInTheDocument(); + }); + + it('mounts external TTS after its configured engine initializes', () => { + renderMessageAudio({ engineTTS: 'external', initialized: true }); + + expect(screen.getByTestId('external-tts')).toBeInTheDocument(); + expect(screen.queryByTestId('browser-tts')).not.toBeInTheDocument(); + }); +}); diff --git a/client/src/hooks/Config/__tests__/useSpeechSettingsInit.spec.tsx b/client/src/hooks/Config/__tests__/useSpeechSettingsInit.spec.tsx new file mode 100644 index 0000000000..f228a816e8 --- /dev/null +++ b/client/src/hooks/Config/__tests__/useSpeechSettingsInit.spec.tsx @@ -0,0 +1,252 @@ +import React from 'react'; +import { RecoilRoot, useRecoilValue } from 'recoil'; +import { renderHook, waitFor } from '@testing-library/react'; + +const mockUseGetCustomConfigSpeechQuery = jest.fn(); + +jest.mock('librechat-data-provider/react-query', () => ({ + useGetCustomConfigSpeechQuery: () => mockUseGetCustomConfigSpeechQuery(), +})); + +jest.mock('~/utils', () => ({ + logger: { log: jest.fn() }, +})); + +import useSpeechSettingsInit from '../useSpeechSettingsInit'; +import store from '~/store'; + +const wrapper = ({ children }: { children: React.ReactNode }) => ( + {children} +); + +const useSpeechSettingsHarness = (isAuthenticated = true) => { + useSpeechSettingsInit(isAuthenticated); + return { + engineSTT: useRecoilValue(store.engineSTT), + engineTTS: useRecoilValue(store.engineTTS), + speechSettingsInitialized: useRecoilValue(store.speechSettingsInitialized), + }; +}; + +describe('useSpeechSettingsInit', () => { + beforeEach(() => { + localStorage.clear(); + mockUseGetCustomConfigSpeechQuery.mockReturnValue({ + data: { sttExternal: true, ttsExternal: true }, + isFetched: true, + }); + }); + + it.each(['openai', 'azureOpenAI'])( + 'migrates the persisted STT provider "%s" to the external engine', + async (engineSTT) => { + localStorage.setItem('engineSTT', JSON.stringify(engineSTT)); + + const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper }); + + await waitFor(() => expect(result.current.engineSTT).toBe('external')); + expect(localStorage.getItem('engineSTT')).toBe(JSON.stringify('external')); + }, + ); + + it.each(['openai', 'azureOpenAI', 'elevenlabs', 'localai'])( + 'migrates the persisted TTS provider "%s" to the external engine', + async (engineTTS) => { + localStorage.setItem('engineTTS', JSON.stringify(engineTTS)); + + const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper }); + + await waitFor(() => expect(result.current.engineTTS).toBe('external')); + expect(localStorage.getItem('engineTTS')).toBe(JSON.stringify('external')); + }, + ); + + it('falls back migrated providers when no external provider is reported', async () => { + localStorage.setItem('engineSTT', JSON.stringify('openai')); + localStorage.setItem('engineTTS', JSON.stringify('elevenlabs')); + mockUseGetCustomConfigSpeechQuery.mockReturnValue({ + data: { sttExternal: false, ttsExternal: false }, + isFetched: true, + }); + + const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper }); + + await waitFor(() => { + expect(result.current).toEqual({ + engineSTT: 'browser', + engineTTS: 'browser', + speechSettingsInitialized: true, + }); + }); + expect(localStorage.getItem('engineSTT')).toBe(JSON.stringify('browser')); + expect(localStorage.getItem('engineTTS')).toBe(JSON.stringify('browser')); + }); + + it('falls back saved external engines when their providers are unavailable', async () => { + localStorage.setItem('engineSTT', JSON.stringify('external')); + localStorage.setItem('engineTTS', JSON.stringify('external')); + mockUseGetCustomConfigSpeechQuery.mockReturnValue({ + data: { sttExternal: false, ttsExternal: false }, + isFetched: true, + }); + + const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper }); + + await waitFor(() => { + expect(result.current).toEqual({ + engineSTT: 'browser', + engineTTS: 'browser', + speechSettingsInitialized: true, + }); + }); + expect(localStorage.getItem('engineSTT')).toBe(JSON.stringify('browser')); + expect(localStorage.getItem('engineTTS')).toBe(JSON.stringify('browser')); + }); + + it.each(['browser', 'external'])('preserves the valid engine "%s"', async (engine) => { + localStorage.setItem('engineSTT', JSON.stringify(engine)); + localStorage.setItem('engineTTS', JSON.stringify(engine)); + + const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper }); + + await waitFor(() => { + expect(result.current).toEqual({ + engineSTT: engine, + engineTTS: engine, + speechSettingsInitialized: true, + }); + }); + }); + + it.each([ + ['loading', { data: undefined, isFetched: false, isError: false }, false], + ['error', { data: undefined, isFetched: true, isError: true }, true], + ])( + 'normalizes legacy providers while the speech configuration query is in the %s state', + async (_state, queryResult, speechSettingsInitialized) => { + localStorage.setItem('engineSTT', JSON.stringify('openai')); + localStorage.setItem('engineTTS', JSON.stringify('localai')); + mockUseGetCustomConfigSpeechQuery.mockReturnValue(queryResult); + + const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper }); + + await waitFor(() => { + expect(result.current).toEqual({ + engineSTT: 'external', + engineTTS: 'external', + speechSettingsInitialized, + }); + }); + expect(localStorage.getItem('engineSTT')).toBe(JSON.stringify('external')); + expect(localStorage.getItem('engineTTS')).toBe(JSON.stringify('external')); + }, + ); + + it('normalizes unknown saved engines to browser', async () => { + localStorage.setItem('engineSTT', JSON.stringify('unknown-stt')); + localStorage.setItem('engineTTS', JSON.stringify('unknown-tts')); + + const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper }); + + await waitFor(() => { + expect(result.current).toEqual({ + engineSTT: 'browser', + engineTTS: 'browser', + speechSettingsInitialized: true, + }); + }); + expect(localStorage.getItem('engineSTT')).toBe(JSON.stringify('browser')); + expect(localStorage.getItem('engineTTS')).toBe(JSON.stringify('browser')); + }); + + it('seeds external engines from the server configuration when storage is empty', async () => { + mockUseGetCustomConfigSpeechQuery.mockReturnValue({ + data: { + sttExternal: true, + ttsExternal: true, + engineSTT: 'external', + engineTTS: 'external', + }, + isFetched: true, + }); + + const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper }); + + await waitFor(() => { + expect(result.current).toEqual({ + engineSTT: 'external', + engineTTS: 'external', + speechSettingsInitialized: true, + }); + }); + expect(localStorage.getItem('engineSTT')).toBe(JSON.stringify('external')); + expect(localStorage.getItem('engineTTS')).toBe(JSON.stringify('external')); + }); + + it('does not write browser defaults before a server default is provided', () => { + mockUseGetCustomConfigSpeechQuery.mockReturnValue({ + data: { sttExternal: false, ttsExternal: false }, + isFetched: true, + }); + + const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper }); + + expect(result.current).toEqual({ + engineSTT: 'browser', + engineTTS: 'browser', + speechSettingsInitialized: true, + }); + expect(localStorage.getItem('engineSTT')).toBeNull(); + expect(localStorage.getItem('engineTTS')).toBeNull(); + }); + + it('keeps speech controls disabled until configuration initialization settles', () => { + mockUseGetCustomConfigSpeechQuery.mockReturnValue({ + data: undefined, + isFetched: false, + }); + + const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper }); + + expect(result.current.speechSettingsInitialized).toBe(false); + }); + + it('enables speech controls after a missing configuration response', async () => { + mockUseGetCustomConfigSpeechQuery.mockReturnValue({ + data: { message: 'not_found' }, + isFetched: true, + }); + + const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper }); + + await waitFor(() => expect(result.current.speechSettingsInitialized).toBe(true)); + }); + + it('keeps speech controls disabled after a configuration error on fresh storage', () => { + mockUseGetCustomConfigSpeechQuery.mockReturnValue({ + data: undefined, + isFetched: true, + isError: true, + }); + + const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper }); + + expect(result.current.speechSettingsInitialized).toBe(false); + }); + + it.each(['engineSTT', 'engineTTS'])( + 'keeps speech controls disabled after an error when only %s is saved', + (savedEngine) => { + localStorage.setItem(savedEngine, JSON.stringify('browser')); + mockUseGetCustomConfigSpeechQuery.mockReturnValue({ + data: undefined, + isFetched: true, + isError: true, + }); + + const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper }); + + expect(result.current.speechSettingsInitialized).toBe(false); + }, + ); +}); diff --git a/client/src/hooks/Config/useSpeechSettingsInit.ts b/client/src/hooks/Config/useSpeechSettingsInit.ts index 9dab2c9e0d..5019f4d813 100644 --- a/client/src/hooks/Config/useSpeechSettingsInit.ts +++ b/client/src/hooks/Config/useSpeechSettingsInit.ts @@ -1,7 +1,7 @@ import { useEffect, useRef } from 'react'; import { useRecoilState, useSetRecoilState } from 'recoil'; import { useGetCustomConfigSpeechQuery } from 'librechat-data-provider/react-query'; -import { TTSEndpoints } from '~/common'; +import { STTEndpoints, TTSEndpoints } from '~/common'; import { logger } from '~/utils'; import store from '~/store'; @@ -12,8 +12,10 @@ const VALID_TTS_ENGINES: string[] = [TTSEndpoints.browser, TTSEndpoints.external * configuration on first load (only when the user is authenticated) */ export default function useSpeechSettingsInit(isAuthenticated: boolean) { - const { data } = useGetCustomConfigSpeechQuery({ enabled: isAuthenticated }); + const { data, isError, isFetched } = useGetCustomConfigSpeechQuery({ enabled: isAuthenticated }); + const [engineSTT, setEngineSTT] = useRecoilState(store.engineSTT); const [engineTTS, setEngineTTS] = useRecoilState(store.engineTTS); + const setSpeechSettingsInitialized = useSetRecoilState(store.speechSettingsInitialized); const setters = useRef({ conversationMode: useSetRecoilState(store.conversationMode), @@ -21,7 +23,7 @@ export default function useSpeechSettingsInit(isAuthenticated: boolean) { speechToText: useSetRecoilState(store.speechToText), textToSpeech: useSetRecoilState(store.textToSpeech), cacheTTS: useSetRecoilState(store.cacheTTS), - engineSTT: useSetRecoilState(store.engineSTT), + engineSTT: setEngineSTT, languageSTT: useSetRecoilState(store.languageSTT), autoTranscribeAudio: useSetRecoilState(store.autoTranscribeAudio), decibelValue: useSetRecoilState(store.decibelValue), @@ -35,22 +37,65 @@ export default function useSpeechSettingsInit(isAuthenticated: boolean) { }).current; useEffect(() => { - if (!isAuthenticated || !data || data.message === 'not_found') return; + if (!isAuthenticated) { + setSpeechSettingsInitialized(false); + return; + } - logger.log('Initializing speech settings from config:', data); + if (!isFetched) return; - Object.entries(data).forEach(([key, value]) => { - if (key === 'sttExternal' || key === 'ttsExternal') return; + if ( + isError && + (localStorage.getItem('engineSTT') === null || localStorage.getItem('engineTTS') === null) + ) { + setSpeechSettingsInitialized(false); + return; + } - if (localStorage.getItem(key) !== null) return; + const hasSavedEngineSTT = localStorage.getItem('engineSTT') !== null; + const hasSavedEngineTTS = localStorage.getItem('engineTTS') !== null; - const setter = setters[key as keyof typeof setters]; - if (setter) { - logger.log(`Setting default speech setting: ${key} = ${value}`); - setter(value as any); - } - }); - }, [isAuthenticated, data, setters]); + if (data && data.message !== 'not_found') { + logger.log('Initializing speech settings from config:', data); + + Object.entries(data).forEach(([key, value]) => { + if (key === 'sttExternal' || key === 'ttsExternal') return; + + if (localStorage.getItem(key) !== null) return; + + const setter = setters[key as keyof typeof setters]; + if (setter) { + logger.log(`Setting default speech setting: ${key} = ${value}`); + setter(value as any); + } + }); + } + + const configuredEngineSTT = hasSavedEngineSTT ? engineSTT : data?.engineSTT; + const configuredEngineTTS = hasSavedEngineTTS ? engineTTS : data?.engineTTS; + const sttExternalUnavailable = data?.sttExternal != null && !data.sttExternal; + const ttsExternalUnavailable = data?.ttsExternal != null && !data.ttsExternal; + + if (sttExternalUnavailable && configuredEngineSTT === STTEndpoints.external) { + setEngineSTT(STTEndpoints.browser); + } + if (ttsExternalUnavailable && configuredEngineTTS === TTSEndpoints.external) { + setEngineTTS(TTSEndpoints.browser); + } + + setSpeechSettingsInitialized(true); + }, [ + data, + engineSTT, + engineTTS, + isAuthenticated, + isError, + isFetched, + setEngineSTT, + setEngineTTS, + setSpeechSettingsInitialized, + setters, + ]); useEffect(() => { if (VALID_TTS_ENGINES.includes(engineTTS)) return; diff --git a/client/src/hooks/Input/useSpeechToText.spec.ts b/client/src/hooks/Input/useSpeechToText.spec.ts new file mode 100644 index 0000000000..34eb38f04a --- /dev/null +++ b/client/src/hooks/Input/useSpeechToText.spec.ts @@ -0,0 +1,79 @@ +import { act, renderHook } from '@testing-library/react'; +import useSpeechToText from './useSpeechToText'; + +let mockSpeechToTextEndpoint = 'external'; +let mockBrowserIsListening = false; +let mockExternalIsListening = false; + +const mockStartSpeechRecordingBrowser = jest.fn(); +const mockStopSpeechRecordingBrowser = jest.fn(); +const mockStartSpeechRecordingExternal = jest.fn(); +const mockStopSpeechRecordingExternal = jest.fn(); + +jest.mock('./useGetAudioSettings', () => ({ + __esModule: true, + default: () => ({ speechToTextEndpoint: mockSpeechToTextEndpoint }), +})); + +jest.mock('./useSpeechToTextBrowser', () => ({ + __esModule: true, + default: () => ({ + isListening: mockBrowserIsListening, + isLoading: false, + startRecording: mockStartSpeechRecordingBrowser, + stopRecording: mockStopSpeechRecordingBrowser, + }), +})); + +jest.mock('./useSpeechToTextExternal', () => ({ + __esModule: true, + default: () => ({ + isListening: mockExternalIsListening, + isLoading: false, + externalStartRecording: mockStartSpeechRecordingExternal, + externalStopRecording: mockStopSpeechRecordingExternal, + }), +})); + +describe('useSpeechToText', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockSpeechToTextEndpoint = 'external'; + mockBrowserIsListening = false; + mockExternalIsListening = false; + }); + + it('selects the externally seeded engine after settings change', () => { + mockSpeechToTextEndpoint = 'browser'; + const { result, rerender } = renderHook(() => useSpeechToText(jest.fn(), jest.fn())); + + mockSpeechToTextEndpoint = 'external'; + rerender(); + + act(() => result.current.startRecording()); + + expect(mockStartSpeechRecordingExternal).toHaveBeenCalledTimes(1); + expect(mockStartSpeechRecordingBrowser).not.toHaveBeenCalled(); + }); + + it('selects the browser engine', () => { + mockSpeechToTextEndpoint = 'browser'; + const { result } = renderHook(() => useSpeechToText(jest.fn(), jest.fn())); + + act(() => result.current.startRecording()); + + expect(mockStartSpeechRecordingBrowser).toHaveBeenCalledTimes(1); + expect(mockStartSpeechRecordingExternal).not.toHaveBeenCalled(); + }); + + it('selects the active engine stop handler', () => { + mockExternalIsListening = true; + const { result } = renderHook(() => useSpeechToText(jest.fn(), jest.fn())); + + act(() => result.current.stopRecording()); + + expect(mockStopSpeechRecordingExternal).toHaveBeenCalledTimes(1); + expect(mockStartSpeechRecordingExternal).not.toHaveBeenCalled(); + expect(mockStopSpeechRecordingBrowser).not.toHaveBeenCalled(); + }); +}); diff --git a/client/src/hooks/Input/useSpeechToText.ts b/client/src/hooks/Input/useSpeechToText.ts index 705b870dc6..f149b72af8 100644 --- a/client/src/hooks/Input/useSpeechToText.ts +++ b/client/src/hooks/Input/useSpeechToText.ts @@ -1,5 +1,5 @@ -import useSpeechToTextBrowser from './useSpeechToTextBrowser'; import useSpeechToTextExternal from './useSpeechToTextExternal'; +import useSpeechToTextBrowser from './useSpeechToTextBrowser'; import useGetAudioSettings from './useGetAudioSettings'; const useSpeechToText = ( diff --git a/client/src/hooks/Input/useSpeechToTextBrowser.ts b/client/src/hooks/Input/useSpeechToTextBrowser.ts index 5053e31d54..a72403a239 100644 --- a/client/src/hooks/Input/useSpeechToTextBrowser.ts +++ b/client/src/hooks/Input/useSpeechToTextBrowser.ts @@ -3,7 +3,6 @@ import { useRecoilState } from 'recoil'; import { useToastContext } from '@librechat/client'; import { useGetCustomConfigSpeechQuery } from 'librechat-data-provider/react-query'; import SpeechRecognitionImport, { useSpeechRecognition } from 'react-speech-recognition'; -import useGetAudioSettings from './useGetAudioSettings'; import { useLocalize } from '~/hooks'; import store from '~/store'; @@ -32,8 +31,6 @@ const useSpeechToTextBrowser = ( ) => { const localize = useLocalize(); const { showToast } = useToastContext(); - const { speechToTextEndpoint } = useGetAudioSettings(); - const isBrowserSTTEnabled = speechToTextEndpoint === 'browser'; const { data: speechConfig } = useGetCustomConfigSpeechQuery({ enabled: true }); const sttExternal = Boolean(speechConfig?.sttExternal); @@ -140,17 +137,6 @@ const useSpeechToTextBrowser = ( sttExternal, ]); - useEffect(() => { - const handleKeyDown = (e: KeyboardEvent) => { - if (e.shiftKey && e.altKey && e.code === 'KeyL' && !isBrowserSTTEnabled) { - toggleListening(); - } - }; - - window.addEventListener('keydown', handleKeyDown); - return () => window.removeEventListener('keydown', handleKeyDown); - }, [isBrowserSTTEnabled, toggleListening]); - return { isListening, isLoading: false, diff --git a/client/src/hooks/Input/useSpeechToTextExternal.ts b/client/src/hooks/Input/useSpeechToTextExternal.ts index bbe2d188a2..8876fffb75 100644 --- a/client/src/hooks/Input/useSpeechToTextExternal.ts +++ b/client/src/hooks/Input/useSpeechToTextExternal.ts @@ -1,8 +1,7 @@ -import { useState, useEffect, useRef } from 'react'; +import { useState, useRef } from 'react'; import { useRecoilState } from 'recoil'; import { useToastContext } from '@librechat/client'; import { useSpeechToTextMutation } from '~/data-provider'; -import useGetAudioSettings from './useGetAudioSettings'; import store from '~/store'; const useSpeechToTextExternal = ( @@ -10,15 +9,12 @@ const useSpeechToTextExternal = ( onTranscriptionComplete: (text: string) => void, ) => { const { showToast } = useToastContext(); - const { speechToTextEndpoint } = useGetAudioSettings(); - const isExternalSTTEnabled = speechToTextEndpoint === 'external'; const audioStream = useRef(null); const animationFrameIdRef = useRef(null); const audioContextRef = useRef(null); const mediaRecorderRef = useRef(null); const audioChunksRef = useRef([]); - const [permission, setPermission] = useState(false); const [isListening, setIsListening] = useState(false); const [isRequestBeingMade, setIsRequestBeingMade] = useState(false); const [audioMimeType, setAudioMimeType] = useState(() => getBestSupportedMimeType()); @@ -102,10 +98,9 @@ const useSpeechToTextExternal = ( audio: true, video: false, }); - setPermission(true); audioStream.current = streamData ?? null; } catch { - setPermission(false); + audioStream.current = null; } }; @@ -221,6 +216,11 @@ const useSpeechToTextExternal = ( }; const externalStartRecording = () => { + if (typeof MediaRecorder === 'undefined') { + showToast({ message: 'MediaRecorder is not supported in this browser', status: 'error' }); + return; + } + if (isListening) { showToast({ message: 'Already listening. Please stop recording first.', status: 'warning' }); return; @@ -241,36 +241,6 @@ const useSpeechToTextExternal = ( stopRecording(); }; - const handleKeyDown = async (e: KeyboardEvent) => { - if (e.shiftKey && e.altKey && e.code === 'KeyL' && isExternalSTTEnabled) { - if (!window.MediaRecorder) { - showToast({ message: 'MediaRecorder is not supported in this browser', status: 'error' }); - return; - } - - if (permission === false) { - await getMicrophonePermission(); - } - - if (isListening) { - stopRecording(); - } else { - startRecording(); - } - - e.preventDefault(); - } - }; - - useEffect(() => { - window.addEventListener('keydown', handleKeyDown); - - return () => { - window.removeEventListener('keydown', handleKeyDown); - }; - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [isListening]); - return { isListening, externalStopRecording, diff --git a/client/src/store/settings.ts b/client/src/store/settings.ts index 9e60b9b418..7a3c2d1397 100644 --- a/client/src/store/settings.ts +++ b/client/src/store/settings.ts @@ -2,6 +2,24 @@ import { atom } from 'recoil'; import { SettingsViews, LocalStorageKeys } from 'librechat-data-provider'; import type { TOptionSettings } from '~/common'; import { atomWithLocalStorage } from '~/store/utils'; +import { STTEndpoints } from '~/common'; + +const VALID_SPEECH_ENGINES = new Set(Object.values(STTEndpoints)); +const LEGACY_EXTERNAL_STT_ENGINES = new Set(['openai', 'azureOpenAI']); +const LEGACY_EXTERNAL_TTS_ENGINES = new Set([ + 'openai', + 'azureOpenAI', + 'elevenlabs', + 'localai', +]); + +const normalizeSavedSpeechEngine = ( + engine: string, + legacyExternalEngines: ReadonlySet, +): string => { + if (VALID_SPEECH_ENGINES.has(engine)) return engine; + return legacyExternalEngines.has(engine) ? STTEndpoints.external : STTEndpoints.browser; +}; // Static atoms without localStorage const staticAtoms = { @@ -12,6 +30,7 @@ const staticAtoms = { default: SettingsViews.default, }), showPopover: atom({ key: 'showPopover', default: false }), + speechSettingsInitialized: atom({ key: 'speechSettingsInitialized', default: false }), }; const localStorageAtoms = { @@ -70,14 +89,18 @@ const localStorageAtoms = { advancedMode: atomWithLocalStorage('advancedMode', false), speechToText: atomWithLocalStorage('speechToText', true), - engineSTT: atomWithLocalStorage('engineSTT', 'browser'), + engineSTT: atomWithLocalStorage('engineSTT', 'browser', (engine) => + normalizeSavedSpeechEngine(engine, LEGACY_EXTERNAL_STT_ENGINES), + ), languageSTT: atomWithLocalStorage('languageSTT', ''), autoTranscribeAudio: atomWithLocalStorage('autoTranscribeAudio', false), decibelValue: atomWithLocalStorage('decibelValue', -45), autoSendText: atomWithLocalStorage('autoSendText', -1), textToSpeech: atomWithLocalStorage('textToSpeech', true), - engineTTS: atomWithLocalStorage('engineTTS', 'browser'), + engineTTS: atomWithLocalStorage('engineTTS', 'browser', (engine) => + normalizeSavedSpeechEngine(engine, LEGACY_EXTERNAL_TTS_ENGINES), + ), voice: atomWithLocalStorage('voice', undefined), cloudBrowserVoices: atomWithLocalStorage('cloudBrowserVoices', false), languageTTS: atomWithLocalStorage('languageTTS', ''), diff --git a/client/src/store/utils.ts b/client/src/store/utils.ts index bf2b7a2389..5551549513 100644 --- a/client/src/store/utils.ts +++ b/client/src/store/utils.ts @@ -1,7 +1,11 @@ import { atom } from 'recoil'; // Improved helper function to create atoms with localStorage -export function atomWithLocalStorage(key: string, defaultValue: T) { +export function atomWithLocalStorage( + key: string, + defaultValue: T, + normalizeSavedValue: (value: T) => T = (value) => value, +) { return atom({ key, default: defaultValue, @@ -10,8 +14,12 @@ export function atomWithLocalStorage(key: string, defaultValue: T) { const savedValue = localStorage.getItem(key); if (savedValue !== null) { try { - const parsedValue = JSON.parse(savedValue); - setSelf(parsedValue); + const parsedValue = JSON.parse(savedValue) as T; + const normalizedValue = normalizeSavedValue(parsedValue); + if (!Object.is(normalizedValue, parsedValue)) { + localStorage.setItem(key, JSON.stringify(normalizedValue)); + } + setSelf(normalizedValue); } catch (e) { console.error( `Error parsing localStorage key "${key}", \`savedValue\`: defaultValue, error:`, diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts index ddc971c770..7427a551ea 100644 --- a/packages/data-provider/src/config.spec.ts +++ b/packages/data-provider/src/config.spec.ts @@ -58,6 +58,50 @@ describe('bedrockEndpointSchema', () => { }); }); +describe('speechTab schema', () => { + it.each(['browser', 'external', 'openai', 'azureOpenAI'])( + 'accepts the speech-to-text engine "%s"', + (engineSTT) => { + const result = configSchema.safeParse({ + version: '1.0', + speech: { speechTab: { speechToText: { engineSTT } } }, + }); + + expect(result.success).toBe(true); + }, + ); + + it('rejects an unknown speech-to-text engine', () => { + const result = configSchema.safeParse({ + version: '1.0', + speech: { speechTab: { speechToText: { engineSTT: 'unknown' } } }, + }); + + expect(result.success).toBe(false); + }); + + it.each(['browser', 'external', 'openai', 'azureOpenAI', 'elevenlabs', 'localai'])( + 'accepts the text-to-speech engine "%s"', + (engineTTS) => { + const result = configSchema.safeParse({ + version: '1.0', + speech: { speechTab: { textToSpeech: { engineTTS } } }, + }); + + expect(result.success).toBe(true); + }, + ); + + it('rejects an unknown text-to-speech engine', () => { + const result = configSchema.safeParse({ + version: '1.0', + speech: { speechTab: { textToSpeech: { engineTTS: 'unknown' } } }, + }); + + expect(result.success).toBe(false); + }); +}); + describe('resolveEndpointType', () => { describe('non-agents endpoints', () => { it('returns the config type for a custom endpoint', () => { diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 5da9c8ca89..4b2ed75743 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -1291,8 +1291,8 @@ const speechTab = z .optional() .or( z.object({ - /** Keep in sync with STTProviders enum (defined below — cannot reference due to eval order) */ - engineSTT: z.enum(['openai', 'azureOpenAI']).optional(), + /** Provider names remain valid for backward compatibility and are normalized for clients. */ + engineSTT: z.enum(['browser', 'external', 'openai', 'azureOpenAI']).optional(), languageSTT: z.string().optional(), autoTranscribeAudio: z.boolean().optional(), decibelValue: z.number().optional(), @@ -1305,8 +1305,10 @@ const speechTab = z .optional() .or( z.object({ - /** Keep in sync with TTSProviders enum (defined below — cannot reference due to eval order) */ - engineTTS: z.enum(['openai', 'azureOpenAI', 'elevenlabs', 'localai']).optional(), + /** Provider names remain valid for backward compatibility and are normalized for clients. */ + engineTTS: z + .enum(['browser', 'external', 'openai', 'azureOpenAI', 'elevenlabs', 'localai']) + .optional(), voice: z.string().optional(), languageTTS: z.string().optional(), automaticPlayback: z.boolean().optional(),