diff --git a/api/cache/redis.js b/api/cache/redis.js index c9bfdd6cae..adf291d02b 100644 --- a/api/cache/redis.js +++ b/api/cache/redis.js @@ -1,9 +1,4 @@ const Redis = require('ioredis'); -const { logger } = require('~/config'); const { REDIS_URI } = process.env ?? {}; -const redis = new Redis(REDIS_URI); -redis - .on('error', (err) => logger.error('ioredis error:', err)) - .on('ready', () => logger.info('ioredis successfully initialized.')) - .on('reconnecting', () => logger.info('ioredis reconnecting...')); +const redis = new Redis.Cluster(REDIS_URI); module.exports = redis; diff --git a/api/server/services/Files/Audio/textToSpeech.js b/api/server/services/Files/Audio/textToSpeech.js index 2d77324ce4..ab7a5e55dc 100644 --- a/api/server/services/Files/Audio/textToSpeech.js +++ b/api/server/services/Files/Audio/textToSpeech.js @@ -16,7 +16,8 @@ const { logger } = require('~/config'); function getProvider(ttsSchema) { if (!ttsSchema) { throw new Error(`No TTS schema is set. Did you configure TTS in the custom config (librechat.yaml)? -# Example TTS configuration`); + + https://www.librechat.ai/docs/configuration/stt_tts#tts`); } const providers = Object.entries(ttsSchema).filter(([, value]) => Object.keys(value).length > 0); diff --git a/api/server/socialLogins.js b/api/server/socialLogins.js index 4abe278b84..66ee5f9e42 100644 --- a/api/server/socialLogins.js +++ b/api/server/socialLogins.js @@ -1,14 +1,15 @@ +const Redis = require('ioredis'); +const passport = require('passport'); const session = require('express-session'); const RedisStore = require('connect-redis').default; -const passport = require('passport'); const { + setupOpenId, googleLogin, githubLogin, discordLogin, facebookLogin, - setupOpenId, -} = require('../strategies'); -const client = require('../cache/redis'); +} = require('~/strategies'); +const { logger } = require('~/config'); /** * @@ -40,6 +41,11 @@ const configureSocialLogins = (app) => { saveUninitialized: false, }; if (process.env.USE_REDIS) { + const client = new Redis(process.env.REDIS_URI); + client + .on('error', (err) => logger.error('ioredis error:', err)) + .on('ready', () => logger.info('ioredis successfully initialized.')) + .on('reconnecting', () => logger.info('ioredis reconnecting...')); sessionOptions.store = new RedisStore({ client, prefix: 'librechat' }); } app.use(session(sessionOptions)); diff --git a/client/src/components/Chat/Input/AudioRecorder.tsx b/client/src/components/Chat/Input/AudioRecorder.tsx index c1d100e50d..d4ea2c4a8e 100644 --- a/client/src/components/Chat/Input/AudioRecorder.tsx +++ b/client/src/components/Chat/Input/AudioRecorder.tsx @@ -1,16 +1,46 @@ -import React from 'react'; -import { ListeningIcon, Spinner, SpeechIcon } from '~/components/svg'; +import { useEffect } from 'react'; +import type { UseFormReturn } from 'react-hook-form'; import { TooltipProvider, Tooltip, TooltipTrigger, TooltipContent } from '~/components/ui/'; -import { useLocalize } from '~/hooks'; +import { ListeningIcon, Spinner } from '~/components/svg'; +import { useLocalize, useSpeechToText } from '~/hooks'; +import { globalAudioId } from '~/common'; export default function AudioRecorder({ - isListening, - isLoading, - startRecording, - stopRecording, + textAreaRef, + methods, + ask, disabled, +}: { + textAreaRef: React.RefObject; + methods: UseFormReturn<{ text: string }>; + ask: (data: { text: string }) => void; + disabled: boolean; }) { const localize = useLocalize(); + + const handleTranscriptionComplete = (text: string) => { + if (text) { + const globalAudio = document.getElementById(globalAudioId) as HTMLAudioElement; + if (globalAudio) { + console.log('Unmuting global audio'); + globalAudio.muted = false; + } + ask({ text }); + methods.reset({ text: '' }); + clearText(); + } + }; + + const { isListening, isLoading, startRecording, stopRecording, speechText, clearText } = + useSpeechToText(handleTranscriptionComplete); + + useEffect(() => { + if (textAreaRef.current) { + textAreaRef.current.value = speechText; + methods.setValue('text', speechText, { shouldValidate: true }); + } + }, [speechText, methods, textAreaRef]); + const handleStartRecording = async () => { await startRecording(); }; @@ -19,6 +49,16 @@ export default function AudioRecorder({ await stopRecording(); }; + const renderIcon = () => { + if (isListening) { + return ; + } + if (isLoading) { + return ; + } + return ; + }; + return ( @@ -29,13 +69,7 @@ export default function AudioRecorder({ className="absolute bottom-1.5 right-12 flex h-[30px] w-[30px] items-center justify-center rounded-lg p-0.5 transition-colors hover:bg-gray-200 dark:hover:bg-gray-700 md:bottom-3 md:right-12" type="button" > - {isListening ? ( - - ) : isLoading ? ( - - ) : ( - - )} + {renderIcon()} diff --git a/client/src/components/Chat/Input/ChatForm.tsx b/client/src/components/Chat/Input/ChatForm.tsx index d3cb04e422..f5cb4232ca 100644 --- a/client/src/components/Chat/Input/ChatForm.tsx +++ b/client/src/components/Chat/Input/ChatForm.tsx @@ -1,6 +1,6 @@ import { useForm } from 'react-hook-form'; import { useRecoilState, useRecoilValue } from 'recoil'; -import { memo, useCallback, useRef, useMemo, useEffect } from 'react'; +import { memo, useCallback, useRef, useMemo } from 'react'; import { supportsFiles, mergeFileConfig, @@ -8,7 +8,7 @@ import { fileConfig as defaultFileConfig, } from 'librechat-data-provider'; import { useChatContext, useAssistantsMapContext } from '~/Providers'; -import { useRequiresKey, useTextarea, useSpeechToText } from '~/hooks'; +import { useRequiresKey, useTextarea } from '~/hooks'; import { TextareaAutosize } from '~/components/ui'; import { useGetFileConfig } from '~/data-provider'; import { cn, removeFocusOutlines } from '~/utils'; @@ -72,24 +72,6 @@ const ChatForm = ({ index = 0 }) => { const { endpoint: _endpoint, endpointType } = conversation ?? { endpoint: null }; const endpoint = endpointType ?? _endpoint; - const handleTranscriptionComplete = (text: string) => { - if (text) { - ask({ text }); - methods.reset({ text: '' }); - clearText(); - } - }; - - const { isListening, isLoading, startRecording, stopRecording, speechText, clearText } = - useSpeechToText(handleTranscriptionComplete); - - useEffect(() => { - if (textAreaRef.current) { - textAreaRef.current.value = speechText; - methods.setValue('text', speechText, { shouldValidate: true }); - } - }, [speechText, methods]); - const { data: fileConfig = defaultFileConfig } = useGetFileConfig({ select: (data) => mergeFileConfig(data), }); @@ -183,11 +165,10 @@ const ChatForm = ({ index = 0 }) => { )} {SpeechToText && ( )} {TextToSpeech && automaticPlayback && } diff --git a/client/src/components/Chat/Input/StreamAudio.tsx b/client/src/components/Chat/Input/StreamAudio.tsx index b89c295d37..ebf0f4e48e 100644 --- a/client/src/components/Chat/Input/StreamAudio.tsx +++ b/client/src/components/Chat/Input/StreamAudio.tsx @@ -88,7 +88,7 @@ export default function StreamAudio({ index = 0 }) { return; } - console.log('Fetching audio...'); + console.log('Fetching audio...', navigator.userAgent); const response = await fetch('/api/files/tts', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${token}` }, @@ -103,8 +103,14 @@ export default function StreamAudio({ index = 0 }) { } const reader = response.body.getReader(); - const mediaSource = new MediaSourceAppender('audio/mpeg'); - setGlobalAudioURL(mediaSource.mediaSourceUrl); + + const type = 'audio/mpeg'; + const browserSupportsType = MediaSource.isTypeSupported(type); + let mediaSource: MediaSourceAppender | undefined; + if (browserSupportsType) { + mediaSource = new MediaSourceAppender(type); + setGlobalAudioURL(mediaSource.mediaSourceUrl); + } setAudioRunId(activeRunId); let done = false; @@ -120,7 +126,7 @@ export default function StreamAudio({ index = 0 }) { if (cacheTTS && value) { chunks.push(value); } - if (value) { + if (value && mediaSource) { mediaSource.addData(value); } done = readerDone; @@ -136,8 +142,19 @@ export default function StreamAudio({ index = 0 }) { if (!cacheKey) { throw new Error('Cache key not found'); } - const audioBlob = new Blob(chunks, { type: 'audio/mpeg' }); - cache.put(cacheKey, new Response(audioBlob)); + const audioBlob = new Blob(chunks, { type }); + const cachedResponse = new Response(audioBlob); + await cache.put(cacheKey, cachedResponse); + if (!browserSupportsType) { + const unconsumedResponse = await cache.match(cacheKey); + if (!unconsumedResponse) { + throw new Error('Failed to fetch audio from cache'); + } + const audioBlob = await unconsumedResponse.blob(); + const blobUrl = URL.createObjectURL(audioBlob); + setGlobalAudioURL(blobUrl); + } + setIsFetching(false); } console.log('Audio stream reading ended'); @@ -194,9 +211,16 @@ export default function StreamAudio({ index = 0 }) { ref={audioRef} controls controlsList="nodownload nofullscreen noremoteplayback" - className="absolute h-0 w-0 overflow-hidden" + style={{ + position: 'absolute', + overflow: 'hidden', + display: 'none', + height: '0px', + width: '0px', + }} src={globalAudioURL || undefined} id={globalAudioId} + muted autoPlay /> ); diff --git a/client/src/components/Chat/Messages/HoverButtons.tsx b/client/src/components/Chat/Messages/HoverButtons.tsx index ab9052dc77..02e4414818 100644 --- a/client/src/components/Chat/Messages/HoverButtons.tsx +++ b/client/src/components/Chat/Messages/HoverButtons.tsx @@ -1,18 +1,10 @@ import React, { useState } from 'react'; import { useRecoilState } from 'recoil'; import type { TConversation, TMessage } from 'librechat-data-provider'; -import { - Clipboard, - CheckMark, - EditIcon, - RegenerateIcon, - ContinueIcon, - VolumeIcon, - VolumeMuteIcon, - Spinner, -} from '~/components/svg'; -import { useGenerationsByLatest, useLocalize, useTextToSpeech } from '~/hooks'; +import { EditIcon, Clipboard, CheckMark, ContinueIcon, RegenerateIcon } from '~/components/svg'; +import { useGenerationsByLatest, useLocalize } from '~/hooks'; import { Fork } from '~/components/Conversations'; +import MessageAudio from './MessageAudio'; import { cn } from '~/utils'; import store from '~/store'; @@ -49,12 +41,6 @@ export default function HoverButtons({ const [isCopied, setIsCopied] = useState(false); const [TextToSpeech] = useRecoilState(store.TextToSpeech); - const { handleMouseDown, handleMouseUp, toggleSpeech, isSpeaking, isLoading } = useTextToSpeech( - message?.content ?? message?.text ?? '', - isLast, - index, - ); - const { hideEditButton, regenerateEnabled, @@ -81,32 +67,9 @@ export default function HoverButtons({ enterEdit(); }; - const renderIcon = (size: string) => { - if (isLoading) { - return ; - } - - if (isSpeaking) { - return ; - } - - return ; - }; - return (
- {TextToSpeech && ( - - )} + {TextToSpeech && } {isEditableEndpoint && ( +