From 3f600f0d3fa349939dbcfc9120b342a9b8e17377 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 29 May 2024 09:15:05 -0400 Subject: [PATCH] =?UTF-8?q?=E2=AC=87=EF=B8=8F=20fix:=20JSON=20LibreChat=20?= =?UTF-8?q?Imports=20(#2897)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: remove unused code * refactor: Update NewChatButtonIcon component to use JSX syntax The NewChatButtonIcon component in the Nav folder has been updated to use JSX syntax instead of calling the Icon function directly. This change improves code readability and maintainability. * remove use memo * refactor: allow passing `select` to messages db query * fix: initial fix for non-recursive messages * ci: first pass, importers test rewrite * fix(groupConversationsByDate): handle edge case of conversation.updatedAt being null * fix: correctly handle non-recursive uploads * feat: imports non-recursive conversations with branches correctly * feat: support retaining original options on import * refactor: Allow `messageTree` field for Import of non-recursive conversations --- api/models/Conversation.js | 2 +- api/models/Message.js | 12 +- api/server/routes/messages.js | 4 +- .../import/__data__/librechat-export.json | 2 +- .../import/__data__/librechat-linear.json | 40 ++ .../librechat-opts-nonr-branches.json | 104 +++++ .../utils/import/__data__/librechat-tree.json | 2 +- api/server/utils/import/importers.js | 110 ++++-- api/server/utils/import/importers.spec.js | 358 +++++++++++++----- client/src/components/Nav/NavLinks.tsx | 17 +- client/src/components/Nav/NewChat.tsx | 20 +- client/src/components/ui/Combobox.tsx | 17 +- client/src/utils/convos.ts | 3 +- 13 files changed, 516 insertions(+), 175 deletions(-) create mode 100644 api/server/utils/import/__data__/librechat-linear.json create mode 100644 api/server/utils/import/__data__/librechat-opts-nonr-branches.json diff --git a/api/models/Conversation.js b/api/models/Conversation.js index 2dfe8d51cd..1cd1b0aa96 100644 --- a/api/models/Conversation.js +++ b/api/models/Conversation.js @@ -21,7 +21,7 @@ module.exports = { Conversation, saveConvo: async (user, { conversationId, newConversationId, ...convo }) => { try { - const messages = await getMessages({ conversationId }); + const messages = await getMessages({ conversationId }, '_id'); const update = { ...convo, messages, user }; if (newConversationId) { update.conversationId = newConversationId; diff --git a/api/models/Message.js b/api/models/Message.js index 6435fa6fe8..f86849fe93 100644 --- a/api/models/Message.js +++ b/api/models/Message.js @@ -179,8 +179,18 @@ module.exports = { } }, - async getMessages(filter) { + /** + * Retrieves messages from the database. + * @param {Record} filter + * @param {string | undefined} [select] + * @returns + */ + async getMessages(filter, select) { try { + if (select) { + return await Message.find(filter).select(select).sort({ createdAt: 1 }).lean(); + } + return await Message.find(filter).sort({ createdAt: 1 }).lean(); } catch (err) { logger.error('Error getting messages:', err); diff --git a/api/server/routes/messages.js b/api/server/routes/messages.js index d53dacae49..e0bdadfc50 100644 --- a/api/server/routes/messages.js +++ b/api/server/routes/messages.js @@ -14,7 +14,7 @@ router.use(requireJwtAuth); router.get('/:conversationId', validateMessageReq, async (req, res) => { const { conversationId } = req.params; - res.status(200).send(await getMessages({ conversationId })); + res.status(200).send(await getMessages({ conversationId }, '-_id -__v -user')); }); // CREATE @@ -28,7 +28,7 @@ router.post('/:conversationId', validateMessageReq, async (req, res) => { // READ router.get('/:conversationId/:messageId', validateMessageReq, async (req, res) => { const { conversationId, messageId } = req.params; - res.status(200).send(await getMessages({ conversationId, messageId })); + res.status(200).send(await getMessages({ conversationId, messageId }, '-_id -__v -user')); }); // UPDATE diff --git a/api/server/utils/import/__data__/librechat-export.json b/api/server/utils/import/__data__/librechat-export.json index 6e28098d4f..001d831401 100644 --- a/api/server/utils/import/__data__/librechat-export.json +++ b/api/server/utils/import/__data__/librechat-export.json @@ -19,7 +19,7 @@ "endpoint": "openAI", "title": "VW Transporter 2014 Fuel Consumption. Web Search" }, - "messagesTree": [ + "messages": [ { "_id": "6615516574dc2ddcdebe40b6", "messageId": "b123942f-ca1a-4b16-9e1f-ea4af5171168", diff --git a/api/server/utils/import/__data__/librechat-linear.json b/api/server/utils/import/__data__/librechat-linear.json new file mode 100644 index 0000000000..79a94d2f13 --- /dev/null +++ b/api/server/utils/import/__data__/librechat-linear.json @@ -0,0 +1,40 @@ +{ + "conversationId": "4c9e1402-f746-479d-8a18-2dfab1bf750f", + "endpoint": "azureOpenAI", + "title": "Adventure Across Time and Space", + "exportAt": "20:33:31 GMT-0400 (Eastern Daylight Time)", + "branches": false, + "recursive": false, + "messages": [ + { + "messageId": "ed3fdaaa-e174-4d70-89a9-b06e70939723", + "conversationId": "4c9e1402-f746-479d-8a18-2dfab1bf750f", + "createdAt": "2024-05-29T00:33:03.661Z", + "endpoint": "azureOpenAI", + "error": false, + "isCreatedByUser": true, + "isEdited": false, + "model": null, + "parentMessageId": null, + "sender": "user", + "text": "tell me a long story", + "unfinished": false, + "updatedAt": "2024-05-29T00:33:03.661Z" + }, + { + "messageId": "113b8336-4b19-4f91-8eed-741b3dc2940e", + "conversationId": "4c9e1402-f746-479d-8a18-2dfab1bf750f", + "createdAt": "2024-05-29T00:33:03.661Z", + "endpoint": "azureOpenAI", + "error": false, + "isCreatedByUser": false, + "isEdited": false, + "model": "gpt-4o", + "parentMessageId": "ed3fdaaa-e174-4d70-89a9-b06e70939723", + "sender": "GPT-4", + "text": "Of course! Here’s a detailed and engaging story:\n\n---\n\n### The Legend of Eldoria\n\nNestled between towering mountains and dense, ancient forests was the enigmatic kingdom of Eldoria. This realm, clo aked in perpetual twilight, was the stuff of legends. It was said that the land was blessed by the gods and guarded by mythical creatures. Eldoria was a place where magic and realism intertwined seamlessly, creating a land of beauty, wonder, and peril.\n\nIn the heart of this kingdom lay the grand city of Lumina, known", + "unfinished": false, + "updatedAt": "2024-05-29T00:33:03.661Z" + } + ] +} diff --git a/api/server/utils/import/__data__/librechat-opts-nonr-branches.json b/api/server/utils/import/__data__/librechat-opts-nonr-branches.json new file mode 100644 index 0000000000..cc8a5e05b4 --- /dev/null +++ b/api/server/utils/import/__data__/librechat-opts-nonr-branches.json @@ -0,0 +1,104 @@ +{ + "conversationId": "27b593be-9500-479c-94cb-050cab8f5033", + "endpoint": "azureOpenAI", + "title": "Original", + "exportAt": "08:12:16 GMT-0400 (Eastern Daylight Time)", + "branches": true, + "recursive": false, + "options": { + "presetId": null, + "model": "gpt-4o", + "chatGptLabel": null, + "promptPrefix": null, + "temperature": 1, + "top_p": 1, + "presence_penalty": 0, + "frequency_penalty": 0, + "resendFiles": true, + "imageDetail": "auto", + "endpoint": "azureOpenAI", + "title": "Original" + }, + "messages": [ + { + "messageId": "115a6247-8fb0-4937-a536-12956669098d", + "conversationId": "27b593be-9500-479c-94cb-050cab8f5033", + "createdAt": "2024-05-28T18:08:55.014Z", + "endpoint": "azureOpenAI", + "error": false, + "isCreatedByUser": true, + "isEdited": false, + "model": null, + "parentMessageId": "00000000-0000-0000-0000-000000000000", + "sender": "User", + "text": "tell me a long story", + "tokenCount": 9, + "unfinished": false, + "updatedAt": "2024-05-28T18:09:27.193Z" + }, + { + "messageId": "069b9c22-7649-45a9-b90b-fc050533ea21", + "conversationId": "27b593be-9500-479c-94cb-050cab8f5033", + "createdAt": "2024-05-28T18:08:55.390Z", + "error": false, + "isCreatedByUser": false, + "isEdited": false, + "model": "gpt-4o", + "parentMessageId": "115a6247-8fb0-4937-a536-12956669098d", + "sender": "GPT-4", + "text": "Of course! Settle in for a tale of adventure across time and space.\n\n---\n\nOnce upon a time in the small, sleepy village of Eldoria, there was a young woman named Elara who longed for adventure. Eldoria was a place of routine and simplicity, nestled between rolling hills and dense forests, but Elara always felt that there was more to the world than the boundaries", + "unfinished": false, + "updatedAt": "2024-05-28T18:08:58.669Z", + "endpoint": "azureOpenAI", + "tokenCount": 78, + "finish_reason": "incomplete" + }, + { + "messageId": "303e4c2c-f03e-4e0a-8551-c96ec73be5fe", + "conversationId": "27b593be-9500-479c-94cb-050cab8f5033", + "createdAt": "2024-05-28T18:09:27.444Z", + "error": false, + "isCreatedByUser": false, + "isEdited": false, + "model": "gpt-4o", + "parentMessageId": "115a6247-8fb0-4937-a536-12956669098d", + "sender": "GPT-4", + "text": "Sure, I can craft a long story for you. Here it goes:\n\n### The Chronicles of Elenor: The Luminary of Anduril\n\nIn an age long forgotten by men, in a world kissed by the glow of dual suns, the Kingdom of Anduril flourished. Verdant valleys graced its land, majestic mountains shielded", + "unfinished": true, + "updatedAt": "2024-05-28T18:09:30.448Z" + }, + { + "messageId": "599e1908-8c52-4a73-ba6b-f6dffbd79ba0", + "conversationId": "27b593be-9500-479c-94cb-050cab8f5033", + "createdAt": "2024-05-28T18:14:07.988Z", + "endpoint": "azureOpenAI", + "error": false, + "isCreatedByUser": true, + "isEdited": false, + "model": null, + "parentMessageId": "00000000-0000-0000-0000-000000000000", + "sender": "User", + "text": "tell me a long long story", + "tokenCount": 9, + "unfinished": false, + "updatedAt": "2024-05-28T18:14:07.988Z" + }, + { + "messageId": "de9a4e7c-020d-4856-a5a6-ce6794efef99", + "conversationId": "27b593be-9500-479c-94cb-050cab8f5033", + "createdAt": "2024-05-28T18:14:08.403Z", + "error": false, + "isCreatedByUser": false, + "isEdited": true, + "model": "gpt-4o", + "parentMessageId": "599e1908-8c52-4a73-ba6b-f6dffbd79ba0", + "sender": "GPT-4", + "text": "Of course! Here’s a detailed and engaging story:\n\n---\n\n### The Legend of Eldoria\n\nNestled between towering mountains and dense, ancient forests was the enigmatic kingdom of Eldoria. This realm, clo aked in perpetual twilight, was the stuff of legends. It was said that the land was blessed by the gods and guarded by mythical creatures. Eldoria was a place where magic and realism intertwined seamlessly, creating a land of beauty, wonder, and peril.\n\nIn the heart of this kingdom lay the grand city of Lumina, known", + "unfinished": false, + "updatedAt": "2024-05-28T18:14:20.349Z", + "endpoint": "azureOpenAI", + "finish_reason": "incomplete", + "tokenCount": 110 + } + ] +} diff --git a/api/server/utils/import/__data__/librechat-tree.json b/api/server/utils/import/__data__/librechat-tree.json index 20eefcdf9b..50130da2dc 100644 --- a/api/server/utils/import/__data__/librechat-tree.json +++ b/api/server/utils/import/__data__/librechat-tree.json @@ -19,7 +19,7 @@ "endpoint": "openAI", "title": "Troubleshooting Python Virtual Environment Activation Issue" }, - "messagesTree": [ + "messages": [ { "_id": "66326f3f04bed94b7f5be68d", "messageId": "9501f99d-9bbb-40cb-bbb2-16d79aeceb72", diff --git a/api/server/utils/import/importers.js b/api/server/utils/import/importers.js index 6e151ead27..7b512cc4c4 100644 --- a/api/server/utils/import/importers.js +++ b/api/server/utils/import/importers.js @@ -1,6 +1,7 @@ const { v4: uuidv4 } = require('uuid'); -const { EModelEndpoint, Constants, openAISettings } = require('librechat-data-provider'); +const { EModelEndpoint, Constants, openAISettings, CacheKeys } = require('librechat-data-provider'); const { createImportBatchBuilder } = require('./importBatchBuilder'); +const getLogStores = require('~/cache/getLogStores'); const logger = require('~/config/winston'); /** @@ -24,7 +25,7 @@ function getImporter(jsonData) { } // For LibreChat - if (jsonData.conversationId && jsonData.messagesTree) { + if (jsonData.conversationId && (jsonData.messagesTree || jsonData.messages)) { logger.info('Importing LibreChat conversation'); return importLibreChatConvo; } @@ -85,47 +86,92 @@ async function importLibreChatConvo( try { /** @type {ImportBatchBuilder} */ const importBatchBuilder = builderFactory(requestUserId); - importBatchBuilder.startConversation(EModelEndpoint.openAI); + const options = jsonData.options || {}; + + /* Endpoint configuration */ + let endpoint = jsonData.endpoint ?? options.endpoint ?? EModelEndpoint.openAI; + const cache = getLogStores(CacheKeys.CONFIG_STORE); + const endpointsConfig = await cache.get(CacheKeys.ENDPOINT_CONFIG); + const endpointConfig = endpointsConfig?.[endpoint]; + if (!endpointConfig && endpointsConfig) { + endpoint = Object.keys(endpointsConfig)[0]; + } else if (!endpointConfig) { + endpoint = EModelEndpoint.openAI; + } + + importBatchBuilder.startConversation(endpoint); let firstMessageDate = null; - const traverseMessages = (messages, parentMessageId = null) => { - for (const message of messages) { - if (!message.text) { - continue; - } + const messagesToImport = jsonData.messagesTree || jsonData.messages; - let savedMessage; - if (message.sender?.toLowerCase() === 'user') { - savedMessage = importBatchBuilder.saveMessage({ - text: message.text, - sender: 'user', - isCreatedByUser: true, - parentMessageId: parentMessageId, - }); - } else { - savedMessage = importBatchBuilder.saveMessage({ - text: message.text, - sender: message.sender, - isCreatedByUser: false, - model: jsonData.options.model, - parentMessageId: parentMessageId, - }); - } + if (jsonData.recursive) { + /** + * Recursively traverse the messages tree and save each message to the database. + * @param {TMessage[]} messages + * @param {string} parentMessageId + */ + const traverseMessages = async (messages, parentMessageId = null) => { + for (const message of messages) { + if (!message.text) { + continue; + } + let savedMessage; + if (message.sender?.toLowerCase() === 'user' || message.isCreatedByUser) { + savedMessage = await importBatchBuilder.saveMessage({ + text: message.text, + sender: 'user', + isCreatedByUser: true, + parentMessageId: parentMessageId, + }); + } else { + savedMessage = await importBatchBuilder.saveMessage({ + text: message.text, + sender: message.sender, + isCreatedByUser: false, + model: options.model, + parentMessageId: parentMessageId, + }); + } + + if (!firstMessageDate) { + firstMessageDate = new Date(message.createdAt); + } + + if (message.children && message.children.length > 0) { + await traverseMessages(message.children, savedMessage.messageId); + } + } + }; + + await traverseMessages(messagesToImport); + } else if (messagesToImport) { + const idMapping = new Map(); + + for (const message of messagesToImport) { if (!firstMessageDate) { firstMessageDate = new Date(message.createdAt); } + const newMessageId = uuidv4(); + idMapping.set(message.messageId, newMessageId); - if (message.children) { - traverseMessages(message.children, savedMessage.messageId); - } + const clonedMessage = { + ...message, + messageId: newMessageId, + parentMessageId: + message.parentMessageId && message.parentMessageId !== Constants.NO_PARENT + ? idMapping.get(message.parentMessageId) || Constants.NO_PARENT + : Constants.NO_PARENT, + }; + + importBatchBuilder.saveMessage(clonedMessage); } - }; + } else { + throw new Error('Invalid LibreChat file format'); + } - traverseMessages(jsonData.messagesTree); - - importBatchBuilder.finishConversation(jsonData.title, firstMessageDate); + importBatchBuilder.finishConversation(jsonData.title, firstMessageDate ?? new Date(), options); await importBatchBuilder.saveBatch(); logger.debug(`user: ${requestUserId} | Conversation "${jsonData.title}" imported`); } catch (error) { diff --git a/api/server/utils/import/importers.spec.js b/api/server/utils/import/importers.spec.js index 38036fe3eb..fa0b2e9f33 100644 --- a/api/server/utils/import/importers.spec.js +++ b/api/server/utils/import/importers.spec.js @@ -1,70 +1,68 @@ const fs = require('fs'); const path = require('path'); -const { EModelEndpoint, Constants } = require('librechat-data-provider'); +const { EModelEndpoint, Constants, openAISettings } = require('librechat-data-provider'); +const { bulkSaveConvos: _bulkSaveConvos } = require('~/models/Conversation'); const { ImportBatchBuilder } = require('./importBatchBuilder'); +const { bulkSaveMessages } = require('~/models/Message'); +const getLogStores = require('~/cache/getLogStores'); const { getImporter } = require('./importers'); -// Mocking the ImportBatchBuilder class and its methods -jest.mock('./importBatchBuilder', () => { - return { - ImportBatchBuilder: jest.fn().mockImplementation(() => { - return { - startConversation: jest.fn().mockResolvedValue(undefined), - addUserMessage: jest.fn().mockResolvedValue(undefined), - addGptMessage: jest.fn().mockResolvedValue(undefined), - saveMessage: jest.fn().mockResolvedValue(undefined), - finishConversation: jest.fn().mockResolvedValue(undefined), - saveBatch: jest.fn().mockResolvedValue(undefined), - }; - }), - }; +jest.mock('~/cache/getLogStores'); +const mockedCacheGet = jest.fn(); +getLogStores.mockImplementation(() => ({ + get: mockedCacheGet, +})); + +// Mock the database methods +jest.mock('~/models/Conversation', () => ({ + bulkSaveConvos: jest.fn(), +})); +jest.mock('~/models/Message', () => ({ + bulkSaveMessages: jest.fn(), +})); + +afterEach(() => { + jest.clearAllMocks(); }); describe('importChatGptConvo', () => { it('should import conversation correctly', async () => { const expectedNumberOfMessages = 19; - const expectedNumberOfConversations = 2; - // Given const jsonData = JSON.parse( fs.readFileSync(path.join(__dirname, '__data__', 'chatgpt-export.json'), 'utf8'), ); const requestUserId = 'user-123'; - const mockedBuilderFactory = jest.fn().mockReturnValue(new ImportBatchBuilder(requestUserId)); + const importBatchBuilder = new ImportBatchBuilder(requestUserId); + + // Spy on instance methods + jest.spyOn(importBatchBuilder, 'startConversation'); + jest.spyOn(importBatchBuilder, 'saveMessage'); + jest.spyOn(importBatchBuilder, 'finishConversation'); + jest.spyOn(importBatchBuilder, 'saveBatch'); - // When const importer = getImporter(jsonData); - await importer(jsonData, requestUserId, mockedBuilderFactory); + await importer(jsonData, requestUserId, () => importBatchBuilder); - // Then - expect(mockedBuilderFactory).toHaveBeenCalledWith(requestUserId); - const mockImportBatchBuilder = mockedBuilderFactory.mock.results[0].value; - - expect(mockImportBatchBuilder.startConversation).toHaveBeenCalledWith(EModelEndpoint.openAI); - expect(mockImportBatchBuilder.saveMessage).toHaveBeenCalledTimes(expectedNumberOfMessages); // Adjust expected number - expect(mockImportBatchBuilder.finishConversation).toHaveBeenCalledTimes( - expectedNumberOfConversations, - ); // Adjust expected number - expect(mockImportBatchBuilder.saveBatch).toHaveBeenCalled(); + expect(importBatchBuilder.startConversation).toHaveBeenCalledWith(EModelEndpoint.openAI); + expect(importBatchBuilder.saveMessage).toHaveBeenCalledTimes(expectedNumberOfMessages); + expect(importBatchBuilder.finishConversation).toHaveBeenCalledTimes(jsonData.length); + expect(importBatchBuilder.saveBatch).toHaveBeenCalled(); }); + it('should maintain correct message hierarchy (tree parent/children relationship)', async () => { - // Prepare test data with known hierarchy const jsonData = JSON.parse( fs.readFileSync(path.join(__dirname, '__data__', 'chatgpt-tree.json'), 'utf8'), ); - const requestUserId = 'user-123'; - const mockedBuilderFactory = jest.fn().mockReturnValue(new ImportBatchBuilder(requestUserId)); + const importBatchBuilder = new ImportBatchBuilder(requestUserId); + + jest.spyOn(importBatchBuilder, 'saveMessage'); + jest.spyOn(importBatchBuilder, 'saveBatch'); - // When const importer = getImporter(jsonData); - await importer(jsonData, requestUserId, mockedBuilderFactory); - - // Then - expect(mockedBuilderFactory).toHaveBeenCalledWith(requestUserId); - const mockImportBatchBuilder = mockedBuilderFactory.mock.results[0].value; + await importer(jsonData, requestUserId, () => importBatchBuilder); const entries = Object.keys(jsonData[0].mapping); - // Filter entries that should be processed (not system and have content) const messageEntries = entries.filter( (id) => jsonData[0].mapping[id].message && @@ -72,20 +70,16 @@ describe('importChatGptConvo', () => { jsonData[0].mapping[id].message.content, ); - // Expect the saveMessage to be called for each valid entry - expect(mockImportBatchBuilder.saveMessage).toHaveBeenCalledTimes(messageEntries.length); + expect(importBatchBuilder.saveMessage).toHaveBeenCalledTimes(messageEntries.length); const idToUUIDMap = new Map(); - // Map original IDs to dynamically generated UUIDs - mockImportBatchBuilder.saveMessage.mock.calls.forEach((call, index) => { + importBatchBuilder.saveMessage.mock.calls.forEach((call, index) => { const originalId = messageEntries[index]; idToUUIDMap.set(originalId, call[0].messageId); }); - // Validate the UUID map contains all expected entries expect(idToUUIDMap.size).toBe(messageEntries.length); - // Validate correct parent-child relationships messageEntries.forEach((id) => { const { parent } = jsonData[0].mapping[id]; @@ -93,72 +87,110 @@ describe('importChatGptConvo', () => { ? idToUUIDMap.get(parent) ?? Constants.NO_PARENT : Constants.NO_PARENT; - const actualParentId = idToUUIDMap.get(id) - ? mockImportBatchBuilder.saveMessage.mock.calls.find( - (call) => call[0].messageId === idToUUIDMap.get(id), + const actualMessageId = idToUUIDMap.get(id); + const actualParentId = actualMessageId + ? importBatchBuilder.saveMessage.mock.calls.find( + (call) => call[0].messageId === actualMessageId, )[0].parentMessageId : Constants.NO_PARENT; expect(actualParentId).toBe(expectedParentId); }); - expect(mockImportBatchBuilder.saveBatch).toHaveBeenCalled(); + expect(importBatchBuilder.saveBatch).toHaveBeenCalled(); }); }); describe('importLibreChatConvo', () => { - it('should import conversation correctly', async () => { - const expectedNumberOfMessages = 6; - const expectedNumberOfConversations = 1; + const jsonDataNonRecursiveBranches = JSON.parse( + fs.readFileSync(path.join(__dirname, '__data__', 'librechat-opts-nonr-branches.json'), 'utf8'), + ); - // Given + it('should import conversation correctly', async () => { + mockedCacheGet.mockResolvedValue({ + [EModelEndpoint.openAI]: {}, + }); + const expectedNumberOfMessages = 6; const jsonData = JSON.parse( fs.readFileSync(path.join(__dirname, '__data__', 'librechat-export.json'), 'utf8'), ); const requestUserId = 'user-123'; - const mockedBuilderFactory = jest.fn().mockReturnValue(new ImportBatchBuilder(requestUserId)); + const importBatchBuilder = new ImportBatchBuilder(requestUserId); + + // Spy on instance methods + jest.spyOn(importBatchBuilder, 'startConversation'); + jest.spyOn(importBatchBuilder, 'saveMessage'); + jest.spyOn(importBatchBuilder, 'finishConversation'); + jest.spyOn(importBatchBuilder, 'saveBatch'); - // When const importer = getImporter(jsonData); - await importer(jsonData, requestUserId, mockedBuilderFactory); + await importer(jsonData, requestUserId, () => importBatchBuilder); - // Then - const mockImportBatchBuilder = mockedBuilderFactory.mock.results[0].value; - expect(mockImportBatchBuilder.startConversation).toHaveBeenCalledWith(EModelEndpoint.openAI); - expect(mockImportBatchBuilder.saveMessage).toHaveBeenCalledTimes(expectedNumberOfMessages); // Adjust expected number - expect(mockImportBatchBuilder.finishConversation).toHaveBeenCalledTimes( - expectedNumberOfConversations, - ); // Adjust expected number - expect(mockImportBatchBuilder.saveBatch).toHaveBeenCalled(); + expect(importBatchBuilder.startConversation).toHaveBeenCalledWith(EModelEndpoint.openAI); + expect(importBatchBuilder.saveMessage).toHaveBeenCalledTimes(expectedNumberOfMessages); + expect(importBatchBuilder.finishConversation).toHaveBeenCalledTimes(1); + expect(importBatchBuilder.saveBatch).toHaveBeenCalled(); }); + + it('should import linear, non-recursive thread correctly with correct endpoint', async () => { + mockedCacheGet.mockResolvedValue({ + [EModelEndpoint.azureOpenAI]: {}, + }); + + const jsonData = JSON.parse( + fs.readFileSync(path.join(__dirname, '__data__', 'librechat-linear.json'), 'utf8'), + ); + const requestUserId = 'user-123'; + const importBatchBuilder = new ImportBatchBuilder(requestUserId); + + jest.spyOn(importBatchBuilder, 'startConversation'); + jest.spyOn(importBatchBuilder, 'saveMessage'); + jest.spyOn(importBatchBuilder, 'finishConversation'); + jest.spyOn(importBatchBuilder, 'saveBatch'); + + const importer = getImporter(jsonData); + await importer(jsonData, requestUserId, () => importBatchBuilder); + + expect(bulkSaveMessages).toHaveBeenCalledTimes(1); + + const messages = bulkSaveMessages.mock.calls[0][0]; + let lastMessageId = Constants.NO_PARENT; + for (const message of messages) { + expect(message.parentMessageId).toBe(lastMessageId); + lastMessageId = message.messageId; + } + + expect(importBatchBuilder.startConversation).toHaveBeenCalledWith(EModelEndpoint.azureOpenAI); + expect(importBatchBuilder.saveMessage).toHaveBeenCalledTimes(jsonData.messages.length); + expect(importBatchBuilder.finishConversation).toHaveBeenCalled(); + expect(importBatchBuilder.saveBatch).toHaveBeenCalled(); + }); + it('should maintain correct message hierarchy (tree parent/children relationship)', async () => { - // Load test data const jsonData = JSON.parse( fs.readFileSync(path.join(__dirname, '__data__', 'librechat-tree.json'), 'utf8'), ); const requestUserId = 'user-123'; - const mockedBuilderFactory = jest.fn().mockReturnValue(new ImportBatchBuilder(requestUserId)); + const importBatchBuilder = new ImportBatchBuilder(requestUserId); + jest.spyOn(importBatchBuilder, 'saveMessage'); + jest.spyOn(importBatchBuilder, 'saveBatch'); // When const importer = getImporter(jsonData); - await importer(jsonData, requestUserId, mockedBuilderFactory); - - // Then - const mockImportBatchBuilder = mockedBuilderFactory.mock.results[0].value; + await importer(jsonData, requestUserId, () => importBatchBuilder); // Create a map to track original message IDs to new UUIDs const idToUUIDMap = new Map(); - mockImportBatchBuilder.saveMessage.mock.calls.forEach((call) => { + importBatchBuilder.saveMessage.mock.calls.forEach((call) => { const message = call[0]; idToUUIDMap.set(message.originalMessageId, message.messageId); }); - // Function to recursively check children const checkChildren = (children, parentId) => { children.forEach((child) => { const childUUID = idToUUIDMap.get(child.messageId); const expectedParentId = idToUUIDMap.get(parentId) ?? null; - const messageCall = mockImportBatchBuilder.saveMessage.mock.calls.find( + const messageCall = importBatchBuilder.saveMessage.mock.calls.find( (call) => call[0].messageId === childUUID, ); @@ -172,75 +204,203 @@ describe('importLibreChatConvo', () => { }; // Start hierarchy validation from root messages - checkChildren(jsonData.messagesTree, null); // Assuming root messages have no parent + checkChildren(jsonData.messages, null); - expect(mockImportBatchBuilder.saveBatch).toHaveBeenCalled(); + expect(importBatchBuilder.saveBatch).toHaveBeenCalled(); + }); + + it('should maintain correct message hierarchy (non-recursive)', async () => { + const jsonData = jsonDataNonRecursiveBranches; + const requestUserId = 'user-123'; + const importBatchBuilder = new ImportBatchBuilder(requestUserId); + jest.spyOn(importBatchBuilder, 'saveMessage'); + jest.spyOn(importBatchBuilder, 'saveBatch'); + + const importer = getImporter(jsonData); + await importer(jsonData, requestUserId, () => importBatchBuilder); + + const textToMessageMap = new Map(); + importBatchBuilder.saveMessage.mock.calls.forEach((call) => { + const message = call[0]; + textToMessageMap.set(message.text, message); + }); + + const relationships = { + 'tell me a long story': [ + 'Of course! Settle in for a tale of adventure across time and space.\n\n---\n\nOnce upon a time in the small, sleepy village of Eldoria, there was a young woman named Elara who longed for adventure. Eldoria was a place of routine and simplicity, nestled between rolling hills and dense forests, but Elara always felt that there was more to the world than the boundaries', + 'Sure, I can craft a long story for you. Here it goes:\n\n### The Chronicles of Elenor: The Luminary of Anduril\n\nIn an age long forgotten by men, in a world kissed by the glow of dual suns, the Kingdom of Anduril flourished. Verdant valleys graced its land, majestic mountains shielded', + ], + 'tell me a long long story': [ + 'Of course! Here’s a detailed and engaging story:\n\n---\n\n### The Legend of Eldoria\n\nNestled between towering mountains and dense, ancient forests was the enigmatic kingdom of Eldoria. This realm, clo aked in perpetual twilight, was the stuff of legends. It was said that the land was blessed by the gods and guarded by mythical creatures. Eldoria was a place where magic and realism intertwined seamlessly, creating a land of beauty, wonder, and peril.\n\nIn the heart of this kingdom lay the grand city of Lumina, known', + ], + }; + + Object.keys(relationships).forEach((parentText) => { + const parentMessage = textToMessageMap.get(parentText); + const childrenTexts = relationships[parentText]; + + childrenTexts.forEach((childText) => { + const childMessage = textToMessageMap.get(childText); + expect(childMessage.parentMessageId).toBe(parentMessage.messageId); + }); + }); + + expect(importBatchBuilder.saveBatch).toHaveBeenCalled(); + }); + + it('should retain properties from the original conversation as well as new settings', async () => { + mockedCacheGet.mockResolvedValue({ + [EModelEndpoint.azureOpenAI]: {}, + }); + const requestUserId = 'user-123'; + const importBatchBuilder = new ImportBatchBuilder(requestUserId); + jest.spyOn(importBatchBuilder, 'finishConversation'); + + const importer = getImporter(jsonDataNonRecursiveBranches); + await importer(jsonDataNonRecursiveBranches, requestUserId, () => importBatchBuilder); + + expect(importBatchBuilder.finishConversation).toHaveBeenCalledTimes(1); + + const [_title, createdAt, originalConvo] = importBatchBuilder.finishConversation.mock.calls[0]; + const convo = importBatchBuilder.conversations[0]; + + expect(convo).toEqual({ + ...jsonDataNonRecursiveBranches.options, + user: requestUserId, + conversationId: importBatchBuilder.conversationId, + title: originalConvo.title || 'Imported Chat', + createdAt: createdAt, + updatedAt: createdAt, + overrideTimestamp: true, + endpoint: importBatchBuilder.endpoint, + model: originalConvo.model || openAISettings.model.default, + }); + + expect(convo.title).toBe('Original'); + expect(convo.createdAt).toBeInstanceOf(Date); + expect(convo.endpoint).toBe(EModelEndpoint.azureOpenAI); + expect(convo.model).toBe('gpt-4o'); + }); + + describe('finishConversation', () => { + it('should retain properties from the original conversation as well as update with new settings', () => { + const requestUserId = 'user-123'; + const builder = new ImportBatchBuilder(requestUserId); + builder.conversationId = 'conv-id-123'; + builder.messages = [{ text: 'Hello, world!' }]; + + const originalConvo = { + _id: 'old-convo-id', + model: 'custom-model', + }; + + builder.endpoint = 'test-endpoint'; + + const title = 'New Chat Title'; + const createdAt = new Date('2023-10-01T00:00:00Z'); + + const result = builder.finishConversation(title, createdAt, originalConvo); + + expect(result).toEqual({ + conversation: { + user: requestUserId, + conversationId: builder.conversationId, + title: 'New Chat Title', + createdAt: createdAt, + updatedAt: createdAt, + overrideTimestamp: true, + endpoint: 'test-endpoint', + model: 'custom-model', + }, + messages: builder.messages, + }); + + expect(builder.conversations).toContainEqual({ + user: requestUserId, + conversationId: builder.conversationId, + title: 'New Chat Title', + createdAt: createdAt, + updatedAt: createdAt, + overrideTimestamp: true, + endpoint: 'test-endpoint', + model: 'custom-model', + }); + }); + + it('should use default values if not provided in the original conversation or as parameters', () => { + const requestUserId = 'user-123'; + const builder = new ImportBatchBuilder(requestUserId); + builder.conversationId = 'conv-id-123'; + builder.messages = [{ text: 'Hello, world!' }]; + builder.endpoint = 'test-endpoint'; + const result = builder.finishConversation(); + expect(result.conversation.title).toBe('Imported Chat'); + expect(result.conversation.model).toBe(openAISettings.model.default); + }); }); }); describe('importChatBotUiConvo', () => { it('should import custom conversation correctly', async () => { - // Given const jsonData = JSON.parse( fs.readFileSync(path.join(__dirname, '__data__', 'chatbotui-export.json'), 'utf8'), ); const requestUserId = 'custom-user-456'; - const mockedBuilderFactory = jest.fn().mockReturnValue(new ImportBatchBuilder(requestUserId)); + const importBatchBuilder = new ImportBatchBuilder(requestUserId); + + // Spy on instance methods + jest.spyOn(importBatchBuilder, 'startConversation'); + jest.spyOn(importBatchBuilder, 'saveMessage'); + jest.spyOn(importBatchBuilder, 'addUserMessage'); + jest.spyOn(importBatchBuilder, 'addGptMessage'); + jest.spyOn(importBatchBuilder, 'finishConversation'); + jest.spyOn(importBatchBuilder, 'saveBatch'); - // When const importer = getImporter(jsonData); - await importer(jsonData, requestUserId, mockedBuilderFactory); + await importer(jsonData, requestUserId, () => importBatchBuilder); - // Then - const mockImportBatchBuilder = mockedBuilderFactory.mock.results[0].value; - expect(mockImportBatchBuilder.startConversation).toHaveBeenCalledWith('openAI'); - - // User messages - expect(mockImportBatchBuilder.addUserMessage).toHaveBeenCalledTimes(3); - expect(mockImportBatchBuilder.addUserMessage).toHaveBeenNthCalledWith( + expect(importBatchBuilder.startConversation).toHaveBeenCalledWith(EModelEndpoint.openAI); + expect(importBatchBuilder.addUserMessage).toHaveBeenCalledTimes(3); + expect(importBatchBuilder.addUserMessage).toHaveBeenNthCalledWith( 1, 'Hello what are you able to do?', ); - expect(mockImportBatchBuilder.addUserMessage).toHaveBeenNthCalledWith( + expect(importBatchBuilder.addUserMessage).toHaveBeenNthCalledWith( 3, 'Give me the code that inverts binary tree in COBOL', ); - // GPT messages - expect(mockImportBatchBuilder.addGptMessage).toHaveBeenCalledTimes(3); - expect(mockImportBatchBuilder.addGptMessage).toHaveBeenNthCalledWith( + expect(importBatchBuilder.addGptMessage).toHaveBeenCalledTimes(3); + expect(importBatchBuilder.addGptMessage).toHaveBeenNthCalledWith( 1, expect.stringMatching(/^Hello! As an AI developed by OpenAI/), 'gpt-4-1106-preview', ); - expect(mockImportBatchBuilder.addGptMessage).toHaveBeenNthCalledWith( + expect(importBatchBuilder.addGptMessage).toHaveBeenNthCalledWith( 3, expect.stringContaining('```cobol'), 'gpt-3.5-turbo', ); - expect(mockImportBatchBuilder.finishConversation).toHaveBeenCalledTimes(2); - expect(mockImportBatchBuilder.finishConversation).toHaveBeenNthCalledWith( + expect(importBatchBuilder.finishConversation).toHaveBeenCalledTimes(2); + expect(importBatchBuilder.finishConversation).toHaveBeenNthCalledWith( 1, 'Hello what are you able to do?', expect.any(Date), ); - expect(mockImportBatchBuilder.finishConversation).toHaveBeenNthCalledWith( + expect(importBatchBuilder.finishConversation).toHaveBeenNthCalledWith( 2, 'Give me the code that inverts ...', expect.any(Date), ); - expect(mockImportBatchBuilder.saveBatch).toHaveBeenCalled(); + expect(importBatchBuilder.saveBatch).toHaveBeenCalled(); }); }); describe('getImporter', () => { it('should throw an error if the import type is not supported', () => { - // Given const jsonData = { unsupported: 'data' }; - - // When expect(() => getImporter(jsonData)).toThrow('Unsupported import type'); }); }); diff --git a/client/src/components/Nav/NavLinks.tsx b/client/src/components/Nav/NavLinks.tsx index 6966a9d278..0e74783e7a 100644 --- a/client/src/components/Nav/NavLinks.tsx +++ b/client/src/components/Nav/NavLinks.tsx @@ -1,10 +1,8 @@ -import { useLocation } from 'react-router-dom'; -import { Fragment, useState, memo } from 'react'; import { FileText } from 'lucide-react'; +import { useRecoilState } from 'recoil'; +import { Fragment, useState, memo } from 'react'; import { Menu, Transition } from '@headlessui/react'; -import { useRecoilValue, useRecoilState } from 'recoil'; import { useGetUserBalance, useGetStartupConfig } from 'librechat-data-provider/react-query'; -import type { TConversation } from 'librechat-data-provider'; import FilesView from '~/components/Chat/Input/Files/FilesView'; import { useAuthContext } from '~/hooks/AuthContext'; import useAvatar from '~/hooks/Messages/useAvatar'; @@ -19,7 +17,6 @@ import store from '~/store'; function NavLinks() { const localize = useLocalize(); - const location = useLocation(); const { user, isAuthenticated } = useAuthContext(); const { data: startupConfig } = useGetStartupConfig(); const balanceQuery = useGetUserBalance({ @@ -28,18 +25,8 @@ function NavLinks() { const [showSettings, setShowSettings] = useState(false); const [showFiles, setShowFiles] = useRecoilState(store.showFiles); - const activeConvo = useRecoilValue(store.conversationByIndex(0)); - const globalConvo = useRecoilValue(store.conversation) ?? ({} as TConversation); - const avatarSrc = useAvatar(user); - let conversation: TConversation | null | undefined; - if (location.state?.from?.pathname.includes('/chat')) { - conversation = globalConvo; - } else { - conversation = activeConvo; - } - return ( <> diff --git a/client/src/components/Nav/NewChat.tsx b/client/src/components/Nav/NewChat.tsx index f0d74cc7b6..a1cc8e8226 100644 --- a/client/src/components/Nav/NewChat.tsx +++ b/client/src/components/Nav/NewChat.tsx @@ -38,16 +38,16 @@ const NewChatButtonIcon = ({ conversation }: { conversation: TConversation | nul ) : (
- {endpoint && - Icon && - Icon({ - size: 41, - context: 'nav', - className: 'h-2/3 w-2/3', - endpoint, - endpointType, - iconURL: endpointIconURL, - })} + {endpoint && Icon && ( + + )}
)} diff --git a/client/src/components/ui/Combobox.tsx b/client/src/components/ui/Combobox.tsx index 61a2382b6b..235174fb0f 100644 --- a/client/src/components/ui/Combobox.tsx +++ b/client/src/components/ui/Combobox.tsx @@ -35,19 +35,12 @@ export default function ComboboxComponent({ isCollapsed: boolean; SelectIcon?: React.ReactNode; }) { - const options: OptionWithIcon[] = useMemo(() => { - if (!items) { - return []; + const options: OptionWithIcon[] = (items ?? []).map((option: string | OptionWithIcon) => { + if (typeof option === 'string') { + return { label: option, value: option }; } - - return items.map((option: string | OptionWithIcon) => { - if (typeof option === 'string') { - return { label: option, value: option }; - } - - return option; - }); - }, [items]); + return option; + }); const { open, setOpen, setSearchValue, matches } = useCombobox({ value: selectedValue, diff --git a/client/src/utils/convos.ts b/client/src/utils/convos.ts index 6f152a4851..86d9ce018c 100644 --- a/client/src/utils/convos.ts +++ b/client/src/utils/convos.ts @@ -6,6 +6,7 @@ import { parseISO, startOfDay, startOfYear, + startOfToday, isWithinInterval, } from 'date-fns'; import { EModelEndpoint, LocalStorageKeys } from 'librechat-data-provider'; @@ -76,7 +77,7 @@ export const groupConversationsByDate = (conversations: TConversation[]): Groupe } seenConversationIds.add(conversation.conversationId); - const date = parseISO(conversation.updatedAt); + const date = conversation.updatedAt ? parseISO(conversation.updatedAt) : startOfToday(); const groupName = getGroupName(date); if (!acc[groupName]) { acc[groupName] = [];