diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index f9dacbe5ad..d8fda5159a 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -512,6 +512,39 @@ class AgentClient extends BaseClient { return withoutKeys; } + /** + * Filters out image URLs from message content + * @param {BaseMessage} message - The message to filter + * @returns {BaseMessage} - A new message with image URLs removed + */ + filterImageUrls(message) { + if (!message.content || typeof message.content === 'string') { + return message; + } + + if (Array.isArray(message.content)) { + const filteredContent = message.content.filter( + (part) => part.type !== ContentTypes.IMAGE_URL, + ); + + if (filteredContent.length === 1 && filteredContent[0].type === ContentTypes.TEXT) { + const MessageClass = message.constructor; + return new MessageClass({ + content: filteredContent[0].text, + additional_kwargs: message.additional_kwargs, + }); + } + + const MessageClass = message.constructor; + return new MessageClass({ + content: filteredContent, + additional_kwargs: message.additional_kwargs, + }); + } + + return message; + } + /** * @param {BaseMessage[]} messages * @returns {Promise} @@ -540,7 +573,8 @@ class AgentClient extends BaseClient { } } - const bufferString = getBufferString(messagesToProcess); + const filteredMessages = messagesToProcess.map((msg) => this.filterImageUrls(msg)); + const bufferString = getBufferString(filteredMessages); const bufferMessage = new HumanMessage(`# Current Chat:\n\n${bufferString}`); return await this.processMemory([bufferMessage]); } catch (error) { diff --git a/api/server/controllers/agents/client.test.js b/api/server/controllers/agents/client.test.js index aab972fecb..92b97ad006 100644 --- a/api/server/controllers/agents/client.test.js +++ b/api/server/controllers/agents/client.test.js @@ -727,4 +727,231 @@ describe('AgentClient - titleConvo', () => { }); }); }); + + describe('runMemory method', () => { + let client; + let mockReq; + let mockRes; + let mockAgent; + let mockOptions; + let mockProcessMemory; + + beforeEach(() => { + jest.clearAllMocks(); + + mockAgent = { + id: 'agent-123', + endpoint: EModelEndpoint.openAI, + provider: EModelEndpoint.openAI, + model_parameters: { + model: 'gpt-4', + }, + }; + + mockReq = { + app: { + locals: { + memory: { + messageWindowSize: 3, + }, + }, + }, + user: { + id: 'user-123', + personalization: { + memories: true, + }, + }, + }; + + mockRes = {}; + + mockOptions = { + req: mockReq, + res: mockRes, + agent: mockAgent, + }; + + mockProcessMemory = jest.fn().mockResolvedValue([]); + + client = new AgentClient(mockOptions); + client.processMemory = mockProcessMemory; + client.conversationId = 'convo-123'; + client.responseMessageId = 'response-123'; + }); + + it('should filter out image URLs from message content', async () => { + const { HumanMessage, AIMessage } = require('@langchain/core/messages'); + const messages = [ + new HumanMessage({ + content: [ + { + type: 'text', + text: 'What is in this image?', + }, + { + type: 'image_url', + image_url: { + url: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==', + detail: 'auto', + }, + }, + ], + }), + new AIMessage('I can see a small red pixel in the image.'), + new HumanMessage({ + content: [ + { + type: 'text', + text: 'What about this one?', + }, + { + type: 'image_url', + image_url: { + url: 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEAYABgAAD/', + detail: 'high', + }, + }, + ], + }), + ]; + + await client.runMemory(messages); + + expect(mockProcessMemory).toHaveBeenCalledTimes(1); + const processedMessage = mockProcessMemory.mock.calls[0][0][0]; + + // Verify the buffer message was created + expect(processedMessage.constructor.name).toBe('HumanMessage'); + expect(processedMessage.content).toContain('# Current Chat:'); + + // Verify that image URLs are not in the buffer string + expect(processedMessage.content).not.toContain('image_url'); + expect(processedMessage.content).not.toContain('data:image'); + expect(processedMessage.content).not.toContain('base64'); + + // Verify text content is preserved + expect(processedMessage.content).toContain('What is in this image?'); + expect(processedMessage.content).toContain('I can see a small red pixel in the image.'); + expect(processedMessage.content).toContain('What about this one?'); + }); + + it('should handle messages with only text content', async () => { + const { HumanMessage, AIMessage } = require('@langchain/core/messages'); + const messages = [ + new HumanMessage('Hello, how are you?'), + new AIMessage('I am doing well, thank you!'), + new HumanMessage('That is great to hear.'), + ]; + + await client.runMemory(messages); + + expect(mockProcessMemory).toHaveBeenCalledTimes(1); + const processedMessage = mockProcessMemory.mock.calls[0][0][0]; + + expect(processedMessage.content).toContain('Hello, how are you?'); + expect(processedMessage.content).toContain('I am doing well, thank you!'); + expect(processedMessage.content).toContain('That is great to hear.'); + }); + + it('should handle mixed content types correctly', async () => { + const { HumanMessage } = require('@langchain/core/messages'); + const { ContentTypes } = require('librechat-data-provider'); + + const messages = [ + new HumanMessage({ + content: [ + { + type: 'text', + text: 'Here is some text', + }, + { + type: ContentTypes.IMAGE_URL, + image_url: { + url: 'https://example.com/image.png', + }, + }, + { + type: 'text', + text: ' and more text', + }, + ], + }), + ]; + + await client.runMemory(messages); + + expect(mockProcessMemory).toHaveBeenCalledTimes(1); + const processedMessage = mockProcessMemory.mock.calls[0][0][0]; + + // Should contain text parts but not image URLs + expect(processedMessage.content).toContain('Here is some text'); + expect(processedMessage.content).toContain('and more text'); + expect(processedMessage.content).not.toContain('example.com/image.png'); + expect(processedMessage.content).not.toContain('IMAGE_URL'); + }); + + it('should preserve original messages without mutation', async () => { + const { HumanMessage } = require('@langchain/core/messages'); + const originalContent = [ + { + type: 'text', + text: 'Original text', + }, + { + type: 'image_url', + image_url: { + url: 'data:image/png;base64,ABC123', + }, + }, + ]; + + const messages = [ + new HumanMessage({ + content: [...originalContent], + }), + ]; + + await client.runMemory(messages); + + // Verify original message wasn't mutated + expect(messages[0].content).toHaveLength(2); + expect(messages[0].content[1].type).toBe('image_url'); + expect(messages[0].content[1].image_url.url).toBe('data:image/png;base64,ABC123'); + }); + + it('should handle message window size correctly', async () => { + const { HumanMessage, AIMessage } = require('@langchain/core/messages'); + const messages = [ + new HumanMessage('Message 1'), + new AIMessage('Response 1'), + new HumanMessage('Message 2'), + new AIMessage('Response 2'), + new HumanMessage('Message 3'), + new AIMessage('Response 3'), + ]; + + // Window size is set to 3 in mockReq + await client.runMemory(messages); + + expect(mockProcessMemory).toHaveBeenCalledTimes(1); + const processedMessage = mockProcessMemory.mock.calls[0][0][0]; + + // Should only include last 3 messages due to window size + expect(processedMessage.content).toContain('Message 3'); + expect(processedMessage.content).toContain('Response 3'); + expect(processedMessage.content).not.toContain('Message 1'); + expect(processedMessage.content).not.toContain('Response 1'); + }); + + it('should return early if processMemory is not set', async () => { + const { HumanMessage } = require('@langchain/core/messages'); + client.processMemory = null; + + const result = await client.runMemory([new HumanMessage('Test')]); + + expect(result).toBeUndefined(); + expect(mockProcessMemory).not.toHaveBeenCalled(); + }); + }); });