diff --git a/client/src/components/Chat/Messages/Content/ToolCallInfo.tsx b/client/src/components/Chat/Messages/Content/ToolCallInfo.tsx index 7a7930bba5..1cc472f834 100644 --- a/client/src/components/Chat/Messages/Content/ToolCallInfo.tsx +++ b/client/src/components/Chat/Messages/Content/ToolCallInfo.tsx @@ -1,7 +1,7 @@ import React from 'react'; import { useLocalize } from '~/hooks'; import { UIResourceRenderer } from '@mcp-ui/client'; -import UIResourceGrid from './UIResourceGrid'; +import UIResourceCarousel from './UIResourceCarousel'; import type { UIResource } from '~/common'; function OptimizedCodeBlock({ text, maxHeight = 320 }: { text: string; maxHeight?: number }) { @@ -57,16 +57,21 @@ export default function ToolCallInfo({ // Extract ui_resources from the output to display them in the UI let uiResources: UIResource[] = []; if (output?.includes('ui_resources')) { - const parsedOutput = JSON.parse(output); - const uiResourcesItem = parsedOutput.find( - (contentItem) => contentItem.metadata === 'ui_resources', - ); - if (uiResourcesItem?.text) { - uiResources = JSON.parse(atob(uiResourcesItem.text)) as UIResource[]; + try { + const parsedOutput = JSON.parse(output); + const uiResourcesItem = parsedOutput.find( + (contentItem) => contentItem.metadata?.type === 'ui_resources', + ); + if (uiResourcesItem?.metadata?.data) { + uiResources = uiResourcesItem.metadata.data; + output = JSON.stringify( + parsedOutput.filter((contentItem) => contentItem.metadata?.type !== 'ui_resources'), + ); + } + } catch (error) { + // If JSON parsing fails, keep original output + console.error('Failed to parse output:', error); } - output = JSON.stringify( - parsedOutput.filter((contentItem) => contentItem.metadata !== 'ui_resources'), - ); } return ( @@ -90,7 +95,7 @@ export default function ToolCallInfo({ )}
- {uiResources.length > 1 && } + {uiResources.length > 1 && } {uiResources.length === 1 && ( = React.memo(({ uiResources }) => { +const UIResourceCarousel: React.FC = React.memo(({ uiResources }) => { const [showLeftArrow, setShowLeftArrow] = useState(false); const [showRightArrow, setShowRightArrow] = useState(true); const [isContainerHovered, setIsContainerHovered] = useState(false); @@ -142,4 +142,4 @@ const UIResourceGrid: React.FC = React.memo(({ uiResources ); }); -export default UIResourceGrid; +export default UIResourceCarousel; diff --git a/client/src/components/Chat/Messages/Content/__tests__/ToolCallInfo.test.tsx b/client/src/components/Chat/Messages/Content/__tests__/ToolCallInfo.test.tsx new file mode 100644 index 0000000000..d612fb8e97 --- /dev/null +++ b/client/src/components/Chat/Messages/Content/__tests__/ToolCallInfo.test.tsx @@ -0,0 +1,273 @@ +import React from 'react'; +import { render, screen } from '@testing-library/react'; +import ToolCallInfo from '../ToolCallInfo'; +import { UIResourceRenderer } from '@mcp-ui/client'; +import UIResourceCarousel from '../UIResourceCarousel'; + +// Mock the dependencies +jest.mock('~/hooks', () => ({ + useLocalize: () => (key: string, values?: any) => { + const translations: Record = { + com_assistants_domain_info: `Used ${values?.[0]}`, + com_assistants_function_use: `Used ${values?.[0]}`, + com_assistants_action_attempt: `Attempted to use ${values?.[0]}`, + com_assistants_attempt_info: 'Attempted to use function', + com_ui_result: 'Result', + com_ui_ui_resources: 'UI Resources', + }; + return translations[key] || key; + }, +})); + +jest.mock('@mcp-ui/client', () => ({ + UIResourceRenderer: jest.fn(() => null), +})); + +jest.mock('../UIResourceCarousel', () => ({ + __esModule: true, + default: jest.fn(() => null), +})); + +// Add TextEncoder/TextDecoder polyfill for Jest environment +import { TextEncoder, TextDecoder } from 'util'; + +if (typeof global.TextEncoder === 'undefined') { + global.TextEncoder = TextEncoder as any; + global.TextDecoder = TextDecoder as any; +} + +describe('ToolCallInfo', () => { + const mockProps = { + input: '{"test": "input"}', + function_name: 'testFunction', + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('ui_resources extraction', () => { + it('should extract single ui_resource from output', () => { + const uiResource = { + type: 'text', + data: 'Test resource', + }; + + const output = JSON.stringify([ + { type: 'text', text: 'Regular output' }, + { + metadata: { + type: 'ui_resources', + data: [uiResource], + }, + }, + ]); + + render(); + + // Should render UIResourceRenderer for single resource + expect(UIResourceRenderer).toHaveBeenCalledWith( + expect.objectContaining({ + resource: uiResource, + onUIAction: expect.any(Function), + htmlProps: { + autoResizeIframe: { width: true, height: true }, + }, + }), + expect.any(Object), + ); + + // Should not render carousel for single resource + expect(UIResourceCarousel).not.toHaveBeenCalled(); + }); + + it('should extract multiple ui_resources from output', () => { + const uiResources = [ + { type: 'text', data: 'Resource 1' }, + { type: 'text', data: 'Resource 2' }, + { type: 'text', data: 'Resource 3' }, + ]; + + const output = JSON.stringify([ + { type: 'text', text: 'Regular output' }, + { + metadata: { + type: 'ui_resources', + data: uiResources, + }, + }, + ]); + + render(); + + // Should render carousel for multiple resources + expect(UIResourceCarousel).toHaveBeenCalledWith( + expect.objectContaining({ + uiResources, + }), + expect.any(Object), + ); + + // Should not render individual UIResourceRenderer + expect(UIResourceRenderer).not.toHaveBeenCalled(); + }); + + it('should filter out ui_resources from displayed output', () => { + const regularContent = [ + { type: 'text', text: 'Regular output 1' }, + { type: 'text', text: 'Regular output 2' }, + ]; + + const output = JSON.stringify([ + ...regularContent, + { + metadata: { + type: 'ui_resources', + data: [{ type: 'text', data: 'UI Resource' }], + }, + }, + ]); + + const { container } = render(); + + // Check that the displayed output doesn't contain ui_resources + const codeBlocks = container.querySelectorAll('code'); + const outputCode = codeBlocks[1]?.textContent; // Second code block is the output + + expect(outputCode).toContain('Regular output 1'); + expect(outputCode).toContain('Regular output 2'); + expect(outputCode).not.toContain('ui_resources'); + }); + + it('should handle output without ui_resources', () => { + const output = JSON.stringify([{ type: 'text', text: 'Regular output' }]); + + render(); + + expect(UIResourceRenderer).not.toHaveBeenCalled(); + expect(UIResourceCarousel).not.toHaveBeenCalled(); + }); + + it('should handle malformed ui_resources gracefully', () => { + const output = JSON.stringify([ + { + metadata: 'ui_resources', // metadata should be an object, not a string + text: 'some text content', + }, + ]); + + // Component should not throw error and should render without UI resources + const { container } = render(); + + // Should render the component without crashing + expect(container).toBeTruthy(); + + // UIResourceCarousel should not be called since the metadata structure is invalid + expect(UIResourceCarousel).not.toHaveBeenCalled(); + }); + + it('should handle ui_resources as plain text without breaking', () => { + const outputWithTextOnly = + 'This output contains ui_resources as plain text but not as a proper structure'; + + render(); + + // Should render normally without errors + expect(screen.getByText(`Used ${mockProps.function_name}`)).toBeInTheDocument(); + expect(screen.getByText('Result')).toBeInTheDocument(); + + // The output text should be displayed in a code block + const codeBlocks = screen.getAllByText((content, element) => { + return element?.tagName === 'CODE' && content.includes(outputWithTextOnly); + }); + expect(codeBlocks.length).toBeGreaterThan(0); + + // Should not render UI resources components + expect(UIResourceRenderer).not.toHaveBeenCalled(); + expect(UIResourceCarousel).not.toHaveBeenCalled(); + }); + }); + + describe('rendering logic', () => { + it('should render UI Resources heading when ui_resources exist', () => { + const output = JSON.stringify([ + { + metadata: { + type: 'ui_resources', + data: [{ type: 'text', data: 'Test' }], + }, + }, + ]); + + render(); + + expect(screen.getByText('UI Resources')).toBeInTheDocument(); + }); + + it('should not render UI Resources heading when no ui_resources', () => { + const output = JSON.stringify([{ type: 'text', text: 'Regular output' }]); + + render(); + + expect(screen.queryByText('UI Resources')).not.toBeInTheDocument(); + }); + + it('should pass correct props to UIResourceRenderer', () => { + const uiResource = { + type: 'form', + data: { fields: [{ name: 'test', type: 'text' }] }, + }; + + const output = JSON.stringify([ + { + metadata: { + type: 'ui_resources', + data: [uiResource], + }, + }, + ]); + + render(); + + expect(UIResourceRenderer).toHaveBeenCalledWith( + expect.objectContaining({ + resource: uiResource, + onUIAction: expect.any(Function), + htmlProps: { + autoResizeIframe: { width: true, height: true }, + }, + }), + expect.any(Object), + ); + }); + + it('should console.log when UIAction is triggered', async () => { + const consoleSpy = jest.spyOn(console, 'log').mockImplementation(); + + const output = JSON.stringify([ + { + metadata: { + type: 'ui_resources', + data: [{ type: 'text', data: 'Test' }], + }, + }, + ]); + + render(); + + const mockUIResourceRenderer = UIResourceRenderer as jest.MockedFunction< + typeof UIResourceRenderer + >; + const onUIAction = mockUIResourceRenderer.mock.calls[0]?.[0]?.onUIAction; + const testResult = { action: 'submit', data: { test: 'value' } }; + + if (onUIAction) { + await onUIAction(testResult as any); + } + + expect(consoleSpy).toHaveBeenCalledWith('Action:', testResult); + + consoleSpy.mockRestore(); + }); + }); +}); diff --git a/client/src/components/Chat/Messages/Content/__tests__/UIResourceCarousel.test.tsx b/client/src/components/Chat/Messages/Content/__tests__/UIResourceCarousel.test.tsx new file mode 100644 index 0000000000..faeddd1e0c --- /dev/null +++ b/client/src/components/Chat/Messages/Content/__tests__/UIResourceCarousel.test.tsx @@ -0,0 +1,219 @@ +import React from 'react'; +import { render, screen, fireEvent, waitFor } from '@testing-library/react'; +import '@testing-library/jest-dom'; +import UIResourceCarousel from '../UIResourceCarousel'; +import type { UIResource } from '~/common'; + +// Mock the UIResourceRenderer component +jest.mock('@mcp-ui/client', () => ({ + UIResourceRenderer: ({ resource, onUIAction }: any) => ( +
onUIAction({ action: 'test' })}> + {resource.text || 'UI Resource'} +
+ ), +})); + +// Mock scrollTo +const mockScrollTo = jest.fn(); +Object.defineProperty(HTMLElement.prototype, 'scrollTo', { + configurable: true, + value: mockScrollTo, +}); + +describe('UIResourceCarousel', () => { + const mockUIResources: UIResource[] = [ + { uri: 'resource1', mimeType: 'text/html', text: 'Resource 1' }, + { uri: 'resource2', mimeType: 'text/html', text: 'Resource 2' }, + { uri: 'resource3', mimeType: 'text/html', text: 'Resource 3' }, + { uri: 'resource4', mimeType: 'text/html', text: 'Resource 4' }, + { uri: 'resource5', mimeType: 'text/html', text: 'Resource 5' }, + ]; + + beforeEach(() => { + jest.clearAllMocks(); + // Reset scroll properties + Object.defineProperty(HTMLElement.prototype, 'scrollLeft', { + configurable: true, + value: 0, + }); + Object.defineProperty(HTMLElement.prototype, 'scrollWidth', { + configurable: true, + value: 1000, + }); + Object.defineProperty(HTMLElement.prototype, 'clientWidth', { + configurable: true, + value: 500, + }); + }); + + it('renders nothing when no resources provided', () => { + const { container } = render(); + expect(container.firstChild).toBeNull(); + }); + + it('renders all UI resources', () => { + render(); + const renderers = screen.getAllByTestId('ui-resource-renderer'); + expect(renderers).toHaveLength(5); + expect(screen.getByText('Resource 1')).toBeInTheDocument(); + expect(screen.getByText('Resource 5')).toBeInTheDocument(); + }); + + it('shows/hides navigation arrows on hover', async () => { + const { container } = render(); + const carouselContainer = container.querySelector('.relative.mb-4.pt-3'); + + // Initially arrows should be hidden (opacity-0) + const leftArrow = screen.queryByLabelText('Scroll left'); + const rightArrow = screen.queryByLabelText('Scroll right'); + + // Right arrow should exist but left should not (at start) + expect(leftArrow).not.toBeInTheDocument(); + expect(rightArrow).toBeInTheDocument(); + expect(rightArrow).toHaveClass('opacity-0'); + + // Hover over container + fireEvent.mouseEnter(carouselContainer!); + await waitFor(() => { + expect(rightArrow).toHaveClass('opacity-100'); + }); + + // Leave hover + fireEvent.mouseLeave(carouselContainer!); + await waitFor(() => { + expect(rightArrow).toHaveClass('opacity-0'); + }); + }); + + it('handles scroll navigation', async () => { + const { container } = render(); + const scrollContainer = container.querySelector('.hide-scrollbar'); + + // Simulate being scrolled to show left arrow + Object.defineProperty(scrollContainer, 'scrollLeft', { + configurable: true, + value: 200, + }); + + // Trigger scroll event + fireEvent.scroll(scrollContainer!); + + // Both arrows should now be visible + await waitFor(() => { + expect(screen.getByLabelText('Scroll left')).toBeInTheDocument(); + expect(screen.getByLabelText('Scroll right')).toBeInTheDocument(); + }); + + // Hover to make arrows interactive + const carouselContainer = container.querySelector('.relative.mb-4.pt-3'); + fireEvent.mouseEnter(carouselContainer!); + + // Click right arrow + fireEvent.click(screen.getByLabelText('Scroll right')); + expect(mockScrollTo).toHaveBeenCalledWith({ + left: 650, // 200 + (500 * 0.9) + behavior: 'smooth', + }); + + // Click left arrow + fireEvent.click(screen.getByLabelText('Scroll left')); + expect(mockScrollTo).toHaveBeenCalledWith({ + left: -250, // 200 - (500 * 0.9) + behavior: 'smooth', + }); + }); + + it('hides right arrow when scrolled to end', async () => { + const { container } = render(); + const scrollContainer = container.querySelector('.hide-scrollbar'); + + // Simulate scrolled to end + Object.defineProperty(scrollContainer, 'scrollLeft', { + configurable: true, + value: 490, // scrollWidth - clientWidth - 10 + }); + + fireEvent.scroll(scrollContainer!); + + await waitFor(() => { + expect(screen.getByLabelText('Scroll left')).toBeInTheDocument(); + expect(screen.queryByLabelText('Scroll right')).not.toBeInTheDocument(); + }); + }); + + it('handles UIResource actions', async () => { + const consoleSpy = jest.spyOn(console, 'log').mockImplementation(); + render(); + + const renderer = screen.getByTestId('ui-resource-renderer'); + fireEvent.click(renderer); + + await waitFor(() => { + expect(consoleSpy).toHaveBeenCalledWith('Action:', { action: 'test' }); + }); + + consoleSpy.mockRestore(); + }); + + it('applies correct dimensions to resource containers', () => { + render(); + const containers = screen + .getAllByTestId('ui-resource-renderer') + .map((el) => el.parentElement?.parentElement); + + containers.forEach((container, index) => { + expect(container).toHaveStyle({ + width: '230px', + minHeight: '360px', + animationDelay: `${index * 100}ms`, + }); + }); + }); + + it('shows correct gradient overlays based on scroll position', () => { + const { container } = render(); + + // At start, left gradient should be hidden, right should be visible + const leftGradient = container.querySelector('.bg-gradient-to-r'); + const rightGradient = container.querySelector('.bg-gradient-to-l'); + + expect(leftGradient).toHaveClass('opacity-0'); + expect(rightGradient).toHaveClass('opacity-100'); + }); + + it('cleans up event listeners on unmount', () => { + const { container, unmount } = render(); + const scrollContainer = container.querySelector('.hide-scrollbar'); + + const removeEventListenerSpy = jest.spyOn(scrollContainer!, 'removeEventListener'); + + unmount(); + + expect(removeEventListenerSpy).toHaveBeenCalledWith('scroll', expect.any(Function)); + }); + + it('renders with animation delays for each resource', () => { + render(); + const resourceContainers = screen + .getAllByTestId('ui-resource-renderer') + .map((el) => el.parentElement?.parentElement); + + resourceContainers.forEach((container, index) => { + expect(container).toHaveStyle({ + animationDelay: `${index * 100}ms`, + }); + }); + }); + + it('memoizes component properly', () => { + const { rerender } = render(); + const firstRender = screen.getAllByTestId('ui-resource-renderer'); + + // Re-render with same props + rerender(); + const secondRender = screen.getAllByTestId('ui-resource-renderer'); + + // Component should not re-render with same props (React.memo) + expect(firstRender.length).toBe(secondRender.length); + }); +}); diff --git a/packages/api/src/mcp/__tests__/parsers.test.ts b/packages/api/src/mcp/__tests__/parsers.test.ts new file mode 100644 index 0000000000..de3d4cd70b --- /dev/null +++ b/packages/api/src/mcp/__tests__/parsers.test.ts @@ -0,0 +1,417 @@ +import { formatToolContent } from '../parsers'; +import type * as t from '../types'; + +describe('formatToolContent', () => { + describe('unrecognized providers', () => { + it('should return string for unrecognized provider', () => { + const result: t.MCPToolCallResponse = { + content: [ + { type: 'text', text: 'Hello world' }, + { type: 'text', text: 'Another text' }, + ], + }; + + const [content, artifacts] = formatToolContent(result, 'unknown' as t.Provider); + expect(content).toBe('Hello world\n\nAnother text'); + expect(artifacts).toBeUndefined(); + }); + + it('should return "(No response)" for empty content with unrecognized provider', () => { + const result: t.MCPToolCallResponse = { content: [] }; + const [content, artifacts] = formatToolContent(result, 'unknown' as t.Provider); + expect(content).toBe('(No response)'); + expect(artifacts).toBeUndefined(); + }); + + it('should return "(No response)" for undefined result with unrecognized provider', () => { + const result: t.MCPToolCallResponse = undefined; + const [content, artifacts] = formatToolContent(result, 'unknown' as t.Provider); + expect(content).toBe('(No response)'); + expect(artifacts).toBeUndefined(); + }); + }); + + describe('recognized providers - content array providers', () => { + const contentArrayProviders: t.Provider[] = ['google', 'anthropic', 'openai', 'azureopenai']; + + contentArrayProviders.forEach((provider) => { + describe(`${provider} provider`, () => { + it('should format text content as content array', () => { + const result: t.MCPToolCallResponse = { + content: [ + { type: 'text', text: 'First text' }, + { type: 'text', text: 'Second text' }, + ], + }; + + const [content, artifacts] = formatToolContent(result, provider); + expect(content).toEqual([{ type: 'text', text: 'First text\n\nSecond text' }]); + expect(artifacts).toBeUndefined(); + }); + + it('should separate text blocks when images are present', () => { + const result: t.MCPToolCallResponse = { + content: [ + { type: 'text', text: 'Before image' }, + { type: 'image', data: 'base64data', mimeType: 'image/png' }, + { type: 'text', text: 'After image' }, + ], + }; + + const [content, artifacts] = formatToolContent(result, provider); + expect(content).toEqual([ + { type: 'text', text: 'Before image' }, + { type: 'text', text: 'After image' }, + ]); + expect(artifacts).toEqual({ + content: [ + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,base64data' }, + }, + ], + }); + }); + + it('should handle empty content', () => { + const result: t.MCPToolCallResponse = { content: [] }; + const [content, artifacts] = formatToolContent(result, provider); + expect(content).toEqual([{ type: 'text', text: '(No response)' }]); + expect(artifacts).toBeUndefined(); + }); + }); + }); + }); + + describe('recognized providers - string providers', () => { + const stringProviders: t.Provider[] = ['openrouter', 'xai', 'deepseek', 'ollama', 'bedrock']; + + stringProviders.forEach((provider) => { + describe(`${provider} provider`, () => { + it('should format content as string', () => { + const result: t.MCPToolCallResponse = { + content: [ + { type: 'text', text: 'First text' }, + { type: 'text', text: 'Second text' }, + ], + }; + + const [content, artifacts] = formatToolContent(result, provider); + expect(content).toBe('First text\n\nSecond text'); + expect(artifacts).toBeUndefined(); + }); + + it('should handle images with string output', () => { + const result: t.MCPToolCallResponse = { + content: [ + { type: 'text', text: 'Some text' }, + { type: 'image', data: 'base64data', mimeType: 'image/png' }, + ], + }; + + const [content, artifacts] = formatToolContent(result, provider); + expect(content).toBe('Some text'); + expect(artifacts).toEqual({ + content: [ + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,base64data' }, + }, + ], + }); + }); + }); + }); + }); + + describe('image handling', () => { + it('should handle images with http URLs', () => { + const result: t.MCPToolCallResponse = { + content: [{ type: 'image', data: 'https://example.com/image.png', mimeType: 'image/png' }], + }; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const [_content, artifacts] = formatToolContent(result, 'openai'); + expect(artifacts).toEqual({ + content: [ + { + type: 'image_url', + image_url: { url: 'https://example.com/image.png' }, + }, + ], + }); + }); + + it('should handle images with base64 data', () => { + const result: t.MCPToolCallResponse = { + content: [{ type: 'image', data: 'iVBORw0KGgoAAAA...', mimeType: 'image/png' }], + }; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const [_content, artifacts] = formatToolContent(result, 'openai'); + expect(artifacts).toEqual({ + content: [ + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,iVBORw0KGgoAAAA...' }, + }, + ], + }); + }); + }); + + describe('resource handling', () => { + it('should handle UI resources', () => { + const result: t.MCPToolCallResponse = { + content: [ + { + type: 'resource', + resource: { + uri: 'ui://carousel', + mimeType: 'application/json', + text: '{"items": []}', + name: 'carousel', + description: 'A carousel component', + }, + }, + ], + }; + + const [content, artifacts] = formatToolContent(result, 'openai'); + expect(content).toEqual([ + { + type: 'text', + text: '', + metadata: { + type: 'ui_resources', + data: [ + { + uri: 'ui://carousel', + mimeType: 'application/json', + text: '{"items": []}', + name: 'carousel', + description: 'A carousel component', + }, + ], + }, + }, + ]); + expect(artifacts).toBeUndefined(); + }); + + it('should handle regular resources', () => { + const result: t.MCPToolCallResponse = { + content: [ + { + type: 'resource', + resource: { + uri: 'file://document.pdf', + name: 'Document', + description: 'Important document', + mimeType: 'application/pdf', + text: 'Document content', + }, + }, + ], + }; + + const [content, artifacts] = formatToolContent(result, 'openai'); + expect(content).toEqual([ + { + type: 'text', + text: + 'Resource Text: Document content\n' + + 'Resource URI: file://document.pdf\n' + + 'Resource: Document\n' + + 'Resource Description: Important document\n' + + 'Resource MIME Type: application/pdf', + }, + ]); + expect(artifacts).toBeUndefined(); + }); + + it('should handle resources with partial data', () => { + const result: t.MCPToolCallResponse = { + content: [ + { + type: 'resource', + resource: { + uri: 'https://example.com/resource', + name: 'Example Resource', + text: '', + }, + }, + ], + }; + + const [content, artifacts] = formatToolContent(result, 'openai'); + expect(content).toEqual([ + { + type: 'text', + text: 'Resource URI: https://example.com/resource\n' + 'Resource: Example Resource', + }, + ]); + expect(artifacts).toBeUndefined(); + }); + + it('should handle mixed UI and regular resources', () => { + const result: t.MCPToolCallResponse = { + content: [ + { type: 'text', text: 'Some text' }, + { + type: 'resource', + resource: { + uri: 'ui://button', + mimeType: 'application/json', + text: '{"label": "Click me"}', + }, + }, + { + type: 'resource', + resource: { + uri: 'file://data.csv', + name: 'Data file', + text: '', + }, + }, + ], + }; + + const [content, artifacts] = formatToolContent(result, 'openai'); + expect(content).toEqual([ + { + type: 'text', + text: 'Some text\n\n' + 'Resource URI: file://data.csv\n' + 'Resource: Data file', + }, + { + type: 'text', + text: '', + metadata: { + type: 'ui_resources', + data: [ + { + uri: 'ui://button', + mimeType: 'application/json', + text: '{"label": "Click me"}', + }, + ], + }, + }, + ]); + expect(artifacts).toBeUndefined(); + }); + }); + + describe('unknown content types', () => { + it('should stringify unknown content types', () => { + const result: t.MCPToolCallResponse = { + content: [ + { type: 'text', text: 'Normal text' }, + { type: 'unknown', data: 'some data' } as unknown as t.ToolContentPart, + ], + }; + + const [content, artifacts] = formatToolContent(result, 'openai'); + expect(content).toEqual([ + { + type: 'text', + text: 'Normal text\n\n' + JSON.stringify({ type: 'unknown', data: 'some data' }, null, 2), + }, + ]); + expect(artifacts).toBeUndefined(); + }); + }); + + describe('complex scenarios', () => { + it('should handle mixed content with all types', () => { + const result: t.MCPToolCallResponse = { + content: [ + { type: 'text', text: 'Introduction' }, + { type: 'image', data: 'image1.png', mimeType: 'image/png' }, + { type: 'text', text: 'Middle section' }, + { + type: 'resource', + resource: { + uri: 'ui://chart', + mimeType: 'application/json', + text: '{"type": "bar"}', + }, + }, + { + type: 'resource', + resource: { + uri: 'https://api.example.com/data', + name: 'API Data', + description: 'External data source', + text: '', + }, + }, + { type: 'image', data: 'https://example.com/image2.jpg', mimeType: 'image/jpeg' }, + { type: 'text', text: 'Conclusion' }, + ], + }; + + const [content, artifacts] = formatToolContent(result, 'anthropic'); + expect(content).toEqual([ + { type: 'text', text: 'Introduction' }, + { + type: 'text', + text: + 'Middle section\n\n' + + 'Resource URI: https://api.example.com/data\n' + + 'Resource: API Data\n' + + 'Resource Description: External data source', + }, + { type: 'text', text: 'Conclusion' }, + { + type: 'text', + text: '', + metadata: { + type: 'ui_resources', + data: [ + { + uri: 'ui://chart', + mimeType: 'application/json', + text: '{"type": "bar"}', + }, + ], + }, + }, + ]); + expect(artifacts).toEqual({ + content: [ + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,image1.png' }, + }, + { + type: 'image_url', + image_url: { url: 'https://example.com/image2.jpg' }, + }, + ], + }); + }); + + it('should handle error responses gracefully', () => { + const result: t.MCPToolCallResponse = { + content: [{ type: 'text', text: 'Error occurred' }], + isError: true, + }; + + const [content, artifacts] = formatToolContent(result, 'openai'); + expect(content).toEqual([{ type: 'text', text: 'Error occurred' }]); + expect(artifacts).toBeUndefined(); + }); + + it('should handle metadata in responses', () => { + const result: t.MCPToolCallResponse = { + _meta: { timestamp: Date.now(), source: 'test' }, + content: [{ type: 'text', text: 'Response with metadata' }], + }; + + const [content, artifacts] = formatToolContent(result, 'google'); + expect(content).toEqual([{ type: 'text', text: 'Response with metadata' }]); + expect(artifacts).toBeUndefined(); + }); + }); +}); diff --git a/packages/api/src/mcp/parsers.ts b/packages/api/src/mcp/parsers.ts index 3f6cc51e50..77af29bee9 100644 --- a/packages/api/src/mcp/parsers.ts +++ b/packages/api/src/mcp/parsers.ts @@ -111,7 +111,7 @@ export function formatToolContent( const formattedContent: t.FormattedContent[] = []; const imageUrls: t.FormattedContent[] = []; let currentTextBlock = ''; - let uiResources: t.UIResource[] = []; + const uiResources: t.UIResource[] = []; type ContentHandler = undefined | ((item: t.ToolContentPart) => void); @@ -183,7 +183,14 @@ export function formatToolContent( } if (uiResources.length) { - formattedContent.push({ type: 'text', metadata: 'ui_resources', text: btoa(JSON.stringify(uiResources))}); + formattedContent.push({ + type: 'text', + metadata: { + type: 'ui_resources', + data: uiResources, + }, + text: '', + }); } const artifacts = imageUrls.length ? { content: imageUrls } : undefined; diff --git a/packages/api/src/mcp/types/index.ts b/packages/api/src/mcp/types/index.ts index 0e5672ea6f..95086a3672 100644 --- a/packages/api/src/mcp/types/index.ts +++ b/packages/api/src/mcp/types/index.ts @@ -69,13 +69,25 @@ export type MCPToolCallResponse = isError?: boolean; }; -export type Provider = 'google' | 'anthropic' | 'openAI'; +export type Provider = + | 'google' + | 'anthropic' + | 'openai' + | 'azureopenai' + | 'openrouter' + | 'xai' + | 'deepseek' + | 'ollama' + | 'bedrock'; export type FormattedContent = | { type: 'text'; - text: string; - metadata?: string; + metadata?: { + type: string; + data: UIResource[]; + } + text?: string; } | { type: 'image'; @@ -109,7 +121,7 @@ export type UIResource = { mimeType: string; text: string; [key: string]: unknown; -} +}; export type ImageFormatter = (item: ImageContent) => FormattedContent;