diff --git a/packages/api/src/files/encode/document.spec.ts b/packages/api/src/files/encode/document.spec.ts index 295bafe769..f58d591ed6 100644 --- a/packages/api/src/files/encode/document.spec.ts +++ b/packages/api/src/files/encode/document.spec.ts @@ -909,6 +909,53 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { expect(mockedGetFileStream).not.toHaveBeenCalled(); }); + it('should send textual documents as text parts to Claude through an OpenAI-compatible gateway', async () => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockDocFile(1, 'text/plain', 'notes.txt'); + mockedGetFileStream.mockResolvedValue({ + file, + content: Buffer.from('hello from test').toString('base64'), + metadata: file, + }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI, model: 'claude-auto-latest' }, + mockStrategyFunctions, + ); + + expect(result.documents).toEqual([ + { type: 'text', text: 'File: "notes.txt"\n\nhello from test' }, + ]); + expect(result.files).toHaveLength(1); + }); + + it('should keep textual documents as file parts for non-Claude OpenAI models', async () => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockDocFile(1, 'text/html', 'page.html'); + const mockContent = Buffer.from('

hi

').toString('base64'); + mockedGetFileStream.mockResolvedValue({ + file, + content: mockContent, + metadata: file, + }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI, model: 'gemini-auto-latest' }, + mockStrategyFunctions, + ); + + expect(result.documents).toEqual([ + { + type: 'file', + file: { filename: 'page.html', file_data: `data:text/html;base64,${mockContent}` }, + }, + ]); + }); + it('should retain XLSX support for non-Claude OpenAI models', async () => { const req = createMockRequest(30) as ServerRequest; const file = createMockDocFile( diff --git a/packages/api/src/files/encode/document.ts b/packages/api/src/files/encode/document.ts index 4ad0780ca6..57ef72db9a 100644 --- a/packages/api/src/files/encode/document.ts +++ b/packages/api/src/files/encode/document.ts @@ -47,6 +47,17 @@ function getAnthropicDocumentSource( return null; } +/** + * Whether the model behind this provider is Claude, which accepts only PDFs as base64 + * documents. OpenAI-compatible gateways report an OpenAI-like provider for Claude models. + */ +function usesAnthropicDocumentCapabilities(provider: Providers, model?: string): boolean { + return ( + provider === Providers.ANTHROPIC || + (isOpenAILikeProvider(provider) && (model?.toLowerCase().includes('claude') ?? false)) + ); +} + /** * Formats a base64-encoded document into the appropriate provider-specific block. * Returns `null` when the provider has no matching handler. @@ -57,6 +68,7 @@ function formatDocumentBlock( content: string, filename: string | undefined, useResponsesApi: boolean | undefined, + model?: string, ): DocumentBlock | null { if (provider === Providers.ANTHROPIC) { const source = getAnthropicDocumentSource(mimeType, content); @@ -87,6 +99,19 @@ function formatDocumentBlock( const resolvedFilename = filename ?? 'document'; + /* A gateway translates an OpenAI `file` part into a base64 document with the file's own + * media type, which Claude rejects for anything but PDF. Send textual files as text. */ + if ( + !useResponsesApi && + isAnthropicTextDocumentType(mimeType) && + usesAnthropicDocumentCapabilities(provider, model) + ) { + return { + type: 'text', + text: `File: "${resolvedFilename}"\n\n${Buffer.from(content, 'base64').toString('utf8')}`, + }; + } + if (useResponsesApi) { return { type: 'input_file', @@ -123,11 +148,7 @@ function filterProviderDocumentFiles( return files.filter((file) => isBedrockDocumentType(file.type)); } - const usesAnthropicDocumentCapabilities = - provider === Providers.ANTHROPIC || - (isOpenAILikeProvider(provider) && model?.toLowerCase().includes('claude')); - - if (!usesAnthropicDocumentCapabilities) { + if (!usesAnthropicDocumentCapabilities(provider, model)) { return files; } @@ -283,6 +304,7 @@ export async function encodeAndFormatDocuments( content, file.filename, useResponsesApi, + model, ); if (block) { result.documents.push(block); @@ -302,6 +324,7 @@ export async function encodeAndFormatDocuments( content, file.filename, useResponsesApi, + model, ); if (block) { result.documents.push(block); diff --git a/packages/api/src/types/files.ts b/packages/api/src/types/files.ts index 4d606f2482..83c3bfc123 100644 --- a/packages/api/src/types/files.ts +++ b/packages/api/src/types/files.ts @@ -91,6 +91,12 @@ export interface OpenAIFileBlock { }; } +/** OpenAI-compatible text part carrying a textual document's decoded contents */ +export interface OpenAITextDocumentBlock { + type: 'text'; + text: string; +} + /** OpenAI Responses API file format */ export interface OpenAIInputFileBlock { type: 'input_file'; @@ -114,6 +120,7 @@ export type DocumentBlock = | AnthropicDocumentBlock | GoogleDocumentBlock | OpenAIFileBlock + | OpenAITextDocumentBlock | OpenAIInputFileBlock | BedrockDocumentBlock;