diff --git a/packages/api/src/files/encode/document.spec.ts b/packages/api/src/files/encode/document.spec.ts index 3a955312c2..15c2e3b716 100644 --- a/packages/api/src/files/encode/document.spec.ts +++ b/packages/api/src/files/encode/document.spec.ts @@ -784,7 +784,7 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { }); describe('Generic document encoding path', () => { - it('should format text/plain for Anthropic with citations enabled', async () => { + it('should format text/plain for Anthropic as a plain-text document source', async () => { const req = createMockRequest(30) as ServerRequest; const file = createMockDocFile(1, 'text/plain', 'notes.txt'); @@ -806,9 +806,9 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { expect(result.documents[0]).toMatchObject({ type: 'document', source: { - type: 'base64', + type: 'text', media_type: 'text/plain', - data: mockContent, + data: 'plain text content', }, citations: { enabled: true }, context: 'File: "notes.txt"', @@ -816,7 +816,7 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { expect(result.files).toHaveLength(1); }); - it('should format text/html for Anthropic with citations enabled', async () => { + it('should format text/html for Anthropic as a plain-text document source', async () => { const req = createMockRequest(30) as ServerRequest; const file = createMockDocFile(1, 'text/html', 'page.html'); @@ -837,12 +837,12 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { expect(result.documents).toHaveLength(1); expect(result.documents[0]).toMatchObject({ type: 'document', - source: { type: 'base64', media_type: 'text/html', data: mockContent }, + source: { type: 'text', media_type: 'text/plain', data: 'content' }, citations: { enabled: true }, }); }); - it('should format application/json for Anthropic without citations', async () => { + it('should format application/json for Anthropic as a plain-text document source', async () => { const req = createMockRequest(30) as ServerRequest; const file = createMockDocFile(1, 'application/json', 'data.json'); @@ -861,7 +861,59 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { ); expect(result.documents).toHaveLength(1); - expect(result.documents[0]).not.toHaveProperty('citations'); + expect(result.documents[0]).toMatchObject({ + type: 'document', + source: { type: 'text', media_type: 'text/plain', data: '{"key":"value"}' }, + citations: { enabled: true }, + }); + }); + + it('should skip non-PDF binary documents for Anthropic without contacting storage', async () => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockDocFile( + 1, + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'report.docx', + ); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.ANTHROPIC }, + mockStrategyFunctions, + ); + + expect(result.documents).toHaveLength(0); + expect(result.files).toHaveLength(0); + expect(mockedGetFileStream).not.toHaveBeenCalled(); + }); + + it('should still encode supported Anthropic documents when mixed with unsupported ones', async () => { + const req = createMockRequest(30) as ServerRequest; + const docxFile = createMockDocFile(1, 'application/vnd.ms-excel', 'sheet.xls'); + const textFile = createMockDocFile(1, 'text/markdown', 'readme.md'); + + const mockContent = Buffer.from('# heading').toString('base64'); + mockedGetFileStream.mockResolvedValue({ + file: textFile, + content: mockContent, + metadata: textFile, + }); + + const result = await encodeAndFormatDocuments( + req, + [docxFile, textFile], + { provider: Providers.ANTHROPIC }, + mockStrategyFunctions, + ); + + expect(result.documents).toHaveLength(1); + expect(result.documents[0]).toMatchObject({ + type: 'document', + source: { type: 'text', media_type: 'text/plain', data: '# heading' }, + }); + expect(result.files).toHaveLength(1); + expect(mockedGetFileStream).toHaveBeenCalledTimes(1); }); it('should format text/csv for OpenAI responses API', async () => { diff --git a/packages/api/src/files/encode/document.ts b/packages/api/src/files/encode/document.ts index e83abe30db..7ef4b684e5 100644 --- a/packages/api/src/files/encode/document.ts +++ b/packages/api/src/files/encode/document.ts @@ -3,7 +3,9 @@ import { isOpenAILikeProvider, isBedrockDocumentType, bedrockDocumentFormats, + isAnthropicDocumentType, isDocumentSupportedProvider, + isAnthropicTextDocumentType, } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { @@ -17,12 +19,29 @@ import { validatePdf, validateBedrockDocument } from '~/files/validation'; import { getFileStream, getConfiguredFileSizeLimit } from './utils'; import { runGuardedEncode } from './memoryGuard'; -const ANTHROPIC_CITATION_TYPES = new Set([ - 'application/pdf', - 'text/plain', - 'text/html', - 'text/markdown', -]); +/** Anthropic only accepts PDFs as base64 documents; textual types must use a text source */ +function getAnthropicDocumentSource( + mimeType: string, + content: string, +): AnthropicDocumentBlock['source'] | null { + if (isAnthropicTextDocumentType(mimeType)) { + return { + type: 'text', + media_type: 'text/plain', + data: Buffer.from(content, 'base64').toString('utf8'), + }; + } + + if (mimeType === 'application/pdf') { + return { + type: 'base64', + media_type: mimeType, + data: content, + }; + } + + return null; +} /** * Formats a base64-encoded document into the appropriate provider-specific block. @@ -36,19 +55,17 @@ function formatDocumentBlock( useResponsesApi: boolean | undefined, ): DocumentBlock | null { if (provider === Providers.ANTHROPIC) { + const source = getAnthropicDocumentSource(mimeType, content); + if (!source) { + return null; + } + const document: AnthropicDocumentBlock = { type: 'document', - source: { - type: 'base64', - media_type: mimeType, - data: content, - }, + source, + citations: { enabled: true }, }; - if (ANTHROPIC_CITATION_TYPES.has(mimeType)) { - document.citations = { enabled: true }; - } - if (filename) { document.context = `File: "${filename}"`; } @@ -87,6 +104,39 @@ function formatDocumentBlock( return null; } +/** + * Filters out files the provider's document path cannot send to the model. + * Anthropic rejects non-PDF base64 documents with a 400 that recurs on every + * retry, so unsupported types are skipped instead of bricking the conversation. + */ +function filterProviderDocumentFiles(provider: Providers, files: IMongoFile[]): IMongoFile[] { + if (provider === Providers.BEDROCK) { + return files.filter((file) => isBedrockDocumentType(file.type)); + } + + if (provider !== Providers.ANTHROPIC) { + return files; + } + + const processable: IMongoFile[] = []; + const skipped: string[] = []; + for (const file of files) { + if (isAnthropicDocumentType(file.type)) { + processable.push(file); + } else { + skipped.push(`"${file.filename}" (${file.type})`); + } + } + + if (skipped.length) { + console.warn( + `Skipping attachment(s) unsupported by Anthropic document input: ${skipped.join(', ')}`, + ); + } + + return processable; +} + function getBase64DecodedByteCount(content: string): number { let paddingChars = 0; @@ -106,6 +156,8 @@ function getBase64DecodedByteCount(content: string): number { * (e.g., via `supportedMimeTypes` in `processAttachments`). This function processes * every file it receives and dispatches to the appropriate provider format: * - **Bedrock**: Only encodes types in `bedrockDocumentFormats`; all others are skipped. + * - **Anthropic**: Only encodes PDFs (base64 source) and textual types (plain-text source); + * all others are skipped. * - **PDF**: Validated via `validatePdf` before encoding. * - **Generic types**: Encoded with a provider-specific size check. */ @@ -130,9 +182,7 @@ export async function encodeAndFormatDocuments( return result; } - const processableFiles = isBedrock - ? files.filter((file) => isBedrockDocumentType(file.type)) - : files; + const processableFiles = filterProviderDocumentFiles(provider, files); if (!processableFiles.length) { return result; diff --git a/packages/data-provider/src/file-config.spec.ts b/packages/data-provider/src/file-config.spec.ts index b12666f865..3fb851a86c 100644 --- a/packages/data-provider/src/file-config.spec.ts +++ b/packages/data-provider/src/file-config.spec.ts @@ -2,8 +2,10 @@ import type { MimeUploadCapability } from './file-config'; import type { FileConfig } from './types/files'; import { fileConfig as baseFileConfig, + isAnthropicTextDocumentType, getConfiguredMimeAccept, bedrockDocumentMimeTypes, + isAnthropicDocumentType, isPermissiveMimeConfig, convertStringsToRegex, documentParserMimeTypes, @@ -218,6 +220,47 @@ describe('documentParserMimeTypes', () => { }); }); +describe('isAnthropicDocumentType', () => { + it.each([ + 'application/pdf', + 'text/plain', + 'text/html', + 'text/markdown', + 'text/csv', + 'text/x-python', + 'application/json', + 'application/xml', + 'application/yaml', + 'application/sql', + 'application/csv', + ])('accepts type the Anthropic document path can send: %s', (mimeType) => { + expect(isAnthropicDocumentType(mimeType)).toBe(true); + }); + + it.each([ + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + 'application/msword', + 'application/vnd.ms-excel', + 'application/epub+zip', + 'application/zip', + 'application/vnd.apache.parquet', + 'image/png', + '', + ])('rejects type Anthropic would 400 on: %s', (mimeType) => { + expect(isAnthropicDocumentType(mimeType)).toBe(false); + }); + + it('rejects undefined', () => { + expect(isAnthropicDocumentType(undefined)).toBe(false); + }); + + it('excludes PDF from the plain-text subset', () => { + expect(isAnthropicTextDocumentType('application/pdf')).toBe(false); + expect(isAnthropicTextDocumentType('text/plain')).toBe(true); + }); +}); + describe('getEndpointFileConfig', () => { describe('custom endpoint lookup', () => { it('should find custom endpoint by direct lookup', () => { diff --git a/packages/data-provider/src/file-config.ts b/packages/data-provider/src/file-config.ts index a6cf478c15..ed9a92ff39 100644 --- a/packages/data-provider/src/file-config.ts +++ b/packages/data-provider/src/file-config.ts @@ -184,6 +184,32 @@ export const bedrockDocumentMimeTypes: readonly string[] = Object.keys(bedrockDo export const bedrockDocumentExtensions = '.pdf,.csv,.doc,.docx,.xls,.xlsx,.html,.htm,.txt,.md,application/pdf,text/csv,application/csv,application/msword,application/vnd.openxmlformats-officedocument.wordprocessingml.document,application/vnd.ms-excel,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,text/html,text/plain,text/markdown'; +/** Textual `application/*` MIME types that can be decoded and sent as plain text */ +const textualApplicationTypes = new Set([ + 'application/json', + 'application/xml', + 'application/yaml', + 'application/sql', + 'application/typescript', + 'application/x-sh', + 'application/csv', +]); + +/** + * MIME types the Anthropic Messages API accepts as a plain-text document source + * (`source.type: 'text'`) + */ +export const isAnthropicTextDocumentType = (mimeType?: string): boolean => + mimeType != null && (mimeType.startsWith('text/') || textualApplicationTypes.has(mimeType)); + +/** + * MIME types the Anthropic Messages API document path can send to the model + * (mirrors `isBedrockDocumentType`): PDF via base64, textual types via a + * plain-text document source. All other types are rejected with a provider 400. + */ +export const isAnthropicDocumentType = (mimeType?: string): boolean => + mimeType === 'application/pdf' || isAnthropicTextDocumentType(mimeType); + export const excelMimeTypes = /^application\/(vnd\.ms-excel|msexcel|x-msexcel|x-ms-excel|x-excel|x-dos_ms_excel|xls|x-xls|vnd\.openxmlformats-officedocument\.spreadsheetml\.sheet)$/;