mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-08-04 14:57:42 +00:00
📄 fix: Filter Non-PDF Documents on the Anthropic Encode Path (#14535)
Anthropic's Messages API only accepts application/pdf for base64 document sources, but encodeAndFormatDocuments sent every allowlisted file (docx, xlsx, csv, html) through the base64 branch unfiltered. The provider 400 recurs on every retry because attachments are re-encoded each request, permanently breaking the conversation. - Add isAnthropicDocumentType / isAnthropicTextDocumentType to data-provider, mirroring isBedrockDocumentType - Filter unsupported types before encoding (matching Bedrock semantics) and log the skipped attachments - Send textual types as plain-text document sources (source.type 'text'), which Anthropic accepts and supports citations for, instead of invalid base64 blocks Fixes #14485
This commit is contained in:
Danny Avila
GitHub
parent
8e165eb451
commit
f0d3bcb622
4 changed files with 196 additions and 25 deletions
|
|
@ -784,7 +784,7 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => {
|
|||
});
|
||||
|
||||
describe('Generic document encoding path', () => {
|
||||
it('should format text/plain for Anthropic with citations enabled', async () => {
|
||||
it('should format text/plain for Anthropic as a plain-text document source', async () => {
|
||||
const req = createMockRequest(30) as ServerRequest;
|
||||
const file = createMockDocFile(1, 'text/plain', 'notes.txt');
|
||||
|
||||
|
|
@ -806,9 +806,9 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => {
|
|||
expect(result.documents[0]).toMatchObject({
|
||||
type: 'document',
|
||||
source: {
|
||||
type: 'base64',
|
||||
type: 'text',
|
||||
media_type: 'text/plain',
|
||||
data: mockContent,
|
||||
data: 'plain text content',
|
||||
},
|
||||
citations: { enabled: true },
|
||||
context: 'File: "notes.txt"',
|
||||
|
|
@ -816,7 +816,7 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => {
|
|||
expect(result.files).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('should format text/html for Anthropic with citations enabled', async () => {
|
||||
it('should format text/html for Anthropic as a plain-text document source', async () => {
|
||||
const req = createMockRequest(30) as ServerRequest;
|
||||
const file = createMockDocFile(1, 'text/html', 'page.html');
|
||||
|
||||
|
|
@ -837,12 +837,12 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => {
|
|||
expect(result.documents).toHaveLength(1);
|
||||
expect(result.documents[0]).toMatchObject({
|
||||
type: 'document',
|
||||
source: { type: 'base64', media_type: 'text/html', data: mockContent },
|
||||
source: { type: 'text', media_type: 'text/plain', data: '<html>content</html>' },
|
||||
citations: { enabled: true },
|
||||
});
|
||||
});
|
||||
|
||||
it('should format application/json for Anthropic without citations', async () => {
|
||||
it('should format application/json for Anthropic as a plain-text document source', async () => {
|
||||
const req = createMockRequest(30) as ServerRequest;
|
||||
const file = createMockDocFile(1, 'application/json', 'data.json');
|
||||
|
||||
|
|
@ -861,7 +861,59 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => {
|
|||
);
|
||||
|
||||
expect(result.documents).toHaveLength(1);
|
||||
expect(result.documents[0]).not.toHaveProperty('citations');
|
||||
expect(result.documents[0]).toMatchObject({
|
||||
type: 'document',
|
||||
source: { type: 'text', media_type: 'text/plain', data: '{"key":"value"}' },
|
||||
citations: { enabled: true },
|
||||
});
|
||||
});
|
||||
|
||||
it('should skip non-PDF binary documents for Anthropic without contacting storage', async () => {
|
||||
const req = createMockRequest(30) as ServerRequest;
|
||||
const file = createMockDocFile(
|
||||
1,
|
||||
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
'report.docx',
|
||||
);
|
||||
|
||||
const result = await encodeAndFormatDocuments(
|
||||
req,
|
||||
[file],
|
||||
{ provider: Providers.ANTHROPIC },
|
||||
mockStrategyFunctions,
|
||||
);
|
||||
|
||||
expect(result.documents).toHaveLength(0);
|
||||
expect(result.files).toHaveLength(0);
|
||||
expect(mockedGetFileStream).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should still encode supported Anthropic documents when mixed with unsupported ones', async () => {
|
||||
const req = createMockRequest(30) as ServerRequest;
|
||||
const docxFile = createMockDocFile(1, 'application/vnd.ms-excel', 'sheet.xls');
|
||||
const textFile = createMockDocFile(1, 'text/markdown', 'readme.md');
|
||||
|
||||
const mockContent = Buffer.from('# heading').toString('base64');
|
||||
mockedGetFileStream.mockResolvedValue({
|
||||
file: textFile,
|
||||
content: mockContent,
|
||||
metadata: textFile,
|
||||
});
|
||||
|
||||
const result = await encodeAndFormatDocuments(
|
||||
req,
|
||||
[docxFile, textFile],
|
||||
{ provider: Providers.ANTHROPIC },
|
||||
mockStrategyFunctions,
|
||||
);
|
||||
|
||||
expect(result.documents).toHaveLength(1);
|
||||
expect(result.documents[0]).toMatchObject({
|
||||
type: 'document',
|
||||
source: { type: 'text', media_type: 'text/plain', data: '# heading' },
|
||||
});
|
||||
expect(result.files).toHaveLength(1);
|
||||
expect(mockedGetFileStream).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should format text/csv for OpenAI responses API', async () => {
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@ import {
|
|||
isOpenAILikeProvider,
|
||||
isBedrockDocumentType,
|
||||
bedrockDocumentFormats,
|
||||
isAnthropicDocumentType,
|
||||
isDocumentSupportedProvider,
|
||||
isAnthropicTextDocumentType,
|
||||
} from 'librechat-data-provider';
|
||||
import type { IMongoFile } from '@librechat/data-schemas';
|
||||
import type {
|
||||
|
|
@ -17,12 +19,29 @@ import { validatePdf, validateBedrockDocument } from '~/files/validation';
|
|||
import { getFileStream, getConfiguredFileSizeLimit } from './utils';
|
||||
import { runGuardedEncode } from './memoryGuard';
|
||||
|
||||
const ANTHROPIC_CITATION_TYPES = new Set([
|
||||
'application/pdf',
|
||||
'text/plain',
|
||||
'text/html',
|
||||
'text/markdown',
|
||||
]);
|
||||
/** Anthropic only accepts PDFs as base64 documents; textual types must use a text source */
|
||||
function getAnthropicDocumentSource(
|
||||
mimeType: string,
|
||||
content: string,
|
||||
): AnthropicDocumentBlock['source'] | null {
|
||||
if (isAnthropicTextDocumentType(mimeType)) {
|
||||
return {
|
||||
type: 'text',
|
||||
media_type: 'text/plain',
|
||||
data: Buffer.from(content, 'base64').toString('utf8'),
|
||||
};
|
||||
}
|
||||
|
||||
if (mimeType === 'application/pdf') {
|
||||
return {
|
||||
type: 'base64',
|
||||
media_type: mimeType,
|
||||
data: content,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats a base64-encoded document into the appropriate provider-specific block.
|
||||
|
|
@ -36,19 +55,17 @@ function formatDocumentBlock(
|
|||
useResponsesApi: boolean | undefined,
|
||||
): DocumentBlock | null {
|
||||
if (provider === Providers.ANTHROPIC) {
|
||||
const source = getAnthropicDocumentSource(mimeType, content);
|
||||
if (!source) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const document: AnthropicDocumentBlock = {
|
||||
type: 'document',
|
||||
source: {
|
||||
type: 'base64',
|
||||
media_type: mimeType,
|
||||
data: content,
|
||||
},
|
||||
source,
|
||||
citations: { enabled: true },
|
||||
};
|
||||
|
||||
if (ANTHROPIC_CITATION_TYPES.has(mimeType)) {
|
||||
document.citations = { enabled: true };
|
||||
}
|
||||
|
||||
if (filename) {
|
||||
document.context = `File: "${filename}"`;
|
||||
}
|
||||
|
|
@ -87,6 +104,39 @@ function formatDocumentBlock(
|
|||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Filters out files the provider's document path cannot send to the model.
|
||||
* Anthropic rejects non-PDF base64 documents with a 400 that recurs on every
|
||||
* retry, so unsupported types are skipped instead of bricking the conversation.
|
||||
*/
|
||||
function filterProviderDocumentFiles(provider: Providers, files: IMongoFile[]): IMongoFile[] {
|
||||
if (provider === Providers.BEDROCK) {
|
||||
return files.filter((file) => isBedrockDocumentType(file.type));
|
||||
}
|
||||
|
||||
if (provider !== Providers.ANTHROPIC) {
|
||||
return files;
|
||||
}
|
||||
|
||||
const processable: IMongoFile[] = [];
|
||||
const skipped: string[] = [];
|
||||
for (const file of files) {
|
||||
if (isAnthropicDocumentType(file.type)) {
|
||||
processable.push(file);
|
||||
} else {
|
||||
skipped.push(`"${file.filename}" (${file.type})`);
|
||||
}
|
||||
}
|
||||
|
||||
if (skipped.length) {
|
||||
console.warn(
|
||||
`Skipping attachment(s) unsupported by Anthropic document input: ${skipped.join(', ')}`,
|
||||
);
|
||||
}
|
||||
|
||||
return processable;
|
||||
}
|
||||
|
||||
function getBase64DecodedByteCount(content: string): number {
|
||||
let paddingChars = 0;
|
||||
|
||||
|
|
@ -106,6 +156,8 @@ function getBase64DecodedByteCount(content: string): number {
|
|||
* (e.g., via `supportedMimeTypes` in `processAttachments`). This function processes
|
||||
* every file it receives and dispatches to the appropriate provider format:
|
||||
* - **Bedrock**: Only encodes types in `bedrockDocumentFormats`; all others are skipped.
|
||||
* - **Anthropic**: Only encodes PDFs (base64 source) and textual types (plain-text source);
|
||||
* all others are skipped.
|
||||
* - **PDF**: Validated via `validatePdf` before encoding.
|
||||
* - **Generic types**: Encoded with a provider-specific size check.
|
||||
*/
|
||||
|
|
@ -130,9 +182,7 @@ export async function encodeAndFormatDocuments(
|
|||
return result;
|
||||
}
|
||||
|
||||
const processableFiles = isBedrock
|
||||
? files.filter((file) => isBedrockDocumentType(file.type))
|
||||
: files;
|
||||
const processableFiles = filterProviderDocumentFiles(provider, files);
|
||||
|
||||
if (!processableFiles.length) {
|
||||
return result;
|
||||
|
|
|
|||
|
|
@ -2,8 +2,10 @@ import type { MimeUploadCapability } from './file-config';
|
|||
import type { FileConfig } from './types/files';
|
||||
import {
|
||||
fileConfig as baseFileConfig,
|
||||
isAnthropicTextDocumentType,
|
||||
getConfiguredMimeAccept,
|
||||
bedrockDocumentMimeTypes,
|
||||
isAnthropicDocumentType,
|
||||
isPermissiveMimeConfig,
|
||||
convertStringsToRegex,
|
||||
documentParserMimeTypes,
|
||||
|
|
@ -218,6 +220,47 @@ describe('documentParserMimeTypes', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('isAnthropicDocumentType', () => {
|
||||
it.each([
|
||||
'application/pdf',
|
||||
'text/plain',
|
||||
'text/html',
|
||||
'text/markdown',
|
||||
'text/csv',
|
||||
'text/x-python',
|
||||
'application/json',
|
||||
'application/xml',
|
||||
'application/yaml',
|
||||
'application/sql',
|
||||
'application/csv',
|
||||
])('accepts type the Anthropic document path can send: %s', (mimeType) => {
|
||||
expect(isAnthropicDocumentType(mimeType)).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||
'application/msword',
|
||||
'application/vnd.ms-excel',
|
||||
'application/epub+zip',
|
||||
'application/zip',
|
||||
'application/vnd.apache.parquet',
|
||||
'image/png',
|
||||
'',
|
||||
])('rejects type Anthropic would 400 on: %s', (mimeType) => {
|
||||
expect(isAnthropicDocumentType(mimeType)).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects undefined', () => {
|
||||
expect(isAnthropicDocumentType(undefined)).toBe(false);
|
||||
});
|
||||
|
||||
it('excludes PDF from the plain-text subset', () => {
|
||||
expect(isAnthropicTextDocumentType('application/pdf')).toBe(false);
|
||||
expect(isAnthropicTextDocumentType('text/plain')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getEndpointFileConfig', () => {
|
||||
describe('custom endpoint lookup', () => {
|
||||
it('should find custom endpoint by direct lookup', () => {
|
||||
|
|
|
|||
|
|
@ -184,6 +184,32 @@ export const bedrockDocumentMimeTypes: readonly string[] = Object.keys(bedrockDo
|
|||
export const bedrockDocumentExtensions =
|
||||
'.pdf,.csv,.doc,.docx,.xls,.xlsx,.html,.htm,.txt,.md,application/pdf,text/csv,application/csv,application/msword,application/vnd.openxmlformats-officedocument.wordprocessingml.document,application/vnd.ms-excel,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,text/html,text/plain,text/markdown';
|
||||
|
||||
/** Textual `application/*` MIME types that can be decoded and sent as plain text */
|
||||
const textualApplicationTypes = new Set([
|
||||
'application/json',
|
||||
'application/xml',
|
||||
'application/yaml',
|
||||
'application/sql',
|
||||
'application/typescript',
|
||||
'application/x-sh',
|
||||
'application/csv',
|
||||
]);
|
||||
|
||||
/**
|
||||
* MIME types the Anthropic Messages API accepts as a plain-text document source
|
||||
* (`source.type: 'text'`)
|
||||
*/
|
||||
export const isAnthropicTextDocumentType = (mimeType?: string): boolean =>
|
||||
mimeType != null && (mimeType.startsWith('text/') || textualApplicationTypes.has(mimeType));
|
||||
|
||||
/**
|
||||
* MIME types the Anthropic Messages API document path can send to the model
|
||||
* (mirrors `isBedrockDocumentType`): PDF via base64, textual types via a
|
||||
* plain-text document source. All other types are rejected with a provider 400.
|
||||
*/
|
||||
export const isAnthropicDocumentType = (mimeType?: string): boolean =>
|
||||
mimeType === 'application/pdf' || isAnthropicTextDocumentType(mimeType);
|
||||
|
||||
export const excelMimeTypes =
|
||||
/^application\/(vnd\.ms-excel|msexcel|x-msexcel|x-ms-excel|x-excel|x-dos_ms_excel|xls|x-xls|vnd\.openxmlformats-officedocument\.spreadsheetml\.sheet)$/;
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue