mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-20 23:13:35 +00:00
📄 fix: Send Textual Documents to Claude Through Gateways as Text (#16055)
Claude accepts base64 documents only for PDF. Through an OpenAI-compatible gateway, a textual file was sent as an OpenAI `file` part with its own media type, which the gateway maps to a base64 document that Claude rejects with a 400 on every later turn. Send it as a text part with the decoded contents, using the same Claude detection as the document filter. Fixes #16054 Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
1080f90bed
commit
61d034c20f
3 changed files with 82 additions and 5 deletions
|
|
@ -909,6 +909,53 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => {
|
|||
expect(mockedGetFileStream).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should send textual documents as text parts to Claude through an OpenAI-compatible gateway', async () => {
|
||||
const req = createMockRequest(30) as ServerRequest;
|
||||
const file = createMockDocFile(1, 'text/plain', 'notes.txt');
|
||||
mockedGetFileStream.mockResolvedValue({
|
||||
file,
|
||||
content: Buffer.from('hello from test').toString('base64'),
|
||||
metadata: file,
|
||||
});
|
||||
|
||||
const result = await encodeAndFormatDocuments(
|
||||
req,
|
||||
[file],
|
||||
{ provider: Providers.OPENAI, model: 'claude-auto-latest' },
|
||||
mockStrategyFunctions,
|
||||
);
|
||||
|
||||
expect(result.documents).toEqual([
|
||||
{ type: 'text', text: 'File: "notes.txt"\n\nhello from test' },
|
||||
]);
|
||||
expect(result.files).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('should keep textual documents as file parts for non-Claude OpenAI models', async () => {
|
||||
const req = createMockRequest(30) as ServerRequest;
|
||||
const file = createMockDocFile(1, 'text/html', 'page.html');
|
||||
const mockContent = Buffer.from('<p>hi</p>').toString('base64');
|
||||
mockedGetFileStream.mockResolvedValue({
|
||||
file,
|
||||
content: mockContent,
|
||||
metadata: file,
|
||||
});
|
||||
|
||||
const result = await encodeAndFormatDocuments(
|
||||
req,
|
||||
[file],
|
||||
{ provider: Providers.OPENAI, model: 'gemini-auto-latest' },
|
||||
mockStrategyFunctions,
|
||||
);
|
||||
|
||||
expect(result.documents).toEqual([
|
||||
{
|
||||
type: 'file',
|
||||
file: { filename: 'page.html', file_data: `data:text/html;base64,${mockContent}` },
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('should retain XLSX support for non-Claude OpenAI models', async () => {
|
||||
const req = createMockRequest(30) as ServerRequest;
|
||||
const file = createMockDocFile(
|
||||
|
|
|
|||
|
|
@ -47,6 +47,17 @@ function getAnthropicDocumentSource(
|
|||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the model behind this provider is Claude, which accepts only PDFs as base64
|
||||
* documents. OpenAI-compatible gateways report an OpenAI-like provider for Claude models.
|
||||
*/
|
||||
function usesAnthropicDocumentCapabilities(provider: Providers, model?: string): boolean {
|
||||
return (
|
||||
provider === Providers.ANTHROPIC ||
|
||||
(isOpenAILikeProvider(provider) && (model?.toLowerCase().includes('claude') ?? false))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats a base64-encoded document into the appropriate provider-specific block.
|
||||
* Returns `null` when the provider has no matching handler.
|
||||
|
|
@ -57,6 +68,7 @@ function formatDocumentBlock(
|
|||
content: string,
|
||||
filename: string | undefined,
|
||||
useResponsesApi: boolean | undefined,
|
||||
model?: string,
|
||||
): DocumentBlock | null {
|
||||
if (provider === Providers.ANTHROPIC) {
|
||||
const source = getAnthropicDocumentSource(mimeType, content);
|
||||
|
|
@ -87,6 +99,19 @@ function formatDocumentBlock(
|
|||
|
||||
const resolvedFilename = filename ?? 'document';
|
||||
|
||||
/* A gateway translates an OpenAI `file` part into a base64 document with the file's own
|
||||
* media type, which Claude rejects for anything but PDF. Send textual files as text. */
|
||||
if (
|
||||
!useResponsesApi &&
|
||||
isAnthropicTextDocumentType(mimeType) &&
|
||||
usesAnthropicDocumentCapabilities(provider, model)
|
||||
) {
|
||||
return {
|
||||
type: 'text',
|
||||
text: `File: "${resolvedFilename}"\n\n${Buffer.from(content, 'base64').toString('utf8')}`,
|
||||
};
|
||||
}
|
||||
|
||||
if (useResponsesApi) {
|
||||
return {
|
||||
type: 'input_file',
|
||||
|
|
@ -123,11 +148,7 @@ function filterProviderDocumentFiles(
|
|||
return files.filter((file) => isBedrockDocumentType(file.type));
|
||||
}
|
||||
|
||||
const usesAnthropicDocumentCapabilities =
|
||||
provider === Providers.ANTHROPIC ||
|
||||
(isOpenAILikeProvider(provider) && model?.toLowerCase().includes('claude'));
|
||||
|
||||
if (!usesAnthropicDocumentCapabilities) {
|
||||
if (!usesAnthropicDocumentCapabilities(provider, model)) {
|
||||
return files;
|
||||
}
|
||||
|
||||
|
|
@ -283,6 +304,7 @@ export async function encodeAndFormatDocuments(
|
|||
content,
|
||||
file.filename,
|
||||
useResponsesApi,
|
||||
model,
|
||||
);
|
||||
if (block) {
|
||||
result.documents.push(block);
|
||||
|
|
@ -302,6 +324,7 @@ export async function encodeAndFormatDocuments(
|
|||
content,
|
||||
file.filename,
|
||||
useResponsesApi,
|
||||
model,
|
||||
);
|
||||
if (block) {
|
||||
result.documents.push(block);
|
||||
|
|
|
|||
|
|
@ -91,6 +91,12 @@ export interface OpenAIFileBlock {
|
|||
};
|
||||
}
|
||||
|
||||
/** OpenAI-compatible text part carrying a textual document's decoded contents */
|
||||
export interface OpenAITextDocumentBlock {
|
||||
type: 'text';
|
||||
text: string;
|
||||
}
|
||||
|
||||
/** OpenAI Responses API file format */
|
||||
export interface OpenAIInputFileBlock {
|
||||
type: 'input_file';
|
||||
|
|
@ -114,6 +120,7 @@ export type DocumentBlock =
|
|||
| AnthropicDocumentBlock
|
||||
| GoogleDocumentBlock
|
||||
| OpenAIFileBlock
|
||||
| OpenAITextDocumentBlock
|
||||
| OpenAIInputFileBlock
|
||||
| BedrockDocumentBlock;
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue