diff --git a/client/src/components/Chat/Input/Files/AttachFileMenu.tsx b/client/src/components/Chat/Input/Files/AttachFileMenu.tsx index 952b337b3c..65cbd0801e 100644 --- a/client/src/components/Chat/Input/Files/AttachFileMenu.tsx +++ b/client/src/components/Chat/Input/Files/AttachFileMenu.tsx @@ -19,12 +19,17 @@ import { Providers, EToolResources, EModelEndpoint, - isPermissiveMimeConfig, + getConfiguredMimeAccept, + bedrockDocumentMimeTypes, defaultAgentCapabilities, bedrockDocumentExtensions, isDocumentSupportedProvider, } from 'librechat-data-provider'; -import type { EndpointFileConfig, TConversation } from 'librechat-data-provider'; +import type { + TConversation, + EndpointFileConfig, + MimeUploadCapability, +} from 'librechat-data-provider'; import type { ExtendedFile, FileSetter } from '~/common'; import { useAgentToolPermissions, @@ -48,6 +53,22 @@ type FileUploadType = | 'image_document_extended' | 'image_document_video_audio'; +/** What each provider upload path can actually send, used to scope the picker filter to selectable files. */ +const fileTypeCapabilities: Record = { + image: { categories: ['image'] }, + document: { categories: ['document'] }, + image_document: { categories: ['image', 'document'] }, + image_document_extended: { + categories: ['image', 'document'], + documentMimeTypes: bedrockDocumentMimeTypes, + }, + /** Google/Vertex/OpenRouter media path: documents are limited to PDF (see isProviderAttachType). */ + image_document_video_audio: { + categories: ['image', 'document', 'audio', 'video'], + documentMimeTypes: ['application/pdf'], + }, +}; + interface AttachFileMenuProps { agentId?: string | null; endpoint?: string | null; @@ -120,11 +141,15 @@ const AttachFileMenu = ({ return; } inputRef.current.value = ''; - if ( - fileType !== undefined && - isPermissiveMimeConfig(endpointFileConfig?.supportedMimeTypes) - ) { - inputRef.current.accept = ''; + const configuredAccept = + fileType !== undefined + ? getConfiguredMimeAccept( + endpointFileConfig?.supportedMimeTypes, + fileTypeCapabilities[fileType], + ) + : undefined; + if (configuredAccept != null) { + inputRef.current.accept = configuredAccept; } else if (fileType === 'image') { inputRef.current.accept = 'image/*,.heif,.heic'; } else if (fileType === 'document') { diff --git a/packages/data-provider/src/file-config.spec.ts b/packages/data-provider/src/file-config.spec.ts index c5380c12b6..b12666f865 100644 --- a/packages/data-provider/src/file-config.spec.ts +++ b/packages/data-provider/src/file-config.spec.ts @@ -1,6 +1,9 @@ +import type { MimeUploadCapability } from './file-config'; import type { FileConfig } from './types/files'; import { fileConfig as baseFileConfig, + getConfiguredMimeAccept, + bedrockDocumentMimeTypes, isPermissiveMimeConfig, convertStringsToRegex, documentParserMimeTypes, @@ -79,6 +82,26 @@ describe('inferMimeType', () => { const normalized = inferMimeType('test.eml', ''); expect(baseFileConfig.checkType(normalized)).toBe(true); }); + + it('infers Office MIME types from extension when the browser reports none', () => { + expect(inferMimeType('legacy.doc', '')).toBe('application/msword'); + expect(inferMimeType('report.docx', '')).toBe( + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + ); + expect(inferMimeType('legacy.xls', '')).toBe('application/vnd.ms-excel'); + expect(inferMimeType('sheet.xlsx', '')).toBe( + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + ); + expect(inferMimeType('deck.pptx', '')).toBe( + 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + ); + }); + + it('produces Office types accepted by checkType after inference', () => { + expect(baseFileConfig.checkType(inferMimeType('report.docx', ''))).toBe(true); + expect(baseFileConfig.checkType(inferMimeType('sheet.xlsx', ''))).toBe(true); + expect(baseFileConfig.checkType(inferMimeType('legacy.doc', ''))).toBe(true); + }); }); describe('applicationMimeTypes', () => { @@ -1359,3 +1382,192 @@ describe('isPermissiveMimeConfig', () => { expect(isPermissiveMimeConfig(converted)).toBe(true); }); }); + +describe('getConfiguredMimeAccept', () => { + const toSet = (accept?: string) => new Set((accept ?? '').split(',').filter(Boolean)); + + /** Provider capability tiers: image-only, document providers, Google/OpenRouter, and Bedrock. */ + const IMAGE_ONLY: MimeUploadCapability = { categories: ['image'] }; + const IMAGE_DOC: MimeUploadCapability = { categories: ['image', 'document'] }; + const ALL: MimeUploadCapability = { categories: ['image', 'document', 'audio', 'video'] }; + const BEDROCK: MimeUploadCapability = { + categories: ['image', 'document'], + documentMimeTypes: bedrockDocumentMimeTypes, + }; + const GOOGLE: MimeUploadCapability = { + categories: ['image', 'document', 'audio', 'video'], + documentMimeTypes: ['application/pdf'], + }; + + it('returns undefined for undefined', () => { + expect(getConfiguredMimeAccept(undefined, ALL)).toBeUndefined(); + }); + + it('returns undefined for empty array', () => { + expect(getConfiguredMimeAccept([], ALL)).toBeUndefined(); + }); + + it('returns undefined for the built-in default supportedMimeTypes (keep provider filter)', () => { + expect(getConfiguredMimeAccept(supportedMimeTypes, ALL)).toBeUndefined(); + }); + + it("returns '' for permissive configs so the picker stays unrestricted", () => { + expect(getConfiguredMimeAccept(convertStringsToRegex(['.*']), ALL)).toBe(''); + expect(getConfiguredMimeAccept([/^.+$/], ALL)).toBe(''); + }); + + it('translates the finite Office + image allowlist from issue #14162', () => { + const config = convertStringsToRegex([ + '^image/.*', + '^application/pdf$', + '^application/msword$', + '^application/vnd\\.openxmlformats-officedocument\\.wordprocessingml\\.document$', + '^application/vnd\\.ms-excel$', + '^application/vnd\\.openxmlformats-officedocument\\.spreadsheetml\\.sheet$', + ]); + const accept = toSet(getConfiguredMimeAccept(config, IMAGE_DOC)); + + for (const token of [ + 'image/*', + '.pdf', + 'application/pdf', + '.doc', + 'application/msword', + '.docx', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.xls', + 'application/vnd.ms-excel', + '.xlsx', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + ]) { + expect(accept.has(token)).toBe(true); + } + expect(accept.has('.pptx')).toBe(false); + expect(accept.has('audio/*')).toBe(false); + expect(accept.has('video/*')).toBe(false); + }); + + it('emits image/* for an image-only allowlist', () => { + const accept = toSet(getConfiguredMimeAccept([/^image\/(jpeg|png)$/], IMAGE_DOC)); + expect(accept.has('image/*')).toBe(true); + expect(accept.has('.pdf')).toBe(false); + }); + + it('emits media wildcards for audio and video allowlists on a media-capable path', () => { + const accept = toSet(getConfiguredMimeAccept([/^audio\/.*$/, /^video\/.*$/], ALL)); + expect(accept.has('audio/*')).toBe(true); + expect(accept.has('video/*')).toBe(true); + }); + + it('returns undefined for a finite config with no recognized types (fall back to provider filter)', () => { + expect(getConfiguredMimeAccept([/^application\/x-librechat-unknown$/], ALL)).toBeUndefined(); + }); + + it('falls back when any configured type is unrepresentable, rather than emitting a partial accept', () => { + expect( + getConfiguredMimeAccept([/^application\/pdf$/, /^text\/x-python$/], IMAGE_DOC), + ).toBeUndefined(); + }); + + it('translates a fully-representable mixed pdf + audio allowlist on a media-capable path', () => { + const accept = toSet(getConfiguredMimeAccept([/^application\/pdf$/, /^audio\/mp3$/], ALL)); + expect(accept.has('.pdf')).toBe(true); + expect(accept.has('application/pdf')).toBe(true); + expect(accept.has('audio/*')).toBe(true); + expect(accept.has('video/*')).toBe(false); + expect(accept.has('image/*')).toBe(false); + }); + + it('keeps the image upload path image-only even when the allowlist adds documents', () => { + const config = convertStringsToRegex([ + '^image/.*', + '^application/pdf$', + '^application/vnd\\.openxmlformats-officedocument\\.wordprocessingml\\.document$', + ]); + const accept = toSet(getConfiguredMimeAccept(config, IMAGE_ONLY)); + expect(accept.has('image/*')).toBe(true); + expect(accept.has('.pdf')).toBe(false); + expect(accept.has('.docx')).toBe(false); + }); + + it('excludes audio/video for document providers that cannot send them', () => { + const accept = toSet( + getConfiguredMimeAccept([/^image\/.*$/, /^application\/pdf$/, /^audio\/mp3$/], IMAGE_DOC), + ); + expect(accept.has('image/*')).toBe(true); + expect(accept.has('.pdf')).toBe(true); + expect(accept.has('audio/*')).toBe(false); + }); + + it('falls back for a broad application/.* regex that reaches unrepresentable types', () => { + expect(getConfiguredMimeAccept([/^application\/.*$/], IMAGE_DOC)).toBeUndefined(); + }); + + it('restricts Bedrock document accepts to Bedrock-supported formats', () => { + const config = convertStringsToRegex([ + '^application/pdf$', + '^application/vnd\\.openxmlformats-officedocument\\.wordprocessingml\\.document$', + '^application/vnd\\.openxmlformats-officedocument\\.presentationml\\.presentation$', + ]); + const accept = toSet(getConfiguredMimeAccept(config, BEDROCK)); + expect(accept.has('.pdf')).toBe(true); + expect(accept.has('.docx')).toBe(true); + expect(accept.has('.pptx')).toBe(false); + expect( + accept.has('application/vnd.openxmlformats-officedocument.presentationml.presentation'), + ).toBe(false); + }); + + it('includes both .html and .htm when translating text/html', () => { + const accept = toSet(getConfiguredMimeAccept([/^text\/html$/], BEDROCK)); + expect(accept.has('.html')).toBe(true); + expect(accept.has('.htm')).toBe(true); + expect(accept.has('text/html')).toBe(true); + }); + + it('restricts Google/OpenRouter documents to PDF while keeping media categories', () => { + const config = convertStringsToRegex([ + '^image/.*$', + '^application/pdf$', + '^application/vnd\\.openxmlformats-officedocument\\.wordprocessingml\\.document$', + '^audio/.*$', + ]); + const accept = toSet(getConfiguredMimeAccept(config, GOOGLE)); + expect(accept.has('image/*')).toBe(true); + expect(accept.has('.pdf')).toBe(true); + expect(accept.has('audio/*')).toBe(true); + expect(accept.has('.docx')).toBe(false); + }); + + it('translates a sampled audio subtype (audio/webm) rather than hiding it', () => { + const accept = toSet(getConfiguredMimeAccept([/^image\/.*$/, /^audio\/webm$/], GOOGLE)); + expect(accept.has('image/*')).toBe(true); + expect(accept.has('audio/*')).toBe(true); + }); + + it('falls back when a configured pattern matches no known MIME type', () => { + expect( + getConfiguredMimeAccept([/^image\/.*$/, /^audio\/x-librechat-unknown$/], GOOGLE), + ).toBeUndefined(); + }); + + it('translates a finite Excel allowlist that includes legacy MIME aliases', () => { + const config = convertStringsToRegex([ + '^application/(vnd\\.ms-excel|msexcel|x-msexcel|x-ms-excel|x-excel|x-dos_ms_excel|xls|x-xls|vnd\\.openxmlformats-officedocument\\.spreadsheetml\\.sheet)$', + ]); + const accept = toSet(getConfiguredMimeAccept(config, IMAGE_DOC)); + expect(accept.has('.xls')).toBe(true); + expect(accept.has('.xlsx')).toBe(true); + }); + + it('translates finite epub and parquet document allowlists', () => { + const accept = toSet( + getConfiguredMimeAccept( + [/^application\/epub\+zip$/, /^application\/vnd\.apache\.parquet$/], + IMAGE_DOC, + ), + ); + expect(accept.has('.epub')).toBe(true); + expect(accept.has('.parquet')).toBe(true); + }); +}); diff --git a/packages/data-provider/src/file-config.ts b/packages/data-provider/src/file-config.ts index 8ca650c7e4..a6cf478c15 100644 --- a/packages/data-provider/src/file-config.ts +++ b/packages/data-provider/src/file-config.ts @@ -177,6 +177,9 @@ export const bedrockDocumentFormats: Record = { export const isBedrockDocumentType = (mimeType?: string): boolean => mimeType != null && mimeType in bedrockDocumentFormats; +/** MIME types Bedrock's Converse document path can send to the model (mirrors `bedrockDocumentFormats`). */ +export const bedrockDocumentMimeTypes: readonly string[] = Object.keys(bedrockDocumentFormats); + /** File extensions accepted by Bedrock document uploads (for input accept attributes) */ export const bedrockDocumentExtensions = '.pdf,.csv,.doc,.docx,.xls,.xlsx,.html,.htm,.txt,.md,application/pdf,text/csv,application/csv,application/msword,application/vnd.openxmlformats-officedocument.wordprocessingml.document,application/vnd.ms-excel,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,text/html,text/plain,text/markdown'; @@ -359,6 +362,12 @@ export const codeTypeMapping: { [key: string]: string } = { ods: 'application/vnd.oasis.opendocument.spreadsheet', // .ods - OpenDocument Spreadsheet odp: 'application/vnd.oasis.opendocument.presentation', // .odp - OpenDocument Presentation odg: 'application/vnd.oasis.opendocument.graphics', // .odg - OpenDocument Graphics + doc: 'application/msword', // .doc - Word (legacy) + docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', // .docx - Word + xls: 'application/vnd.ms-excel', // .xls - Excel (legacy) + xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', // .xlsx - Excel + ppt: 'application/vnd.ms-powerpoint', // .ppt - PowerPoint (legacy) + pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', // .pptx - PowerPoint ics: 'text/calendar', // .ics - iCalendar ical: 'text/calendar', // .ical - iCalendar ifb: 'text/calendar', // .ifb - iCalendar free/busy @@ -528,6 +537,248 @@ export const isPermissiveMimeConfig = (types?: RegExp[]): boolean => { return types.some((regex) => regex.test('x-librechat/x-probe')); }; +/** The kind of content a provider upload path can actually send to the model. */ +export type MimeUploadCategory = 'image' | 'document' | 'audio' | 'video'; + +/** Describes what an upload path can send, used to scope a configured allowlist to selectable files. */ +export interface MimeUploadCapability { + /** Content categories the path forwards to the model. */ + categories: ReadonlyArray; + /** When `document` is permitted, restrict document types to this set (e.g. Bedrock formats); default: all. */ + documentMimeTypes?: readonly string[]; +} + +/** Media categories that collapse to a wildcard `accept` token when any member type is allowed. */ +const mimeAcceptCategories: ReadonlyArray<{ + category: Exclude; + token: string; + samples: readonly string[]; + extras?: readonly string[]; +}> = [ + { + /** Mirrors `imageMimeTypes` (+ the code-interpreter svg variants) so every accepted type is known. */ + category: 'image', + token: 'image/*', + samples: [ + 'image/jpeg', + 'image/gif', + 'image/png', + 'image/webp', + 'image/heic', + 'image/heif', + 'image/svg', + 'image/svg+xml', + ], + extras: ['.heif', '.heic'], + }, + { + /** Mirrors `audioMimeTypes`. */ + category: 'audio', + token: 'audio/*', + samples: [ + 'audio/mp3', + 'audio/mpeg', + 'audio/mpeg3', + 'audio/wav', + 'audio/wave', + 'audio/x-wav', + 'audio/ogg', + 'audio/vorbis', + 'audio/mp4', + 'audio/m4a', + 'audio/x-m4a', + 'audio/flac', + 'audio/x-flac', + 'audio/webm', + 'audio/aac', + 'audio/wma', + 'audio/opus', + ], + }, + { + /** Mirrors `videoMimeTypes`. */ + category: 'video', + token: 'video/*', + samples: [ + 'video/mp4', + 'video/avi', + 'video/mov', + 'video/wmv', + 'video/flv', + 'video/webm', + 'video/mkv', + 'video/m4v', + 'video/3gp', + 'video/ogv', + ], + }, +]; + +/** Document/text MIME types paired with the extension(s) browsers filter on in the file picker. */ +const documentMimeExtensions: ReadonlyArray = [ + ['application/pdf', ['.pdf']], + ['application/msword', ['.doc']], + ['application/vnd.openxmlformats-officedocument.wordprocessingml.document', ['.docx']], + ['application/vnd.ms-excel', ['.xls']], + ['application/msexcel', ['.xls']], + ['application/x-msexcel', ['.xls']], + ['application/x-ms-excel', ['.xls']], + ['application/x-excel', ['.xls']], + ['application/x-dos_ms_excel', ['.xls']], + ['application/xls', ['.xls']], + ['application/x-xls', ['.xls']], + ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', ['.xlsx']], + ['application/vnd.ms-powerpoint', ['.ppt']], + ['application/vnd.openxmlformats-officedocument.presentationml.presentation', ['.pptx']], + ['application/vnd.oasis.opendocument.text', ['.odt']], + ['application/vnd.oasis.opendocument.spreadsheet', ['.ods']], + ['application/vnd.oasis.opendocument.presentation', ['.odp']], + ['application/vnd.oasis.opendocument.graphics', ['.odg']], + ['application/rtf', ['.rtf']], + ['application/json', ['.json']], + ['application/xml', ['.xml']], + ['application/yaml', ['.yaml', '.yml']], + ['application/zip', ['.zip']], + ['application/x-zip-compressed', ['.zip']], + ['application/epub+zip', ['.epub']], + ['application/x-parquet', ['.parquet']], + ['application/vnd.apache.parquet', ['.parquet']], + ['text/csv', ['.csv']], + ['application/csv', ['.csv']], + ['text/tab-separated-values', ['.tsv']], + ['text/plain', ['.txt']], + ['text/markdown', ['.md']], + ['text/html', ['.html', '.htm']], + ['text/calendar', ['.ics']], + ['message/rfc822', ['.eml']], +]; + +const documentMimeSet = new Set(documentMimeExtensions.map(([mimeType]) => mimeType)); + +/** Every MIME type LibreChat may accept, used to detect patterns that reach beyond the representable set. */ +const knownMimeUniverse: readonly string[] = Array.from( + new Set([ + ...fullMimeTypesList, + ...documentMimeExtensions.map(([mimeType]) => mimeType), + ...mimeAcceptCategories.flatMap((category) => category.samples), + ]), +); + +const categoryOf = (mimeType: string): MimeUploadCategory => { + if (mimeType.startsWith('image/')) { + return 'image'; + } + if (mimeType.startsWith('audio/')) { + return 'audio'; + } + if (mimeType.startsWith('video/')) { + return 'video'; + } + return 'document'; +}; + +/** Media types are covered by their `/*` wildcard token; document types need an explicit entry. */ +const isRepresentable = (mimeType: string): boolean => + categoryOf(mimeType) !== 'document' || documentMimeSet.has(mimeType); + +/** + * Translates a finite MIME allowlist into a file-input `accept` string, intersected with what the + * provider upload path can actually send. Returns `undefined` (keep the provider filter) when a + * configured pattern matches a supported, path-handleable type that cannot be represented, so the + * picker never hides a file the path would have accepted. + */ +const buildMimeAccept = ( + types: RegExp[], + { categories, documentMimeTypes }: MimeUploadCapability, +): string | undefined => { + const permittedSet = new Set(categories); + const documentAllowSet = documentMimeTypes ? new Set(documentMimeTypes) : null; + const emittedMedia = new Set(); + const emittedDocuments = new Set(); + + /** A pattern matching nothing known may still accept a supported type we can't represent; fall back. */ + const everyPatternKnown = types.every((regex) => + knownMimeUniverse.some((mimeType) => regex.test(mimeType)), + ); + if (!everyPatternKnown) { + return undefined; + } + + for (const regex of types) { + for (const mimeType of knownMimeUniverse) { + if (!regex.test(mimeType)) { + continue; + } + const category = categoryOf(mimeType); + if (!permittedSet.has(category)) { + continue; + } + /** The path handles documents but drops this specific type (e.g. Bedrock ignores pptx/ODF). */ + if (category === 'document' && documentAllowSet && !documentAllowSet.has(mimeType)) { + continue; + } + if (!isRepresentable(mimeType)) { + return undefined; + } + if (category === 'document') { + emittedDocuments.add(mimeType); + } else { + emittedMedia.add(category); + } + } + } + + const tokens: string[] = []; + const seen = new Set(); + const push = (token: string): void => { + if (!seen.has(token)) { + seen.add(token); + tokens.push(token); + } + }; + + for (const category of mimeAcceptCategories) { + if (emittedMedia.has(category.category)) { + push(category.token); + category.extras?.forEach(push); + } + } + + for (const [mimeType, extensions] of documentMimeExtensions) { + if (emittedDocuments.has(mimeType)) { + extensions.forEach(push); + push(mimeType); + } + } + + return tokens.length > 0 ? tokens.join(',') : undefined; +}; + +/** + * Resolves the file-input `accept` value for a configured `supportedMimeTypes` allowlist, scoped to + * what the current upload path (`capability`) can send to the model. + * - `undefined` for the built-in default or a config that can't be represented safely, so callers + * keep their provider-specific filter. + * - `''` for permissive configs (e.g. `.*`), leaving the picker unrestricted. + * - a translated `accept` string for a recognized finite allowlist (images, PDFs, Office docs, etc.). + * + * The picker `accept` is a UX convenience, not a security boundary: the backend still enforces + * `supportedMimeTypes` on upload. + */ +export const getConfiguredMimeAccept = ( + types: RegExp[] | undefined, + capability: MimeUploadCapability, +): string | undefined => { + /** Referential identity with the built-in list signals an unconfigured endpoint (keep provider filter). */ + if (!types || types.length === 0 || types === supportedMimeTypes) { + return undefined; + } + if (isPermissiveMimeConfig(types)) { + return ''; + } + return buildMimeAccept(types, capability); +}; + /** * Gets the appropriate endpoint file configuration with standardized lookup logic. *