From debb6e80e854f03d80e770b102fafd5a3d4ab3e8 Mon Sep 17 00:00:00 2001 From: Atef Bellaaj Date: Sun, 12 Apr 2026 19:58:15 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A7=20feat:=20Unified=20file=20upload?= =?UTF-8?q?=20=E2=80=94=20per-mime-type=20routing=20with=20lazy=20provisio?= =?UTF-8?q?ning?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/app/clients/BaseClient.js | 4 + api/app/clients/specs/BaseClient.test.js | 100 ++++- api/server/index.js | 26 ++ api/server/services/Files/process.js | 120 ++++-- api/server/services/Files/process.spec.js | 401 +++++++++++++++++- api/server/services/Files/provision.js | 5 + .../Chat/Input/Files/AttachFileMenu.tsx | 4 +- .../Files/__tests__/AttachFileMenu.spec.tsx | 10 + client/src/utils/files.ts | 5 +- packages/api/src/agents/resources.test.ts | 110 +++++ packages/api/src/agents/resources.ts | 9 +- packages/api/src/files/context.test.ts | 64 +++ packages/api/src/files/context.ts | 7 +- .../files/encode/processAttachments.spec.ts | 26 ++ .../data-provider/src/file-config.spec.ts | 88 ++++ packages/data-provider/src/file-config.ts | 49 ++- packages/data-provider/src/index.ts | 1 + .../src/resolve-llm-delivery-path.spec.ts | 103 +++++ .../src/resolve-llm-delivery-path.ts | 56 +++ packages/data-provider/src/types/files.ts | 11 +- packages/data-schemas/src/schema/file.ts | 4 + packages/data-schemas/src/types/file.ts | 1 + 22 files changed, 1132 insertions(+), 72 deletions(-) create mode 100644 packages/api/src/files/context.test.ts create mode 100644 packages/data-provider/src/resolve-llm-delivery-path.spec.ts create mode 100644 packages/data-provider/src/resolve-llm-delivery-path.ts diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index e641017c74..f6342595db 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -1234,6 +1234,10 @@ class BaseClient { allFiles.push(file); continue; } + if (file.llmDeliveryPath === 'text' || file.llmDeliveryPath === 'none') { + allFiles.push(file); + continue; + } if ( file.embedded === true || file.metadata?.codeEnvRef != null || diff --git a/api/app/clients/specs/BaseClient.test.js b/api/app/clients/specs/BaseClient.test.js index eb6ae656e9..4e39d6d96a 100644 --- a/api/app/clients/specs/BaseClient.test.js +++ b/api/app/clients/specs/BaseClient.test.js @@ -1,4 +1,4 @@ -const { Constants } = require('librechat-data-provider'); +const { Constants, EModelEndpoint } = require('librechat-data-provider'); const { initializeFakeClient } = require('./FakeClient'); jest.mock('~/db/connect'); @@ -1295,4 +1295,102 @@ describe('BaseClient', () => { expect(userSave[0].files[0].file_id).toBe('file-abc'); }); }); + + describe('processAttachments llmDeliveryPath handling', () => { + beforeEach(() => { + TestClient.options = { + endpoint: EModelEndpoint.openAI, + }; + TestClient.addImageURLs = jest.fn(async (message, files) => { + message.image_urls = ['encoded-image']; + return files; + }); + TestClient.addDocuments = jest.fn(async (message, files) => { + message.documents = [{ type: 'file' }]; + return files; + }); + TestClient.addVideos = jest.fn(async (_message, files) => files); + TestClient.addAudios = jest.fn(async (_message, files) => files); + }); + + test('keeps a none image in returned files without adding image URLs', async () => { + const message = {}; + const file = { + user: 'user1', + file_id: 'none-image', + filename: 'image.png', + filepath: '/uploads/image.png', + type: 'image/png', + bytes: 100, + source: 'local', + llmDeliveryPath: 'none', + }; + + const result = await TestClient.processAttachments(message, [file]); + + expect(result).toEqual([file]); + expect(message.image_urls).toBeUndefined(); + expect(TestClient.addImageURLs).not.toHaveBeenCalled(); + }); + + test('keeps a none PDF in returned files without adding documents', async () => { + const message = {}; + const file = { + user: 'user1', + file_id: 'none-pdf', + filename: 'document.pdf', + filepath: '/uploads/document.pdf', + type: 'application/pdf', + bytes: 100, + source: 'local', + llmDeliveryPath: 'none', + }; + + const result = await TestClient.processAttachments(message, [file]); + + expect(result).toEqual([file]); + expect(message.documents).toBeUndefined(); + expect(TestClient.addDocuments).not.toHaveBeenCalled(); + }); + + test('keeps a text-delivery markdown file in returned files without adding documents', async () => { + const message = {}; + const file = { + user: 'user1', + file_id: 'text-markdown', + filename: 'notes.md', + filepath: '/uploads/notes.md', + type: 'text/markdown', + bytes: 100, + source: 'local', + text: 'extracted markdown', + llmDeliveryPath: 'text', + }; + + const result = await TestClient.processAttachments(message, [file]); + + expect(result).toEqual([file]); + expect(message.documents).toBeUndefined(); + expect(TestClient.addDocuments).not.toHaveBeenCalled(); + }); + + test('routes legacy files without llmDeliveryPath normally', async () => { + const message = {}; + const file = { + user: 'user1', + file_id: 'legacy-pdf', + filename: 'document.pdf', + filepath: '/uploads/document.pdf', + type: 'application/pdf', + bytes: 100, + source: 'local', + }; + + const result = await TestClient.processAttachments(message, [file]); + + expect(result).toEqual([file]); + expect(message.documents).toEqual([{ type: 'file' }]); + expect(TestClient.addDocuments).toHaveBeenCalledWith(message, [file]); + }); + }); }); diff --git a/api/server/index.js b/api/server/index.js index 6bc4a131e6..11d83c1df1 100644 --- a/api/server/index.js +++ b/api/server/index.js @@ -82,6 +82,32 @@ const startServer = async () => { logger.error('[sweepOrphanedPreviews] Background sweep failed:', err); }); const appConfig = await getAppConfig({ baseOnly: true }); + if (appConfig?.fileConfig?.defaultLLMDeliveryPath?.overrides) { + for (const [mimeType, destination] of Object.entries( + appConfig.fileConfig.defaultLLMDeliveryPath.overrides, + )) { + if (destination === 'none') { + logger.warn( + `[Config] defaultLLMDeliveryPath: "${mimeType}" is set to "none" — files of this type will only be accessible through tool provisioning`, + ); + } + } + } + if (appConfig?.fileConfig?.endpoints) { + for (const [endpoint, config] of Object.entries(appConfig.fileConfig.endpoints)) { + if (config?.defaultLLMDeliveryPath?.overrides) { + for (const [mimeType, destination] of Object.entries( + config.defaultLLMDeliveryPath.overrides, + )) { + if (destination === 'none') { + logger.warn( + `[Config] defaultLLMDeliveryPath for "${endpoint}": "${mimeType}" is set to "none" — files of this type will only be accessible through tool provisioning`, + ); + } + } + } + } + } initializeFileStorage(appConfig); await runAsSystem(async () => { await performStartupChecks(appConfig); diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js index 90c259ab2b..5a77748939 100644 --- a/api/server/services/Files/process.js +++ b/api/server/services/Files/process.js @@ -16,6 +16,7 @@ const { removeNullishValues, isAssistantsEndpoint, getEndpointFileConfig, + resolveDefaultLLMDeliveryPath, documentParserMimeTypes, } = require('librechat-data-provider'); const { logger } = require('@librechat/data-schemas'); @@ -317,6 +318,20 @@ const processFileURL = async ({ } }; +const resolveDefaultUploadLLMDeliveryPath = ({ file, endpointConfig, fileConfig }) => { + const isLegacyFileUploadUX = + endpointConfig?.legacyFileUploadUX === true || fileConfig?.legacyFileUploadUX === true; + if (isLegacyFileUploadUX) { + return 'provider'; + } + + return resolveDefaultLLMDeliveryPath( + file.mimetype, + endpointConfig?.defaultLLMDeliveryPath, + fileConfig?.defaultLLMDeliveryPath, + ); +}; + /** * Applies the current strategy for image uploads. * Saves file metadata to the database with an expiry TTL. @@ -334,6 +349,9 @@ const processImageFile = async ({ req, res, metadata, returnFile = false }) => { const source = getFileStrategy(appConfig, { isImage: true }); const { handleImageUpload } = getStrategyFunctions(source); const { file_id, temp_file_id, endpoint } = metadata; + const fileConfig = mergeFileConfig(appConfig?.fileConfig); + const endpointConfig = getEndpointFileConfig({ fileConfig, endpoint }); + const llmDeliveryPath = resolveDefaultUploadLLMDeliveryPath({ file, endpointConfig, fileConfig }); const { filepath, bytes, width, height, storageKey, storageRegion } = await handleImageUpload({ req, @@ -358,6 +376,7 @@ const processImageFile = async ({ req, res, metadata, returnFile = false }) => { width, height, tenantId: req.user.tenantId, + llmDeliveryPath, }, true, ); @@ -528,6 +547,21 @@ const processFileUpload = async ({ req, res, metadata }) => { res.status(200).json({ message: 'File uploaded and processed successfully', ...result }); }; +const resolveUploadLLMDeliveryPath = ({ tool_resource, file, endpointConfig, fileConfig }) => { + if (tool_resource === EToolResources.context || tool_resource === EToolResources.ocr) { + return 'text'; + } + + if ( + tool_resource === EToolResources.file_search || + tool_resource === EToolResources.execute_code + ) { + return 'none'; + } + + return resolveDefaultUploadLLMDeliveryPath({ file, endpointConfig, fileConfig }); +}; + /** * Applies the current strategy for file uploads. * Saves file metadata to the database with an expiry TTL. @@ -539,39 +573,36 @@ const processFileUpload = async ({ req, res, metadata }) => { * @param {FileMetadata} params.metadata - Additional metadata for the file. * @returns {Promise} */ -/** - * Resolves the file interaction mode from the merged file config. - * Checks endpoint-level config first, then global config. - * Returns 'deferred' as the default when nothing is configured. - * - * @param {object} req - The Express request object - * @param {object} appConfig - The application config - * @returns {string} - The resolved interaction mode: 'text' | 'provider' | 'deferred' | 'legacy' - */ -const resolveInteractionMode = (req, appConfig) => { - const fileConfig = mergeFileConfig(appConfig?.fileConfig); - const endpoint = req.body?.endpoint; - const endpointConfig = getEndpointFileConfig({ fileConfig, endpoint }); - return endpointConfig?.defaultFileInteraction ?? fileConfig?.defaultFileInteraction ?? 'deferred'; -}; - const processAgentFileUpload = async ({ req, res, metadata }) => { + // TODO: check and potentially fix — deferred/provider files may be orphaned if effectiveToolResource is undefined const { file } = req; const appConfig = req.config; const { agent_id, tool_resource, file_id, temp_file_id = null } = metadata; let messageAttachment = !!metadata.message_file; - let effectiveToolResource = tool_resource; + let effectiveToolResource = + tool_resource === EToolResources.ocr ? EToolResources.context : tool_resource; + + const fileConfig = mergeFileConfig(appConfig?.fileConfig); + const endpoint = req.body?.endpoint; + const endpointConfig = getEndpointFileConfig({ fileConfig, endpoint }); + if (agent_id && !tool_resource && !messageAttachment) { - const interactionMode = resolveInteractionMode(req, appConfig); - if (interactionMode === 'legacy') { + if (endpointConfig?.legacyFileUploadUX === true || fileConfig?.legacyFileUploadUX === true) { throw new Error('No tool resource provided for agent file upload'); } - // In unified mode: 'text' routes to context processing, 'deferred'/'provider' fall through to standard storage - if (interactionMode === 'text') { - effectiveToolResource = EToolResources.context; - } + } + + const llmDeliveryPath = resolveUploadLLMDeliveryPath({ + tool_resource, + file, + endpointConfig, + fileConfig, + }); + + if (!tool_resource && llmDeliveryPath === 'text') { + effectiveToolResource = EToolResources.context; } if (effectiveToolResource === EToolResources.file_search && file.mimetype.startsWith('image')) { @@ -643,31 +674,45 @@ const processAgentFileUpload = async ({ req, res, metadata }) => { /** * @param {object} params * @param {string} params.text - * @param {number} params.bytes - * @param {string} params.filepath - * @param {string} params.type * @return {Promise} */ - const createTextFile = async ({ text, bytes, filepath, type = 'text/plain' }) => { + const createTextFile = async ({ text }) => { const textBytes = Buffer.byteLength(text, 'utf8'); if (textBytes > 15 * megabyte) { throw new Error( `Extracted text from "${file.originalname}" exceeds the 15MB storage limit (${Math.round(textBytes / megabyte)}MB). Try a shorter document.`, ); } + + const isImageFile = file.mimetype.startsWith('image'); + const source = getFileStrategy(appConfig, { isImage: isImageFile }); + const { handleFileUpload } = getStrategyFunctions(source); + const sanitizedUploadFn = createSanitizedUploadWrapper(handleFileUpload); + const storageResult = await sanitizedUploadFn({ + req, + file, + file_id, + basePath, + entity_id, + }); + const { bytes, filename, filepath, embedded, height, width } = storageResult; const fileInfo = removeNullishValues({ text, bytes, file_id, temp_file_id, user: req.user.id, - type, - filepath: filepath ?? file.path, - source: FileSources.text, - filename: file.originalname, + type: file.mimetype, + filepath, + source, + filename: filename ?? sanitizeFilename(file.originalname), model: messageAttachment ? undefined : req.body.model, context: messageAttachment ? FileContext.message_attachment : FileContext.agents, tenantId: req.user.tenantId, + embedded, + height, + width, + llmDeliveryPath: 'text', }); if (!messageAttachment && effectiveToolResource) { @@ -726,8 +771,8 @@ const processAgentFileUpload = async ({ req, res, metadata }) => { if (shouldUseOCR) { const ocrResult = await resolveDocumentText(); if (ocrResult) { - const { text, bytes, filepath: ocrFileURL } = ocrResult; - return await createTextFile({ text, bytes, filepath: ocrFileURL }); + const { text } = ocrResult; + return await createTextFile({ text }); } throw new Error( `Unable to extract text from "${file.originalname}". The document may be image-based and requires an OCR service to process.`, @@ -741,8 +786,8 @@ const processAgentFileUpload = async ({ req, res, metadata }) => { if (shouldUseSTT) { const sttService = await STTService.getInstance(); - const { text, bytes } = await processAudioFile({ req, file, sttService }); - return await createTextFile({ text, bytes }); + const { text } = await processAudioFile({ req, file, sttService }); + return await createTextFile({ text }); } const shouldUseText = fileConfig.checkType( @@ -754,8 +799,8 @@ const processAgentFileUpload = async ({ req, res, metadata }) => { throw new Error(`File type ${file.mimetype} is not supported for text parsing.`); } - const { text, bytes } = await parseText({ req, file, file_id }); - return await createTextFile({ text, bytes, type: file.mimetype }); + const { text } = await parseText({ req, file, file_id }); + return await createTextFile({ text }); } // Dual storage pattern for RAG files: Storage + Vector DB @@ -866,6 +911,7 @@ const processAgentFileUpload = async ({ req, res, metadata }) => { height, width, tenantId: req.user.tenantId, + llmDeliveryPath, }); const result = await db.createFile(fileInfo, true); diff --git a/api/server/services/Files/process.spec.js b/api/server/services/Files/process.spec.js index 99457522d4..c82b5ea725 100644 --- a/api/server/services/Files/process.spec.js +++ b/api/server/services/Files/process.spec.js @@ -69,7 +69,12 @@ jest.mock('~/server/services/Files/Audio/STTService', () => ({ STTService: { getInstance: jest.fn() }, })); +jest.mock('./VectorDB/crud', () => ({ + uploadVectors: jest.fn().mockResolvedValue({ embedded: true, filename: 'embedded-upload.bin' }), +})); + const { + EModelEndpoint, EToolResources, FileSources, FileContext, @@ -77,9 +82,11 @@ const { } = require('librechat-data-provider'); const { mergeFileConfig } = require('librechat-data-provider'); const { checkCapability } = require('~/server/services/Config'); +const { loadAuthValues } = require('~/server/services/Tools/credentials'); const { getStrategyFunctions } = require('~/server/services/Files/strategies'); const db = require('~/models'); -const { processAgentFileUpload, processFileURL } = require('./process'); +const { uploadVectors } = require('./VectorDB/crud'); +const { processAgentFileUpload, processImageFile, processFileURL } = require('./process'); const PDF_MIME = 'application/pdf'; const DOCX_MIME = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; @@ -102,6 +109,7 @@ const makeReq = ({ mimetype = PDF_MIME, ocrConfig = null } = {}) => ({ config: { fileConfig: {}, fileStrategy: 'local', + imageOutputType: 'webp', ocr: ocrConfig, }, }); @@ -117,11 +125,15 @@ const mockRes = { json: jest.fn().mockReturnValue({}), }; -const makeFileConfig = ({ ocrSupportedMimeTypes = [] } = {}) => ({ +const makeFileConfig = ({ + ocrSupportedMimeTypes = [], + sttSupportedMimeTypes = [], + textSupportedMimeTypes = [], +} = {}) => ({ checkType: (mime, types) => (types ?? []).includes(mime), ocr: { supportedMimeTypes: ocrSupportedMimeTypes }, - stt: { supportedMimeTypes: [] }, - text: { supportedMimeTypes: [] }, + stt: { supportedMimeTypes: sttSupportedMimeTypes }, + text: { supportedMimeTypes: textSupportedMimeTypes }, }); describe('processAgentFileUpload', () => { @@ -130,6 +142,8 @@ describe('processAgentFileUpload', () => { mockRes.status.mockReturnThis(); mockRes.json.mockReturnValue({}); checkCapability.mockResolvedValue(true); + loadAuthValues.mockResolvedValue({ CODE_API_KEY: 'code-key' }); + uploadVectors.mockResolvedValue({ embedded: true, filename: 'embedded-upload.bin' }); getStrategyFunctions.mockReturnValue({ handleFileUpload: jest .fn() @@ -468,6 +482,385 @@ describe('processAgentFileUpload', () => { expect(persisted.metadata).not.toHaveProperty('fileIdentifier'); }); }); + + describe('text delivery storage', () => { + test('stores the original file durably for plain text delivery records', async () => { + const { parseText } = require('@librechat/api'); + const { createFile } = require('~/models'); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 128, + filename: 'upload.bin', + embedded: false, + }); + mergeFileConfig.mockReturnValue(makeFileConfig({ textSupportedMimeTypes: ['text/plain'] })); + parseText.mockResolvedValueOnce({ text: 'plain extracted text', bytes: 20 }); + getStrategyFunctions.mockReturnValue({ handleFileUpload: storageUpload }); + const req = makeReq({ mimetype: 'text/plain', ocrConfig: null }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(storageUpload).toHaveBeenCalledWith( + expect.objectContaining({ + file_id: 'file-uuid-123', + file: expect.objectContaining({ originalname: 'upload.bin' }), + }), + ); + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'plain extracted text', + bytes: 128, + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + source: FileSources.local, + filename: 'upload.bin', + type: 'text/plain', + llmDeliveryPath: 'text', + }), + true, + ); + }); + + test('stores the original file durably for OCR delivery records', async () => { + const { createFile } = require('~/models'); + const documentUpload = jest.fn().mockResolvedValue({ + text: 'ocr extracted text', + bytes: 42, + filepath: 'document_parser', + }); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 4096, + filename: 'upload.bin', + embedded: false, + }); + getStrategyFunctions.mockImplementation((source) => { + if (source === FileSources.document_parser) { + return { handleFileUpload: documentUpload }; + } + return { handleFileUpload: storageUpload }; + }); + const req = makeReq({ mimetype: PDF_MIME, ocrConfig: null }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(documentUpload).toHaveBeenCalled(); + expect(storageUpload).toHaveBeenCalled(); + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'ocr extracted text', + bytes: 4096, + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + source: FileSources.local, + filename: 'upload.bin', + type: PDF_MIME, + llmDeliveryPath: 'text', + }), + true, + ); + }); + }); + + describe('explicit legacy tool delivery path', () => { + test('persists llmDeliveryPath none for explicit file_search uploads', async () => { + const { createFile } = require('~/models'); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 128, + filename: 'upload.bin', + embedded: false, + }); + getStrategyFunctions.mockReturnValue({ handleFileUpload: storageUpload }); + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + defaultLLMDeliveryPath: { + fallback: 'text', + }, + }); + const req = makeReq({ mimetype: 'text/markdown', ocrConfig: null }); + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + ...makeMetadata(), + tool_resource: EToolResources.file_search, + }, + }); + + expect(checkCapability).toHaveBeenCalledWith( + expect.anything(), + AgentCapabilities.file_search, + ); + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + source: FileSources.local, + type: 'text/markdown', + embedded: true, + llmDeliveryPath: 'none', + }), + true, + ); + }); + + test('persists llmDeliveryPath provider for legacy provider uploads without tool_resource', async () => { + const { createFile } = require('~/models'); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 128, + filename: 'upload.bin', + embedded: false, + }); + getStrategyFunctions.mockReturnValue({ handleFileUpload: storageUpload }); + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + legacyFileUploadUX: true, + defaultLLMDeliveryPath: { + fallback: 'none', + }, + }); + const req = makeReq({ mimetype: 'text/markdown', ocrConfig: null }); + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + agent_id: 'agent-abc', + message_file: 'true', + file_id: 'file-uuid-123', + }, + }); + + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + source: FileSources.local, + type: 'text/markdown', + llmDeliveryPath: 'provider', + }), + true, + ); + }); + + test('persists llmDeliveryPath none for explicit execute_code uploads', async () => { + const { createFile } = require('~/models'); + const codeUpload = jest.fn().mockResolvedValue('session-1/file.csv'); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 128, + filename: 'upload.bin', + embedded: false, + }); + getStrategyFunctions.mockImplementation((source) => { + if (source === FileSources.execute_code) { + return { handleFileUpload: codeUpload }; + } + return { handleFileUpload: storageUpload }; + }); + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + defaultLLMDeliveryPath: { + fallback: 'text', + }, + }); + const req = makeReq({ mimetype: 'text/csv', ocrConfig: null }); + req.file.path = __filename; + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + ...makeMetadata(), + tool_resource: EToolResources.execute_code, + }, + }); + + expect(checkCapability).toHaveBeenCalledWith( + expect.anything(), + AgentCapabilities.execute_code, + ); + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + source: FileSources.local, + type: 'text/csv', + metadata: { fileIdentifier: 'session-1/file.csv' }, + llmDeliveryPath: 'none', + }), + true, + ); + }); + + test('persists llmDeliveryPath text for explicit context uploads', async () => { + const { parseText } = require('@librechat/api'); + const { createFile } = require('~/models'); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 128, + filename: 'upload.bin', + embedded: false, + }); + getStrategyFunctions.mockReturnValue({ handleFileUpload: storageUpload }); + mergeFileConfig.mockReturnValue( + makeFileConfig({ textSupportedMimeTypes: ['text/markdown'] }), + ); + parseText.mockResolvedValueOnce({ text: 'markdown text', bytes: 13 }); + const req = makeReq({ mimetype: 'text/markdown', ocrConfig: null }); + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + ...makeMetadata(), + tool_resource: EToolResources.context, + }, + }); + + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'markdown text', + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + source: FileSources.local, + type: 'text/markdown', + llmDeliveryPath: 'text', + }), + true, + ); + }); + + test('normalizes explicit ocr uploads to context text delivery', async () => { + const { parseText } = require('@librechat/api'); + const { createFile, addAgentResourceFile } = require('~/models'); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 128, + filename: 'upload.bin', + embedded: false, + }); + getStrategyFunctions.mockReturnValue({ handleFileUpload: storageUpload }); + mergeFileConfig.mockReturnValue( + makeFileConfig({ textSupportedMimeTypes: ['text/markdown'] }), + ); + parseText.mockResolvedValueOnce({ text: 'markdown text', bytes: 13 }); + const req = makeReq({ mimetype: 'text/markdown', ocrConfig: null }); + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + ...makeMetadata(), + tool_resource: EToolResources.ocr, + }, + }); + + expect(addAgentResourceFile).toHaveBeenCalledWith( + expect.objectContaining({ + file_id: 'file-uuid-123', + agent_id: 'agent-abc', + tool_resource: EToolResources.context, + }), + ); + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'markdown text', + source: FileSources.local, + type: 'text/markdown', + llmDeliveryPath: 'text', + }), + true, + ); + }); + }); +}); + +describe('processImageFile', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockRes.status.mockReturnThis(); + mockRes.json.mockReturnValue({}); + mergeFileConfig.mockReturnValue(makeFileConfig()); + }); + + test('persists resolved llmDeliveryPath for image uploads', async () => { + const { createFile } = require('~/models'); + const handleImageUpload = jest.fn().mockResolvedValue({ + filepath: '/images/user-123/image.webp', + bytes: 256, + width: 100, + height: 80, + }); + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + defaultLLMDeliveryPath: { + overrides: { 'image/*': 'none' }, + }, + }); + getStrategyFunctions.mockReturnValue({ handleImageUpload }); + const req = makeReq({ mimetype: 'image/png', ocrConfig: null }); + + await processImageFile({ + req, + res: mockRes, + metadata: { + file_id: 'image-file-id', + temp_file_id: 'temp-image-file-id', + endpoint: EModelEndpoint.agents, + }, + }); + + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + file_id: 'image-file-id', + temp_file_id: 'temp-image-file-id', + filepath: '/images/user-123/image.webp', + source: FileSources.local, + type: 'image/webp', + llmDeliveryPath: 'none', + }), + true, + ); + }); + + test('persists provider llmDeliveryPath for legacy image provider uploads', async () => { + const { createFile } = require('~/models'); + const handleImageUpload = jest.fn().mockResolvedValue({ + filepath: '/images/user-123/image.webp', + bytes: 256, + width: 100, + height: 80, + }); + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + legacyFileUploadUX: true, + defaultLLMDeliveryPath: { + overrides: { 'image/*': 'none' }, + }, + }); + getStrategyFunctions.mockReturnValue({ handleImageUpload }); + const req = makeReq({ mimetype: 'image/png', ocrConfig: null }); + + await processImageFile({ + req, + res: mockRes, + metadata: { + file_id: 'image-file-id', + temp_file_id: 'temp-image-file-id', + endpoint: EModelEndpoint.agents, + }, + }); + + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + file_id: 'image-file-id', + temp_file_id: 'temp-image-file-id', + filepath: '/images/user-123/image.webp', + source: FileSources.local, + type: 'image/webp', + llmDeliveryPath: 'provider', + }), + true, + ); + }); }); describe('processFileURL', () => { diff --git a/api/server/services/Files/provision.js b/api/server/services/Files/provision.js index 1db7afd341..61cde00266 100644 --- a/api/server/services/Files/provision.js +++ b/api/server/services/Files/provision.js @@ -13,6 +13,11 @@ const { FileSources } = require('librechat-data-provider'); const { loadAuthValues } = require('~/server/services/Tools/credentials'); const { getStrategyFunctions } = require('./strategies'); +// TODO: check and potentially fix — concurrent temp file collision (deterministic path based on file_id) +// TODO: check and potentially fix — query params not forwarded in checkSessionsAlive batch liveness check +// TODO: check and potentially fix — direct mutation of shared file objects in provisionFiles callback +// TODO: check and potentially fix — this file should be TypeScript in packages/api per CLAUDE.md rules + const axios = createAxiosInstance(); /** diff --git a/client/src/components/Chat/Input/Files/AttachFileMenu.tsx b/client/src/components/Chat/Input/Files/AttachFileMenu.tsx index 788e1fb967..59597171dd 100644 --- a/client/src/components/Chat/Input/Files/AttachFileMenu.tsx +++ b/client/src/components/Chat/Input/Files/AttachFileMenu.tsx @@ -111,9 +111,7 @@ const AttachFileMenu = ({ ephemeralAgent, ); - const isUnifiedMode = - endpointFileConfig?.defaultFileInteraction != null && - endpointFileConfig.defaultFileInteraction !== 'legacy'; + const isUnifiedMode = endpointFileConfig?.legacyFileUploadUX !== true; const handleUploadClick = useCallback( (fileType?: FileUploadType) => { diff --git a/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx b/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx index 25b2014d51..5aec4721a5 100644 --- a/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx +++ b/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx @@ -137,6 +137,7 @@ function renderMenu(props: Record = {}) { setFiles={() => {}} setFilesLoading={() => {}} conversation={null} + endpointFileConfig={{ legacyFileUploadUX: true }} {...props} /> @@ -262,6 +263,15 @@ describe('AttachFileMenu', () => { renderMenu({ disabled: false }); expect(screen.getByRole('button', { name: /attach file options/i })).not.toBeDisabled(); }); + + it('renders the unified upload button when legacyFileUploadUX is not true', () => { + setupMocks(); + renderMenu({ endpointFileConfig: { legacyFileUploadUX: false } }); + expect(screen.getByRole('button', { name: /attach files/i })).toBeInTheDocument(); + expect( + screen.queryByRole('button', { name: /attach file options/i }), + ).not.toBeInTheDocument(); + }); }); describe('Agent Capabilities', () => { diff --git a/client/src/utils/files.ts b/client/src/utils/files.ts index cc90110d53..d374482bd7 100644 --- a/client/src/utils/files.ts +++ b/client/src/utils/files.ts @@ -273,10 +273,7 @@ export const validateFiles = ({ } let mimeTypesToCheck = supportedMimeTypes; - const isUnifiedMode = - !toolResource && - endpointFileConfig.defaultFileInteraction != null && - endpointFileConfig.defaultFileInteraction !== 'legacy'; + const isUnifiedMode = !toolResource && endpointFileConfig?.legacyFileUploadUX !== true; if (toolResource === EToolResources.context || isUnifiedMode) { mimeTypesToCheck = [ ...(supportedMimeTypes || []), diff --git a/packages/api/src/agents/resources.test.ts b/packages/api/src/agents/resources.test.ts index 04244d6322..eb18abd8bc 100644 --- a/packages/api/src/agents/resources.test.ts +++ b/packages/api/src/agents/resources.test.ts @@ -1553,4 +1553,114 @@ describe('primeResources', () => { expect(result.tool_resources?.[EToolResources.image_edit]).toBeUndefined(); }); }); + + describe('llmDeliveryPath handling', () => { + it('should keep files with llmDeliveryPath "none" in attachments', async () => { + const providerFile: TFile = { + user: 'user1', + file_id: 'provider-file', + filename: 'image.png', + filepath: '/path/image.png', + type: 'image/png', + bytes: 1000, + object: 'file' as const, + usage: 0, + embedded: false, + source: 'local', + llmDeliveryPath: 'provider', + width: 100, + height: 100, + }; + const noneFile: TFile = { + user: 'user1', + file_id: 'none-file', + filename: 'audio.mp3', + filepath: '/path/audio.mp3', + type: 'audio/mpeg', + bytes: 5000, + object: 'file' as const, + usage: 0, + embedded: false, + source: 'local', + llmDeliveryPath: 'none', + }; + + const result = await primeResources({ + req: mockReq, + appConfig: mockAppConfig, + getFiles: mockGetFiles, + filterFiles: mockFilterFiles, + tool_resources: {}, + attachments: Promise.resolve([providerFile, noneFile]), + requestFileSet, + agentId: 'agent1', + }); + + const attachmentIds = result.attachments.map((f) => f.file_id); + expect(attachmentIds).toContain('provider-file'); + expect(attachmentIds).toContain('none-file'); + }); + + it('should include llmDeliveryPath "none" files in lazy provisioning state', async () => { + const noneFile: TFile = { + user: 'user1', + file_id: 'none-file', + filename: 'data.csv', + filepath: '/path/data.csv', + type: 'text/csv', + bytes: 5000, + object: 'file' as const, + usage: 0, + embedded: false, + source: 'local', + llmDeliveryPath: 'none', + }; + + const result = await primeResources({ + req: mockReq, + appConfig: mockAppConfig, + getFiles: mockGetFiles, + filterFiles: mockFilterFiles, + tool_resources: {}, + attachments: Promise.resolve([noneFile]), + requestFileSet, + agentId: 'agent1', + enabledToolResources: new Set([EToolResources.execute_code, EToolResources.file_search]), + loadCodeApiKey: jest.fn().mockResolvedValue('code-key'), + }); + + expect(result.attachments.map((f) => f.file_id)).toContain('none-file'); + expect(result.provisionState?.codeEnvFiles.map((f) => f.file_id)).toContain('none-file'); + expect(result.provisionState?.vectorDBFiles.map((f) => f.file_id)).toContain('none-file'); + }); + + it('should include files with undefined llmDeliveryPath in attachments (legacy files)', async () => { + const legacyFile: TFile = { + user: 'user1', + file_id: 'legacy-file', + filename: 'doc.pdf', + filepath: '/path/doc.pdf', + type: 'application/pdf', + bytes: 2000, + object: 'file' as const, + usage: 0, + embedded: false, + source: 'local', + }; + + const result = await primeResources({ + req: mockReq, + appConfig: mockAppConfig, + getFiles: mockGetFiles, + filterFiles: mockFilterFiles, + tool_resources: {}, + attachments: Promise.resolve([legacyFile]), + requestFileSet, + agentId: 'agent1', + }); + + const attachmentIds = result.attachments.map((f) => f.file_id); + expect(attachmentIds).toContain('legacy-file'); + }); + }); }); diff --git a/packages/api/src/agents/resources.ts b/packages/api/src/agents/resources.ts index ff7ac25cc2..d5e3eff94c 100644 --- a/packages/api/src/agents/resources.ts +++ b/packages/api/src/agents/resources.ts @@ -328,20 +328,17 @@ export const primeResources = async ({ continue; } - // Clear from attachmentFileIds if it was pre-added attachmentFileIds.delete(file.file_id); - // Add to attachments - attachments.push(file); - attachmentFileIds.add(file.file_id); - - // Categorize for tool resources categorizeFileForToolResources({ file, tool_resources, requestFileSet, processedResourceFiles, }); + + attachments.push(file); + attachmentFileIds.add(file.file_id); } } diff --git a/packages/api/src/files/context.test.ts b/packages/api/src/files/context.test.ts new file mode 100644 index 0000000000..efc399f47e --- /dev/null +++ b/packages/api/src/files/context.test.ts @@ -0,0 +1,64 @@ +import { FileSources } from 'librechat-data-provider'; + +import type { IMongoFile } from '@librechat/data-schemas'; +import type { ServerRequest } from '~/types'; + +import { extractFileContext } from './context'; + +const makeReq = () => + ({ + body: { fileTokenLimit: 1000 }, + config: { fileConfig: {} }, + }) as ServerRequest; + +const countTokens = (text: string) => text.length; + +describe('extractFileContext', () => { + it('should skip files with llmDeliveryPath "none"', async () => { + const file = { + filename: 'hidden.txt', + source: FileSources.text, + text: 'do not include this', + llmDeliveryPath: 'none', + } as IMongoFile; + + await expect( + extractFileContext({ attachments: [file], req: makeReq(), tokenCountFn: countTokens }), + ).resolves.toBeUndefined(); + }); + + it('should include legacy text-source files with undefined llmDeliveryPath', async () => { + const file = { + filename: 'legacy.txt', + source: FileSources.text, + text: 'legacy text', + } as IMongoFile; + + const result = await extractFileContext({ + attachments: [file], + req: makeReq(), + tokenCountFn: countTokens, + }); + + expect(result).toContain('# "legacy.txt"'); + expect(result).toContain('legacy text'); + }); + + it('should include standard-storage files with text and llmDeliveryPath "text"', async () => { + const file = { + filename: 'stored.txt', + source: FileSources.local, + text: 'stored extracted text', + llmDeliveryPath: 'text', + } as IMongoFile; + + const result = await extractFileContext({ + attachments: [file], + req: makeReq(), + tokenCountFn: countTokens, + }); + + expect(result).toContain('# "stored.txt"'); + expect(result).toContain('stored extracted text'); + }); +}); diff --git a/packages/api/src/files/context.ts b/packages/api/src/files/context.ts index 1da6c387ce..0e25ce21c7 100644 --- a/packages/api/src/files/context.ts +++ b/packages/api/src/files/context.ts @@ -38,7 +38,12 @@ export async function extractFileContext({ for (const file of attachments) { const source = file.source ?? FileSources.local; - if (source === FileSources.text && file.text) { + if (file.llmDeliveryPath === 'none') { + continue; + } + + const hasTextDelivery = file.llmDeliveryPath === 'text' || source === FileSources.text; + if (hasTextDelivery && file.text) { const { text: limitedText, wasTruncated } = await processTextWithTokenLimit({ text: file.text, tokenLimit: fileTokenLimit, diff --git a/packages/api/src/files/encode/processAttachments.spec.ts b/packages/api/src/files/encode/processAttachments.spec.ts index f9af6fd1e7..01855dbf67 100644 --- a/packages/api/src/files/encode/processAttachments.spec.ts +++ b/packages/api/src/files/encode/processAttachments.spec.ts @@ -17,6 +17,7 @@ function categorizeFile( source?: string; embedded?: boolean; metadata?: { fileIdentifier?: string; codeEnvRef?: unknown }; + llmDeliveryPath?: 'provider' | 'text' | 'none'; }, isBedrock: boolean, mergedFileConfig: FileConfig | undefined, @@ -26,6 +27,9 @@ function categorizeFile( if (source === FileSources.text) { return 'skipped'; } + if (file.llmDeliveryPath === 'text' || file.llmDeliveryPath === 'none') { + return 'skipped'; + } if ( file.embedded === true || file.metadata?.codeEnvRef != null || @@ -154,6 +158,28 @@ describe('processAttachments — supportedMimeTypes routing logic', () => { expect(result).toBe('skipped'); }); + it('should skip text-delivery markdown files regardless of config', () => { + const { merged, epConfig } = resolveConfig(['.*']); + const result = categorizeFile( + { type: 'text/markdown', llmDeliveryPath: 'text' }, + false, + merged, + epConfig, + ); + expect(result).toBe('skipped'); + }); + + it('should skip none-delivery markdown files regardless of config', () => { + const { merged, epConfig } = resolveConfig(['.*']); + const result = categorizeFile( + { type: 'text/markdown', llmDeliveryPath: 'none' }, + false, + merged, + epConfig, + ); + expect(result).toBe('skipped'); + }); + it('should skip embedded files regardless of config', () => { const { merged, epConfig } = resolveConfig(['.*']); const result = categorizeFile({ type: 'text/csv', embedded: true }, false, merged, epConfig); diff --git a/packages/data-provider/src/file-config.spec.ts b/packages/data-provider/src/file-config.spec.ts index f597353876..9a030f8039 100644 --- a/packages/data-provider/src/file-config.spec.ts +++ b/packages/data-provider/src/file-config.spec.ts @@ -1,3 +1,4 @@ +import type { TDefaultLLMDeliveryPathConfig } from './file-config'; import type { FileConfig } from './types/files'; import { fileConfig as baseFileConfig, @@ -1336,3 +1337,90 @@ describe('isPermissiveMimeConfig', () => { expect(isPermissiveMimeConfig(converted)).toBe(true); }); }); + +describe('defaultLLMDeliveryPath config merging', () => { + it('should include defaultLLMDeliveryPath and legacyFileUploadUX in merged endpoint config', () => { + const merged = mergeFileConfig({ + endpoints: { + [EModelEndpoint.agents]: { + defaultLLMDeliveryPath: { + fallback: 'none', + overrides: { 'image/*': 'text' }, + }, + legacyFileUploadUX: true, + }, + }, + }); + const endpointConfig = getEndpointFileConfig({ + fileConfig: merged, + endpoint: EModelEndpoint.agents, + }); + expect(endpointConfig.defaultLLMDeliveryPath).toEqual({ + fallback: 'none', + overrides: { 'image/*': 'text' }, + }); + expect(endpointConfig.legacyFileUploadUX).toBe(true); + }); + + it('should merge global defaultLLMDeliveryPath into mergedConfig', () => { + const merged = mergeFileConfig({ + defaultLLMDeliveryPath: { + fallback: 'provider', + overrides: { 'audio/*': 'none' }, + }, + legacyFileUploadUX: true, + }); + expect(merged.defaultLLMDeliveryPath).toEqual({ + fallback: 'provider', + overrides: { 'audio/*': 'none' }, + }); + expect(merged.legacyFileUploadUX).toBe(true); + }); + + it('should inherit global legacyFileUploadUX into endpoint config', () => { + const merged = mergeFileConfig({ + legacyFileUploadUX: true, + }); + const endpointConfig = getEndpointFileConfig({ + fileConfig: merged, + endpoint: EModelEndpoint.openAI, + }); + expect(endpointConfig.legacyFileUploadUX).toBe(true); + }); + + it('should allow endpoint legacyFileUploadUX to override global legacyFileUploadUX', () => { + const merged = mergeFileConfig({ + legacyFileUploadUX: true, + endpoints: { + [EModelEndpoint.openAI]: { + legacyFileUploadUX: false, + }, + }, + }); + const endpointConfig = getEndpointFileConfig({ + fileConfig: merged, + endpoint: EModelEndpoint.openAI, + }); + expect(endpointConfig.legacyFileUploadUX).toBe(false); + }); + + it('should pass through endpoint defaultLLMDeliveryPath in mergeWithDefault', () => { + const merged = mergeFileConfig({ + endpoints: { + [EModelEndpoint.openAI]: { + defaultLLMDeliveryPath: { overrides: { 'application/pdf': 'text' } }, + }, + }, + }); + const endpointConfig = getEndpointFileConfig({ + fileConfig: merged, + endpoint: EModelEndpoint.openAI, + }); + expect(endpointConfig.defaultLLMDeliveryPath?.overrides?.['application/pdf']).toBe('text'); + }); + + it('should default legacyFileUploadUX to undefined when not set', () => { + const merged = mergeFileConfig(undefined); + expect(merged.legacyFileUploadUX).toBeUndefined(); + }); +}); diff --git a/packages/data-provider/src/file-config.ts b/packages/data-provider/src/file-config.ts index 9917955702..911ecbaab2 100644 --- a/packages/data-provider/src/file-config.ts +++ b/packages/data-provider/src/file-config.ts @@ -455,8 +455,14 @@ export const fileConfig = { const supportedMimeTypesSchema = z.array(z.string()).optional(); -export const FileInteractionMode = z.enum(['text', 'provider', 'deferred', 'legacy']); -export type TFileInteractionMode = z.infer; +export const DefaultLLMDeliveryPath = z.enum(['provider', 'text', 'none']); +export type TDefaultLLMDeliveryPath = z.infer; + +export const defaultLLMDeliveryPathSchema = z.object({ + fallback: DefaultLLMDeliveryPath.optional(), + overrides: z.record(DefaultLLMDeliveryPath).optional(), +}); +export type TDefaultLLMDeliveryPathConfig = z.infer; export const endpointFileConfigSchema = z.object({ disabled: z.boolean().optional(), @@ -464,7 +470,8 @@ export const endpointFileConfigSchema = z.object({ fileSizeLimit: z.number().min(0).optional(), totalSizeLimit: z.number().min(0).optional(), supportedMimeTypes: supportedMimeTypesSchema.optional(), - defaultFileInteraction: FileInteractionMode.optional(), + defaultLLMDeliveryPath: defaultLLMDeliveryPathSchema.optional(), + legacyFileUploadUX: z.boolean().optional(), }); const skillFileConfigSchema = z.object({ @@ -501,7 +508,8 @@ export const fileConfigSchema = z.object({ supportedMimeTypes: supportedMimeTypesSchema.optional(), }) .optional(), - defaultFileInteraction: FileInteractionMode.optional(), + defaultLLMDeliveryPath: defaultLLMDeliveryPathSchema.optional(), + legacyFileUploadUX: z.boolean().optional(), }); export type TFileConfig = z.infer; @@ -555,8 +563,9 @@ function mergeWithDefault( fileSizeLimit: endpointConfig.fileSizeLimit ?? defaultConfig.fileSizeLimit, totalSizeLimit: endpointConfig.totalSizeLimit ?? defaultConfig.totalSizeLimit, supportedMimeTypes: endpointConfig.supportedMimeTypes ?? defaultMimeTypes, - defaultFileInteraction: - endpointConfig.defaultFileInteraction ?? defaultConfig.defaultFileInteraction, + defaultLLMDeliveryPath: + endpointConfig.defaultLLMDeliveryPath ?? defaultConfig.defaultLLMDeliveryPath, + legacyFileUploadUX: endpointConfig.legacyFileUploadUX ?? defaultConfig.legacyFileUploadUX, }; } @@ -572,11 +581,17 @@ export function getEndpointFileConfig(params: { } /** Compute an effective default by merging user-configured default over the base default */ - const baseDefaultConfig = fileConfig.endpoints.default; + const baseDefaultConfig: EndpointFileConfig = fileConfig.endpoints.default; + const globalDefaultConfig: EndpointFileConfig = { + ...baseDefaultConfig, + defaultLLMDeliveryPath: + mergedFileConfig.defaultLLMDeliveryPath ?? baseDefaultConfig.defaultLLMDeliveryPath, + legacyFileUploadUX: mergedFileConfig.legacyFileUploadUX ?? baseDefaultConfig.legacyFileUploadUX, + }; const userDefaultConfig = mergedFileConfig.endpoints.default; const defaultConfig = userDefaultConfig - ? mergeWithDefault(userDefaultConfig, baseDefaultConfig, 'default') - : baseDefaultConfig; + ? mergeWithDefault(userDefaultConfig, globalDefaultConfig, 'default') + : globalDefaultConfig; const normalizedEndpoint = normalizeEndpointName(endpoint ?? ''); const standardEndpoints = new Set([ @@ -688,8 +703,12 @@ export function mergeFileConfig(dynamic: z.infer | unde return mergedConfig; } - if (dynamic.defaultFileInteraction !== undefined) { - mergedConfig.defaultFileInteraction = dynamic.defaultFileInteraction; + if (dynamic.defaultLLMDeliveryPath !== undefined) { + mergedConfig.defaultLLMDeliveryPath = dynamic.defaultLLMDeliveryPath; + } + + if (dynamic.legacyFileUploadUX !== undefined) { + mergedConfig.legacyFileUploadUX = dynamic.legacyFileUploadUX; } if (dynamic.serverFileSizeLimit !== undefined) { @@ -791,8 +810,12 @@ export function mergeFileConfig(dynamic: z.infer | unde ); } - if (dynamicEndpoint.defaultFileInteraction !== undefined) { - mergedEndpoint.defaultFileInteraction = dynamicEndpoint.defaultFileInteraction; + if (dynamicEndpoint.defaultLLMDeliveryPath !== undefined) { + mergedEndpoint.defaultLLMDeliveryPath = dynamicEndpoint.defaultLLMDeliveryPath; + } + + if (dynamicEndpoint.legacyFileUploadUX !== undefined) { + mergedEndpoint.legacyFileUploadUX = dynamicEndpoint.legacyFileUploadUX; } } diff --git a/packages/data-provider/src/index.ts b/packages/data-provider/src/index.ts index bb166d975a..f7a9d7b4ec 100644 --- a/packages/data-provider/src/index.ts +++ b/packages/data-provider/src/index.ts @@ -4,6 +4,7 @@ export * from './bedrock'; export * from './balance'; export * from './config'; export * from './file-config'; +export * from './resolve-llm-delivery-path'; /* messages */ export * from './messages'; /* artifacts */ diff --git a/packages/data-provider/src/resolve-llm-delivery-path.spec.ts b/packages/data-provider/src/resolve-llm-delivery-path.spec.ts new file mode 100644 index 0000000000..c69c676472 --- /dev/null +++ b/packages/data-provider/src/resolve-llm-delivery-path.spec.ts @@ -0,0 +1,103 @@ +import { resolveDefaultLLMDeliveryPath, SYSTEM_LLM_DELIVERY_DEFAULTS } from './resolve-llm-delivery-path'; +import type { TDefaultLLMDeliveryPathConfig } from './file-config'; + +describe('resolveDefaultLLMDeliveryPath', () => { + it('should return system default for images when no config provided', () => { + expect(resolveDefaultLLMDeliveryPath('image/png')).toBe('provider'); + }); + + it('should return system default for PDFs when no config provided', () => { + expect(resolveDefaultLLMDeliveryPath('application/pdf')).toBe('provider'); + }); + + it('should return system fallback for unknown mime types', () => { + expect(resolveDefaultLLMDeliveryPath('text/plain')).toBe('text'); + }); + + it('should match exact mime type before wildcard', () => { + const config: TDefaultLLMDeliveryPathConfig = { + overrides: { 'image/png': 'text', 'image/*': 'provider' }, + }; + expect(resolveDefaultLLMDeliveryPath('image/png', config)).toBe('text'); + }); + + it('should match wildcard when no exact match', () => { + const config: TDefaultLLMDeliveryPathConfig = { + overrides: { 'image/*': 'none' }, + }; + expect(resolveDefaultLLMDeliveryPath('image/jpeg', config)).toBe('none'); + }); + + it('should use config fallback when no override matches', () => { + const config: TDefaultLLMDeliveryPathConfig = { + fallback: 'none', + overrides: { 'image/*': 'provider' }, + }; + expect(resolveDefaultLLMDeliveryPath('text/plain', config)).toBe('none'); + }); + + it('should resolve endpoint config before global config', () => { + const endpointConfig: TDefaultLLMDeliveryPathConfig = { + overrides: { 'image/*': 'text' }, + }; + const globalConfig: TDefaultLLMDeliveryPathConfig = { + overrides: { 'image/*': 'provider' }, + }; + expect(resolveDefaultLLMDeliveryPath('image/png', endpointConfig, globalConfig)).toBe('text'); + }); + + it('should fall through to global config when endpoint has no match', () => { + const endpointConfig: TDefaultLLMDeliveryPathConfig = { + overrides: { 'audio/*': 'none' }, + }; + const globalConfig: TDefaultLLMDeliveryPathConfig = { + overrides: { 'image/*': 'text' }, + }; + expect(resolveDefaultLLMDeliveryPath('image/png', endpointConfig, globalConfig)).toBe('text'); + }); + + it('should use endpoint fallback before global overrides', () => { + const endpointConfig: TDefaultLLMDeliveryPathConfig = { + fallback: 'none', + }; + const globalConfig: TDefaultLLMDeliveryPathConfig = { + overrides: { 'text/*': 'provider' }, + }; + expect(resolveDefaultLLMDeliveryPath('text/plain', endpointConfig, globalConfig)).toBe('none'); + }); + + it('should fall through entire chain to system defaults', () => { + const endpointConfig: TDefaultLLMDeliveryPathConfig = {}; + const globalConfig: TDefaultLLMDeliveryPathConfig = {}; + expect(resolveDefaultLLMDeliveryPath('image/png', endpointConfig, globalConfig)).toBe('provider'); + expect(resolveDefaultLLMDeliveryPath('application/pdf', endpointConfig, globalConfig)).toBe('provider'); + expect(resolveDefaultLLMDeliveryPath('text/csv', endpointConfig, globalConfig)).toBe('text'); + }); + + it('should resolve none destination correctly', () => { + const config: TDefaultLLMDeliveryPathConfig = { + overrides: { 'audio/*': 'none' }, + }; + expect(resolveDefaultLLMDeliveryPath('audio/mpeg', config)).toBe('none'); + }); + + it('should prefer exact match over wildcard in the same config', () => { + const config: TDefaultLLMDeliveryPathConfig = { + overrides: { 'image/*': 'provider', 'image/svg+xml': 'text' }, + }; + expect(resolveDefaultLLMDeliveryPath('image/svg+xml', config)).toBe('text'); + expect(resolveDefaultLLMDeliveryPath('image/png', config)).toBe('provider'); + }); + + it('should handle undefined configs gracefully', () => { + expect(resolveDefaultLLMDeliveryPath('text/plain', undefined, undefined)).toBe('text'); + }); + + it('should export SYSTEM_LLM_DELIVERY_DEFAULTS with correct shape', () => { + expect(SYSTEM_LLM_DELIVERY_DEFAULTS.fallback).toBe('text'); + expect(SYSTEM_LLM_DELIVERY_DEFAULTS.overrides).toEqual({ + 'image/*': 'provider', + 'application/pdf': 'provider', + }); + }); +}); diff --git a/packages/data-provider/src/resolve-llm-delivery-path.ts b/packages/data-provider/src/resolve-llm-delivery-path.ts new file mode 100644 index 0000000000..5d6e9d47af --- /dev/null +++ b/packages/data-provider/src/resolve-llm-delivery-path.ts @@ -0,0 +1,56 @@ +import type { TDefaultLLMDeliveryPath, TDefaultLLMDeliveryPathConfig } from './file-config'; + +export const SYSTEM_LLM_DELIVERY_DEFAULTS: Required = { + fallback: 'text', + overrides: { + 'image/*': 'provider', + 'application/pdf': 'provider', + }, +}; + +/** + * Resolves the default file path destination for a given mime type. + * Resolution chain: endpoint overrides -> endpoint fallback -> global overrides -> global fallback -> system defaults. + */ +export function resolveDefaultLLMDeliveryPath( + mimeType: string, + endpointConfig?: TDefaultLLMDeliveryPathConfig, + globalConfig?: TDefaultLLMDeliveryPathConfig, +): TDefaultLLMDeliveryPath { + const wildcard = mimeType.split('/')[0] + '/*'; + + if (endpointConfig?.overrides) { + if (endpointConfig.overrides[mimeType]) { + return endpointConfig.overrides[mimeType] as TDefaultLLMDeliveryPath; + } + if (endpointConfig.overrides[wildcard]) { + return endpointConfig.overrides[wildcard] as TDefaultLLMDeliveryPath; + } + } + + if (endpointConfig?.fallback) { + return endpointConfig.fallback; + } + + if (globalConfig?.overrides) { + if (globalConfig.overrides[mimeType]) { + return globalConfig.overrides[mimeType] as TDefaultLLMDeliveryPath; + } + if (globalConfig.overrides[wildcard]) { + return globalConfig.overrides[wildcard] as TDefaultLLMDeliveryPath; + } + } + + if (globalConfig?.fallback) { + return globalConfig.fallback; + } + + if (SYSTEM_LLM_DELIVERY_DEFAULTS.overrides[mimeType]) { + return SYSTEM_LLM_DELIVERY_DEFAULTS.overrides[mimeType] as TDefaultLLMDeliveryPath; + } + if (SYSTEM_LLM_DELIVERY_DEFAULTS.overrides[wildcard]) { + return SYSTEM_LLM_DELIVERY_DEFAULTS.overrides[wildcard] as TDefaultLLMDeliveryPath; + } + + return SYSTEM_LLM_DELIVERY_DEFAULTS.fallback; +} diff --git a/packages/data-provider/src/types/files.ts b/packages/data-provider/src/types/files.ts index 12323f1c84..55fe621473 100644 --- a/packages/data-provider/src/types/files.ts +++ b/packages/data-provider/src/types/files.ts @@ -1,5 +1,6 @@ import { EToolResources } from './assistants'; import type { CodeEnvRef } from '../codeEnvRef'; +import type { TDefaultLLMDeliveryPathConfig } from '../file-config'; export enum FileSources { local = 'local', @@ -45,7 +46,8 @@ export type EndpointFileConfig = { fileSizeLimit?: number; totalSizeLimit?: number; supportedMimeTypes?: RegExp[]; - defaultFileInteraction?: 'text' | 'provider' | 'deferred' | 'legacy'; + defaultLLMDeliveryPath?: TDefaultLLMDeliveryPathConfig; + legacyFileUploadUX?: boolean; }; export type FileConfig = { @@ -74,7 +76,8 @@ export type FileConfig = { supportedMimeTypes?: RegExp[]; }; checkType?: (fileType: string, supportedTypes: RegExp[]) => boolean; - defaultFileInteraction?: 'text' | 'provider' | 'deferred' | 'legacy'; + defaultLLMDeliveryPath?: TDefaultLLMDeliveryPathConfig; + legacyFileUploadUX?: boolean; }; export type FileConfigInput = { @@ -102,7 +105,8 @@ export type FileConfigInput = { supportedMimeTypes?: string[]; }; checkType?: (fileType: string, supportedTypes: RegExp[]) => boolean; - defaultFileInteraction?: 'text' | 'provider' | 'deferred' | 'legacy'; + defaultLLMDeliveryPath?: TDefaultLLMDeliveryPathConfig; + legacyFileUploadUX?: boolean; }; export type TFile = { @@ -163,6 +167,7 @@ export type TFile = { */ codeEnvRef?: CodeEnvRef; }; + llmDeliveryPath?: 'provider' | 'text' | 'none'; createdAt?: string | Date; updatedAt?: string | Date; }; diff --git a/packages/data-schemas/src/schema/file.ts b/packages/data-schemas/src/schema/file.ts index c8e7c72f52..c950093a32 100644 --- a/packages/data-schemas/src/schema/file.ts +++ b/packages/data-schemas/src/schema/file.ts @@ -136,6 +136,10 @@ const file: Schema = new Schema( default: undefined, }, }, + llmDeliveryPath: { + type: String, + enum: ['provider', 'text', 'none'], + }, expiresAt: { type: Date, expires: 3600, // 1 hour in seconds diff --git a/packages/data-schemas/src/types/file.ts b/packages/data-schemas/src/types/file.ts index b47a18abb6..1e5f1f38e2 100644 --- a/packages/data-schemas/src/types/file.ts +++ b/packages/data-schemas/src/types/file.ts @@ -71,6 +71,7 @@ export interface IMongoFile extends Omit { */ codeEnvRef?: CodeEnvRef; }; + llmDeliveryPath?: string; expiresAt?: Date; createdAt?: Date; updatedAt?: Date;