From d12e5e1523dc816e5aa78190bb7f7129588d7658 Mon Sep 17 00:00:00 2001 From: Atef Bellaaj Date: Sun, 12 Apr 2026 19:58:15 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A7=20feat:=20Unified=20file=20upload?= =?UTF-8?q?=20=E2=80=94=20per-mime-type=20routing=20with=20lazy=20provisio?= =?UTF-8?q?ning?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/app/clients/BaseClient.js | 4 + api/app/clients/specs/BaseClient.test.js | 100 ++++- api/server/index.js | 26 ++ api/server/services/Files/process.js | 135 +++--- api/server/services/Files/process.spec.js | 389 ++++++++++++++++++ api/server/services/Files/provision.js | 5 + .../Chat/Input/Files/AttachFileMenu.tsx | 4 +- .../Files/__tests__/AttachFileMenu.spec.tsx | 10 + client/src/utils/files.ts | 5 +- packages/api/src/agents/resources.test.ts | 110 +++++ packages/api/src/agents/resources.ts | 11 +- packages/api/src/files/context.test.ts | 64 +++ packages/api/src/files/context.ts | 7 +- .../files/encode/processAttachments.spec.ts | 26 ++ .../data-provider/src/file-config.spec.ts | 89 +++- packages/data-provider/src/file-config.ts | 49 ++- packages/data-provider/src/index.ts | 1 + .../src/resolve-llm-delivery-path.spec.ts | 103 +++++ .../src/resolve-llm-delivery-path.ts | 56 +++ packages/data-provider/src/types/files.ts | 11 +- packages/data-schemas/src/schema/file.ts | 4 + packages/data-schemas/src/types/file.ts | 1 + 22 files changed, 1129 insertions(+), 81 deletions(-) create mode 100644 packages/api/src/files/context.test.ts create mode 100644 packages/data-provider/src/resolve-llm-delivery-path.spec.ts create mode 100644 packages/data-provider/src/resolve-llm-delivery-path.ts diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index 839a135aed..b36f987190 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -1733,6 +1733,10 @@ class BaseClient { allFiles.push(file); continue; } + if (file.llmDeliveryPath === 'text' || file.llmDeliveryPath === 'none') { + allFiles.push(file); + continue; + } if ( file.embedded === true || file.metadata?.codeEnvRef != null || diff --git a/api/app/clients/specs/BaseClient.test.js b/api/app/clients/specs/BaseClient.test.js index 3bb7f57e63..35f50293d1 100644 --- a/api/app/clients/specs/BaseClient.test.js +++ b/api/app/clients/specs/BaseClient.test.js @@ -1,4 +1,4 @@ -const { Constants, ContentTypes } = require('librechat-data-provider'); +const { Constants, ContentTypes, EModelEndpoint } = require('librechat-data-provider'); const BaseClientClass = require('../BaseClient'); const { ContentFilterError } = require('@librechat/api'); const { FakeClient, initializeFakeClient } = require('./FakeClient'); @@ -2932,4 +2932,102 @@ describe('BaseClient', () => { ]); }); }); + + describe('processAttachments llmDeliveryPath handling', () => { + beforeEach(() => { + TestClient.options = { + endpoint: EModelEndpoint.openAI, + }; + TestClient.addImageURLs = jest.fn(async (message, files) => { + message.image_urls = ['encoded-image']; + return files; + }); + TestClient.addDocuments = jest.fn(async (message, files) => { + message.documents = [{ type: 'file' }]; + return files; + }); + TestClient.addVideos = jest.fn(async (_message, files) => files); + TestClient.addAudios = jest.fn(async (_message, files) => files); + }); + + test('keeps a none image in returned files without adding image URLs', async () => { + const message = {}; + const file = { + user: 'user1', + file_id: 'none-image', + filename: 'image.png', + filepath: '/uploads/image.png', + type: 'image/png', + bytes: 100, + source: 'local', + llmDeliveryPath: 'none', + }; + + const result = await TestClient.processAttachments(message, [file]); + + expect(result).toEqual([file]); + expect(message.image_urls).toBeUndefined(); + expect(TestClient.addImageURLs).not.toHaveBeenCalled(); + }); + + test('keeps a none PDF in returned files without adding documents', async () => { + const message = {}; + const file = { + user: 'user1', + file_id: 'none-pdf', + filename: 'document.pdf', + filepath: '/uploads/document.pdf', + type: 'application/pdf', + bytes: 100, + source: 'local', + llmDeliveryPath: 'none', + }; + + const result = await TestClient.processAttachments(message, [file]); + + expect(result).toEqual([file]); + expect(message.documents).toBeUndefined(); + expect(TestClient.addDocuments).not.toHaveBeenCalled(); + }); + + test('keeps a text-delivery markdown file in returned files without adding documents', async () => { + const message = {}; + const file = { + user: 'user1', + file_id: 'text-markdown', + filename: 'notes.md', + filepath: '/uploads/notes.md', + type: 'text/markdown', + bytes: 100, + source: 'local', + text: 'extracted markdown', + llmDeliveryPath: 'text', + }; + + const result = await TestClient.processAttachments(message, [file]); + + expect(result).toEqual([file]); + expect(message.documents).toBeUndefined(); + expect(TestClient.addDocuments).not.toHaveBeenCalled(); + }); + + test('routes legacy files without llmDeliveryPath normally', async () => { + const message = {}; + const file = { + user: 'user1', + file_id: 'legacy-pdf', + filename: 'document.pdf', + filepath: '/uploads/document.pdf', + type: 'application/pdf', + bytes: 100, + source: 'local', + }; + + const result = await TestClient.processAttachments(message, [file]); + + expect(result).toEqual([file]); + expect(message.documents).toEqual([{ type: 'file' }]); + expect(TestClient.addDocuments).toHaveBeenCalledWith(message, [file]); + }); + }); }); diff --git a/api/server/index.js b/api/server/index.js index 1c4a91244f..d9e7bc76dd 100644 --- a/api/server/index.js +++ b/api/server/index.js @@ -177,6 +177,32 @@ const startServer = async () => { }); const appConfig = await getAppConfig({ baseOnly: true }); configureAgentEventRuntime(appConfig?.endpoints?.agents?.eventDriven); + if (appConfig?.fileConfig?.defaultLLMDeliveryPath?.overrides) { + for (const [mimeType, destination] of Object.entries( + appConfig.fileConfig.defaultLLMDeliveryPath.overrides, + )) { + if (destination === 'none') { + logger.warn( + `[Config] defaultLLMDeliveryPath: "${mimeType}" is set to "none" — files of this type will only be accessible through tool provisioning`, + ); + } + } + } + if (appConfig?.fileConfig?.endpoints) { + for (const [endpoint, config] of Object.entries(appConfig.fileConfig.endpoints)) { + if (config?.defaultLLMDeliveryPath?.overrides) { + for (const [mimeType, destination] of Object.entries( + config.defaultLLMDeliveryPath.overrides, + )) { + if (destination === 'none') { + logger.warn( + `[Config] defaultLLMDeliveryPath for "${endpoint}": "${mimeType}" is set to "none" — files of this type will only be accessible through tool provisioning`, + ); + } + } + } + } + } initializeFileStorage(appConfig); const projectRoot = path.resolve(__dirname, '../..'); // Plugin hooks execute only when the operator opts in via DEPLOYMENT_PLUGIN_HOOKS; diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js index 3d0c521b90..88f2ced03e 100644 --- a/api/server/services/Files/process.js +++ b/api/server/services/Files/process.js @@ -17,6 +17,9 @@ const { removeNullishValues, isAssistantsEndpoint, getEndpointFileConfig, + resolveDefaultLLMDeliveryPath, + documentParserMimeTypes, + isPermissiveMimeConfig, } = require('librechat-data-provider'); const { logger, runAsSystem } = require('@librechat/data-schemas'); const { @@ -448,6 +451,20 @@ const processFileURL = async ({ } }; +const resolveDefaultUploadLLMDeliveryPath = ({ file, endpointConfig, fileConfig }) => { + const isLegacyFileUploadUX = + endpointConfig?.legacyFileUploadUX === true || fileConfig?.legacyFileUploadUX === true; + if (isLegacyFileUploadUX) { + return 'provider'; + } + + return resolveDefaultLLMDeliveryPath( + file.mimetype, + endpointConfig?.defaultLLMDeliveryPath, + fileConfig?.defaultLLMDeliveryPath, + ); +}; + /** * Applies the current strategy for image uploads. * Saves file metadata to the database with an expiry TTL. @@ -466,6 +483,9 @@ const processImageFile = async ({ req, res, metadata, returnFile = false, sseStr const source = getFileStrategy(appConfig, { isImage: true }); const { handleImageUpload } = getStrategyFunctions(source); const { file_id, temp_file_id, endpoint } = metadata; + const fileConfig = mergeFileConfig(appConfig?.fileConfig); + const endpointConfig = getEndpointFileConfig({ fileConfig, endpoint }); + const llmDeliveryPath = resolveDefaultUploadLLMDeliveryPath({ file, endpointConfig, fileConfig }); const { filepath, bytes, width, height, storageKey, storageRegion } = await handleImageUpload({ req, @@ -491,6 +511,7 @@ const processImageFile = async ({ req, res, metadata, returnFile = false, sseStr width, height, tenantId: req.user.tenantId, + llmDeliveryPath, }, true, ); @@ -664,20 +685,19 @@ const processFileUpload = async ({ req, res, metadata, sseStream }) => { sendUploadSuccess(res, sseStream, 'File uploaded and processed successfully', result); }; -/** - * Resolves the file interaction mode from the merged file config. - * Checks endpoint-level config first, then global config. - * Returns 'deferred' as the default when nothing is configured. - * - * @param {object} req - The Express request object - * @param {object} appConfig - The application config - * @returns {string} - The resolved interaction mode: 'text' | 'provider' | 'deferred' | 'legacy' - */ -const resolveInteractionMode = (req, appConfig) => { - const fileConfig = mergeFileConfig(appConfig?.fileConfig); - const endpoint = req.body?.endpoint; - const endpointConfig = getEndpointFileConfig({ fileConfig, endpoint }); - return endpointConfig?.defaultFileInteraction ?? fileConfig?.defaultFileInteraction ?? 'deferred'; +const resolveUploadLLMDeliveryPath = ({ tool_resource, file, endpointConfig, fileConfig }) => { + if (tool_resource === EToolResources.context || tool_resource === EToolResources.ocr) { + return 'text'; + } + + if ( + tool_resource === EToolResources.file_search || + tool_resource === EToolResources.execute_code + ) { + return 'none'; + } + + return resolveDefaultUploadLLMDeliveryPath({ file, endpointConfig, fileConfig }); }; /** @@ -693,22 +713,35 @@ const resolveInteractionMode = (req, appConfig) => { * @returns {Promise} */ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { + // TODO: check and potentially fix — deferred/provider files may be orphaned if effectiveToolResource is undefined const { file } = req; const appConfig = req.config; const { agent_id, tool_resource, file_id, temp_file_id = null } = metadata; let messageAttachment = !!metadata.message_file; - let effectiveToolResource = tool_resource; + let effectiveToolResource = + tool_resource === EToolResources.ocr ? EToolResources.context : tool_resource; + + const fileConfig = mergeFileConfig(appConfig?.fileConfig); + const endpoint = req.body?.endpoint; + const endpointConfig = getEndpointFileConfig({ fileConfig, endpoint }); + if (agent_id && !tool_resource && !messageAttachment) { - const interactionMode = resolveInteractionMode(req, appConfig); - if (interactionMode === 'legacy') { + if (endpointConfig?.legacyFileUploadUX === true || fileConfig?.legacyFileUploadUX === true) { throw new Error('No tool resource provided for agent file upload'); } - // In unified mode: 'text' routes to context processing, 'deferred'/'provider' fall through to standard storage - if (interactionMode === 'text') { - effectiveToolResource = EToolResources.context; - } + } + + const llmDeliveryPath = resolveUploadLLMDeliveryPath({ + tool_resource, + file, + endpointConfig, + fileConfig, + }); + + if (!tool_resource && llmDeliveryPath === 'text') { + effectiveToolResource = EToolResources.context; } if (effectiveToolResource === EToolResources.file_search && file.mimetype.startsWith('image')) { @@ -787,19 +820,10 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { /** * @param {object} params * @param {string} params.text - * @param {number} params.bytes - * @param {string} params.filepath - * @param {string} params.type * @param {boolean} params.isTranscript * @return {Promise} */ - const createTextFile = async ({ - text, - bytes, - filepath, - type = 'text/plain', - isTranscript = false, - }) => { + const createTextFile = async ({ text, isTranscript = false }) => { if (!isTranscript) { assertExtractedTextInspectable({ filters: appConfig?.filters, @@ -837,11 +861,25 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { return; } } + const isImageFile = file.mimetype.startsWith('image'); + const source = getFileStrategy(appConfig, { isImage: isImageFile }); + const { handleFileUpload } = getStrategyFunctions(source); + const sanitizedUploadFn = createSanitizedUploadWrapper(handleFileUpload); + const storageResult = await sanitizedUploadFn({ + req, + file, + file_id, + basePath, + entity_id, + }); + const { bytes, filename, filepath, embedded, height, width } = storageResult; + const retentionExpiry = await getAgentFileRetentionExpiry({ req, messageAttachment, tool_resource, }); + const fileInfo = { ...removeNullishValues({ text, @@ -849,13 +887,17 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { file_id, temp_file_id, user: req.user.id, - type, - filepath: filepath ?? file.path, - source: FileSources.text, - filename: file.originalname, + type: file.mimetype, + filepath, + source, + filename: filename ?? sanitizeFilename(file.originalname), model: messageAttachment ? undefined : req.body.model, context: messageAttachment ? FileContext.message_attachment : FileContext.agents, tenantId: req.user.tenantId, + embedded, + height, + width, + llmDeliveryPath: 'text', }), ...retentionExpiry, }; @@ -924,8 +966,8 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { extract: resolveDocumentText, }); if (ocrResult) { - const { text, bytes, filepath: ocrFileURL } = ocrResult; - return await createTextFile({ text, bytes, filepath: ocrFileURL }); + const { text } = ocrResult; + return await createTextFile({ text }); } throw new Error( `Unable to extract text from "${file.originalname}". The document may be image-based and requires an OCR service to process.`, @@ -939,8 +981,8 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { if (shouldUseSTT) { const sttService = await STTService.getInstance(); - const { text, bytes } = await processAudioFile({ req, file, sttService }); - return await createTextFile({ text, bytes, type: file.mimetype, isTranscript: true }); + const { text } = await processAudioFile({ req, file, sttService }); + return await createTextFile({ text, isTranscript: true }); } const shouldUseText = fileConfig.checkType( @@ -978,21 +1020,17 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { `Unable to extract text from "${file.originalname}". RAG text extraction was unavailable and the built-in parser produced no result.`, ); } - const { text, bytes, filepath: docFileURL } = documentText; - return await createTextFile({ text, bytes, filepath: docFileURL }); + const { text } = documentText; + return await createTextFile({ text }); } - return await createTextFile({ - text: configuredText.text, - bytes: configuredText.bytes, - type: file.mimetype, - }); + return await createTextFile({ text: configuredText.text }); } - const { text, bytes } = await extractInspectableFileText({ + const { text } = await extractInspectableFileText({ filters: appConfig?.filters, extract: () => parseText({ req, file, file_id }), }); - return await createTextFile({ text, bytes, type: file.mimetype }); + return await createTextFile({ text }); } // Dual storage pattern for RAG files: Storage + Vector DB @@ -1109,6 +1147,7 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { height, width, tenantId: req.user.tenantId, + llmDeliveryPath, }), ...retentionExpiry, }; diff --git a/api/server/services/Files/process.spec.js b/api/server/services/Files/process.spec.js index 14eaae5941..042e352805 100644 --- a/api/server/services/Files/process.spec.js +++ b/api/server/services/Files/process.spec.js @@ -211,6 +211,10 @@ jest.mock('~/server/services/Files/Audio/STTService', () => ({ STTService: { getInstance: jest.fn() }, })); +jest.mock('./VectorDB/crud', () => ({ + uploadVectors: jest.fn().mockResolvedValue({ embedded: true, filename: 'embedded-upload.bin' }), +})); + const { getRetentionExpiry, getAgentFileRetentionExpiry, @@ -218,6 +222,7 @@ const { startExpiredFileSweep: startExpiredFileSweepWithDeps, } = require('@librechat/api'); const { + EModelEndpoint, EToolResources, FileSources, FileContext, @@ -226,12 +231,14 @@ const { } = require('librechat-data-provider'); const { mergeFileConfig } = require('librechat-data-provider'); const { checkCapability } = require('~/server/services/Config'); +const { loadAuthValues } = require('~/server/services/Tools/credentials'); const { getStrategyFunctions } = require('~/server/services/Files/strategies'); const { uploadVectors } = require('./VectorDB/crud'); const { logger } = require('@librechat/data-schemas'); const db = require('~/models'); const { processAgentFileUpload, + processImageFile, processDeleteRequest, processFileURL, sweepExpiredFiles, @@ -285,6 +292,7 @@ const makeReq = ({ config: { fileConfig: {}, fileStrategy: 'local', + imageOutputType: 'webp', ocr: ocrConfig, ...(filters ? { filters } : {}), ...(interfaceConfig ? { interfaceConfig } : {}), @@ -331,6 +339,8 @@ describe('processAgentFileUpload', () => { mockRes.status.mockReturnThis(); mockRes.json.mockReturnValue({}); checkCapability.mockResolvedValue(true); + loadAuthValues.mockResolvedValue({ CODE_API_KEY: 'code-key' }); + uploadVectors.mockResolvedValue({ embedded: true, filename: 'embedded-upload.bin' }); getStrategyFunctions.mockReturnValue({ handleFileUpload: jest .fn() @@ -1350,6 +1360,385 @@ describe('processAgentFileUpload', () => { expect(persisted.metadata).not.toHaveProperty('fileIdentifier'); }); }); + + describe('text delivery storage', () => { + test('stores the original file durably for plain text delivery records', async () => { + const { parseText } = require('@librechat/api'); + const { createFile } = require('~/models'); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 128, + filename: 'upload.bin', + embedded: false, + }); + mergeFileConfig.mockReturnValue(makeFileConfig({ textSupportedMimeTypes: ['text/plain'] })); + parseText.mockResolvedValueOnce({ text: 'plain extracted text', bytes: 20 }); + getStrategyFunctions.mockReturnValue({ handleFileUpload: storageUpload }); + const req = makeReq({ mimetype: 'text/plain', ocrConfig: null }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(storageUpload).toHaveBeenCalledWith( + expect.objectContaining({ + file_id: 'file-uuid-123', + file: expect.objectContaining({ originalname: 'upload.bin' }), + }), + ); + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'plain extracted text', + bytes: 128, + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + source: FileSources.local, + filename: 'upload.bin', + type: 'text/plain', + llmDeliveryPath: 'text', + }), + true, + ); + }); + + test('stores the original file durably for OCR delivery records', async () => { + const { createFile } = require('~/models'); + const documentUpload = jest.fn().mockResolvedValue({ + text: 'ocr extracted text', + bytes: 42, + filepath: 'document_parser', + }); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 4096, + filename: 'upload.bin', + embedded: false, + }); + getStrategyFunctions.mockImplementation((source) => { + if (source === FileSources.document_parser) { + return { handleFileUpload: documentUpload }; + } + return { handleFileUpload: storageUpload }; + }); + const req = makeReq({ mimetype: PDF_MIME, ocrConfig: null }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(documentUpload).toHaveBeenCalled(); + expect(storageUpload).toHaveBeenCalled(); + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'ocr extracted text', + bytes: 4096, + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + source: FileSources.local, + filename: 'upload.bin', + type: PDF_MIME, + llmDeliveryPath: 'text', + }), + true, + ); + }); + }); + + describe('explicit legacy tool delivery path', () => { + test('persists llmDeliveryPath none for explicit file_search uploads', async () => { + const { createFile } = require('~/models'); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 128, + filename: 'upload.bin', + embedded: false, + }); + getStrategyFunctions.mockReturnValue({ handleFileUpload: storageUpload }); + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + defaultLLMDeliveryPath: { + fallback: 'text', + }, + }); + const req = makeReq({ mimetype: 'text/markdown', ocrConfig: null }); + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + ...makeMetadata(), + tool_resource: EToolResources.file_search, + }, + }); + + expect(checkCapability).toHaveBeenCalledWith( + expect.anything(), + AgentCapabilities.file_search, + ); + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + source: FileSources.local, + type: 'text/markdown', + embedded: true, + llmDeliveryPath: 'none', + }), + true, + ); + }); + + test('persists llmDeliveryPath provider for legacy provider uploads without tool_resource', async () => { + const { createFile } = require('~/models'); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 128, + filename: 'upload.bin', + embedded: false, + }); + getStrategyFunctions.mockReturnValue({ handleFileUpload: storageUpload }); + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + legacyFileUploadUX: true, + defaultLLMDeliveryPath: { + fallback: 'none', + }, + }); + const req = makeReq({ mimetype: 'text/markdown', ocrConfig: null }); + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + agent_id: 'agent-abc', + message_file: 'true', + file_id: 'file-uuid-123', + }, + }); + + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + source: FileSources.local, + type: 'text/markdown', + llmDeliveryPath: 'provider', + }), + true, + ); + }); + + test('persists llmDeliveryPath none for explicit execute_code uploads', async () => { + const { createFile } = require('~/models'); + const codeUpload = jest.fn().mockResolvedValue('session-1/file.csv'); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 128, + filename: 'upload.bin', + embedded: false, + }); + getStrategyFunctions.mockImplementation((source) => { + if (source === FileSources.execute_code) { + return { handleFileUpload: codeUpload }; + } + return { handleFileUpload: storageUpload }; + }); + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + defaultLLMDeliveryPath: { + fallback: 'text', + }, + }); + const req = makeReq({ mimetype: 'text/csv', ocrConfig: null }); + req.file.path = __filename; + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + ...makeMetadata(), + tool_resource: EToolResources.execute_code, + }, + }); + + expect(checkCapability).toHaveBeenCalledWith( + expect.anything(), + AgentCapabilities.execute_code, + ); + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + source: FileSources.local, + type: 'text/csv', + metadata: { fileIdentifier: 'session-1/file.csv' }, + llmDeliveryPath: 'none', + }), + true, + ); + }); + + test('persists llmDeliveryPath text for explicit context uploads', async () => { + const { parseText } = require('@librechat/api'); + const { createFile } = require('~/models'); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 128, + filename: 'upload.bin', + embedded: false, + }); + getStrategyFunctions.mockReturnValue({ handleFileUpload: storageUpload }); + mergeFileConfig.mockReturnValue( + makeFileConfig({ textSupportedMimeTypes: ['text/markdown'] }), + ); + parseText.mockResolvedValueOnce({ text: 'markdown text', bytes: 13 }); + const req = makeReq({ mimetype: 'text/markdown', ocrConfig: null }); + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + ...makeMetadata(), + tool_resource: EToolResources.context, + }, + }); + + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'markdown text', + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + source: FileSources.local, + type: 'text/markdown', + llmDeliveryPath: 'text', + }), + true, + ); + }); + + test('normalizes explicit ocr uploads to context text delivery', async () => { + const { parseText } = require('@librechat/api'); + const { createFile, addAgentResourceFile } = require('~/models'); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 128, + filename: 'upload.bin', + embedded: false, + }); + getStrategyFunctions.mockReturnValue({ handleFileUpload: storageUpload }); + mergeFileConfig.mockReturnValue( + makeFileConfig({ textSupportedMimeTypes: ['text/markdown'] }), + ); + parseText.mockResolvedValueOnce({ text: 'markdown text', bytes: 13 }); + const req = makeReq({ mimetype: 'text/markdown', ocrConfig: null }); + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + ...makeMetadata(), + tool_resource: EToolResources.ocr, + }, + }); + + expect(addAgentResourceFile).toHaveBeenCalledWith( + expect.objectContaining({ + file_id: 'file-uuid-123', + agent_id: 'agent-abc', + tool_resource: EToolResources.context, + }), + ); + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'markdown text', + source: FileSources.local, + type: 'text/markdown', + llmDeliveryPath: 'text', + }), + true, + ); + }); + }); +}); + +describe('processImageFile', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockRes.status.mockReturnThis(); + mockRes.json.mockReturnValue({}); + mergeFileConfig.mockReturnValue(makeFileConfig()); + }); + + test('persists resolved llmDeliveryPath for image uploads', async () => { + const { createFile } = require('~/models'); + const handleImageUpload = jest.fn().mockResolvedValue({ + filepath: '/images/user-123/image.webp', + bytes: 256, + width: 100, + height: 80, + }); + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + defaultLLMDeliveryPath: { + overrides: { 'image/*': 'none' }, + }, + }); + getStrategyFunctions.mockReturnValue({ handleImageUpload }); + const req = makeReq({ mimetype: 'image/png', ocrConfig: null }); + + await processImageFile({ + req, + res: mockRes, + metadata: { + file_id: 'image-file-id', + temp_file_id: 'temp-image-file-id', + endpoint: EModelEndpoint.agents, + }, + }); + + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + file_id: 'image-file-id', + temp_file_id: 'temp-image-file-id', + filepath: '/images/user-123/image.webp', + source: FileSources.local, + type: 'image/webp', + llmDeliveryPath: 'none', + }), + true, + ); + }); + + test('persists provider llmDeliveryPath for legacy image provider uploads', async () => { + const { createFile } = require('~/models'); + const handleImageUpload = jest.fn().mockResolvedValue({ + filepath: '/images/user-123/image.webp', + bytes: 256, + width: 100, + height: 80, + }); + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + legacyFileUploadUX: true, + defaultLLMDeliveryPath: { + overrides: { 'image/*': 'none' }, + }, + }); + getStrategyFunctions.mockReturnValue({ handleImageUpload }); + const req = makeReq({ mimetype: 'image/png', ocrConfig: null }); + + await processImageFile({ + req, + res: mockRes, + metadata: { + file_id: 'image-file-id', + temp_file_id: 'temp-image-file-id', + endpoint: EModelEndpoint.agents, + }, + }); + + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + file_id: 'image-file-id', + temp_file_id: 'temp-image-file-id', + filepath: '/images/user-123/image.webp', + source: FileSources.local, + type: 'image/webp', + llmDeliveryPath: 'provider', + }), + true, + ); + }); }); describe('processFileURL', () => { diff --git a/api/server/services/Files/provision.js b/api/server/services/Files/provision.js index 1db7afd341..61cde00266 100644 --- a/api/server/services/Files/provision.js +++ b/api/server/services/Files/provision.js @@ -13,6 +13,11 @@ const { FileSources } = require('librechat-data-provider'); const { loadAuthValues } = require('~/server/services/Tools/credentials'); const { getStrategyFunctions } = require('./strategies'); +// TODO: check and potentially fix — concurrent temp file collision (deterministic path based on file_id) +// TODO: check and potentially fix — query params not forwarded in checkSessionsAlive batch liveness check +// TODO: check and potentially fix — direct mutation of shared file objects in provisionFiles callback +// TODO: check and potentially fix — this file should be TypeScript in packages/api per CLAUDE.md rules + const axios = createAxiosInstance(); /** diff --git a/client/src/components/Chat/Input/Files/AttachFileMenu.tsx b/client/src/components/Chat/Input/Files/AttachFileMenu.tsx index 551008d75c..60912ae24f 100644 --- a/client/src/components/Chat/Input/Files/AttachFileMenu.tsx +++ b/client/src/components/Chat/Input/Files/AttachFileMenu.tsx @@ -135,9 +135,7 @@ const AttachFileMenu = ({ ephemeralAgent, ); - const isUnifiedMode = - endpointFileConfig?.defaultFileInteraction != null && - endpointFileConfig.defaultFileInteraction !== 'legacy'; + const isUnifiedMode = endpointFileConfig?.legacyFileUploadUX !== true; const handleUploadClick = useCallback( (fileType?: FileUploadType) => { diff --git a/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx b/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx index 4393f81918..b4481817d0 100644 --- a/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx +++ b/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx @@ -141,6 +141,7 @@ function renderMenu(props: Record = {}) { setFiles={() => {}} setFilesLoading={() => {}} conversation={null} + endpointFileConfig={{ legacyFileUploadUX: true }} {...props} /> @@ -273,6 +274,15 @@ describe('AttachFileMenu', () => { expect(screen.getByTestId('dropdown-popup')).toHaveAttribute('data-modal', 'false'); expect(screen.getByTestId('dropdown-popup')).toHaveAttribute('data-portal', 'true'); }); + + it('renders the unified upload button when legacyFileUploadUX is not true', () => { + setupMocks(); + renderMenu({ endpointFileConfig: { legacyFileUploadUX: false } }); + expect(screen.getByRole('button', { name: /attach files/i })).toBeInTheDocument(); + expect( + screen.queryByRole('button', { name: /attach file options/i }), + ).not.toBeInTheDocument(); + }); }); describe('Agent Capabilities', () => { diff --git a/client/src/utils/files.ts b/client/src/utils/files.ts index c81ebf0960..78f9febcdb 100644 --- a/client/src/utils/files.ts +++ b/client/src/utils/files.ts @@ -357,10 +357,7 @@ export const validateFiles = ({ } let mimeTypesToCheck = supportedMimeTypes; - const isUnifiedMode = - !toolResource && - endpointFileConfig.defaultFileInteraction != null && - endpointFileConfig.defaultFileInteraction !== 'legacy'; + const isUnifiedMode = !toolResource && endpointFileConfig?.legacyFileUploadUX !== true; if (toolResource === EToolResources.context || isUnifiedMode) { mimeTypesToCheck = [ ...(supportedMimeTypes || []), diff --git a/packages/api/src/agents/resources.test.ts b/packages/api/src/agents/resources.test.ts index 8daaeb5645..41fec86742 100644 --- a/packages/api/src/agents/resources.test.ts +++ b/packages/api/src/agents/resources.test.ts @@ -1691,4 +1691,114 @@ describe('primeResources', () => { expect(result.tool_resources?.[EToolResources.image_edit]).toBeUndefined(); }); }); + + describe('llmDeliveryPath handling', () => { + it('should keep files with llmDeliveryPath "none" in attachments', async () => { + const providerFile: TFile = { + user: 'user1', + file_id: 'provider-file', + filename: 'image.png', + filepath: '/path/image.png', + type: 'image/png', + bytes: 1000, + object: 'file' as const, + usage: 0, + embedded: false, + source: 'local', + llmDeliveryPath: 'provider', + width: 100, + height: 100, + }; + const noneFile: TFile = { + user: 'user1', + file_id: 'none-file', + filename: 'audio.mp3', + filepath: '/path/audio.mp3', + type: 'audio/mpeg', + bytes: 5000, + object: 'file' as const, + usage: 0, + embedded: false, + source: 'local', + llmDeliveryPath: 'none', + }; + + const result = await primeResources({ + req: mockReq, + appConfig: mockAppConfig, + getFiles: mockGetFiles, + filterFiles: mockFilterFiles, + tool_resources: {}, + attachments: Promise.resolve([providerFile, noneFile]), + requestFileSet, + agentId: 'agent1', + }); + + const attachmentIds = result.attachments.map((f) => f.file_id); + expect(attachmentIds).toContain('provider-file'); + expect(attachmentIds).toContain('none-file'); + }); + + it('should include llmDeliveryPath "none" files in lazy provisioning state', async () => { + const noneFile: TFile = { + user: 'user1', + file_id: 'none-file', + filename: 'data.csv', + filepath: '/path/data.csv', + type: 'text/csv', + bytes: 5000, + object: 'file' as const, + usage: 0, + embedded: false, + source: 'local', + llmDeliveryPath: 'none', + }; + + const result = await primeResources({ + req: mockReq, + appConfig: mockAppConfig, + getFiles: mockGetFiles, + filterFiles: mockFilterFiles, + tool_resources: {}, + attachments: Promise.resolve([noneFile]), + requestFileSet, + agentId: 'agent1', + enabledToolResources: new Set([EToolResources.execute_code, EToolResources.file_search]), + loadCodeApiKey: jest.fn().mockResolvedValue('code-key'), + }); + + expect(result.attachments.map((f) => f.file_id)).toContain('none-file'); + expect(result.provisionState?.codeEnvFiles.map((f) => f.file_id)).toContain('none-file'); + expect(result.provisionState?.vectorDBFiles.map((f) => f.file_id)).toContain('none-file'); + }); + + it('should include files with undefined llmDeliveryPath in attachments (legacy files)', async () => { + const legacyFile: TFile = { + user: 'user1', + file_id: 'legacy-file', + filename: 'doc.pdf', + filepath: '/path/doc.pdf', + type: 'application/pdf', + bytes: 2000, + object: 'file' as const, + usage: 0, + embedded: false, + source: 'local', + }; + + const result = await primeResources({ + req: mockReq, + appConfig: mockAppConfig, + getFiles: mockGetFiles, + filterFiles: mockFilterFiles, + tool_resources: {}, + attachments: Promise.resolve([legacyFile]), + requestFileSet, + agentId: 'agent1', + }); + + const attachmentIds = result.attachments.map((f) => f.file_id); + expect(attachmentIds).toContain('legacy-file'); + }); + }); }); diff --git a/packages/api/src/agents/resources.ts b/packages/api/src/agents/resources.ts index 8ccaa7592e..1e700c7960 100644 --- a/packages/api/src/agents/resources.ts +++ b/packages/api/src/agents/resources.ts @@ -367,21 +367,18 @@ export const primeResources = async ({ } if (contextFileIds.has(file.file_id)) { - // Clear from attachmentFileIds if it was pre-added attachmentFileIds.delete(file.file_id); - // Add to attachments - attachments.push(file); - agentContextAttachments.push(file); - attachmentFileIds.add(file.file_id); - - // Categorize for tool resources categorizeFileForToolResources({ file, tool_resources, requestFileSet, processedResourceFiles, }); + + attachments.push(file); + agentContextAttachments.push(file); + attachmentFileIds.add(file.file_id); } if (imageEditFileIdSet.has(file.file_id)) { diff --git a/packages/api/src/files/context.test.ts b/packages/api/src/files/context.test.ts new file mode 100644 index 0000000000..efc399f47e --- /dev/null +++ b/packages/api/src/files/context.test.ts @@ -0,0 +1,64 @@ +import { FileSources } from 'librechat-data-provider'; + +import type { IMongoFile } from '@librechat/data-schemas'; +import type { ServerRequest } from '~/types'; + +import { extractFileContext } from './context'; + +const makeReq = () => + ({ + body: { fileTokenLimit: 1000 }, + config: { fileConfig: {} }, + }) as ServerRequest; + +const countTokens = (text: string) => text.length; + +describe('extractFileContext', () => { + it('should skip files with llmDeliveryPath "none"', async () => { + const file = { + filename: 'hidden.txt', + source: FileSources.text, + text: 'do not include this', + llmDeliveryPath: 'none', + } as IMongoFile; + + await expect( + extractFileContext({ attachments: [file], req: makeReq(), tokenCountFn: countTokens }), + ).resolves.toBeUndefined(); + }); + + it('should include legacy text-source files with undefined llmDeliveryPath', async () => { + const file = { + filename: 'legacy.txt', + source: FileSources.text, + text: 'legacy text', + } as IMongoFile; + + const result = await extractFileContext({ + attachments: [file], + req: makeReq(), + tokenCountFn: countTokens, + }); + + expect(result).toContain('# "legacy.txt"'); + expect(result).toContain('legacy text'); + }); + + it('should include standard-storage files with text and llmDeliveryPath "text"', async () => { + const file = { + filename: 'stored.txt', + source: FileSources.local, + text: 'stored extracted text', + llmDeliveryPath: 'text', + } as IMongoFile; + + const result = await extractFileContext({ + attachments: [file], + req: makeReq(), + tokenCountFn: countTokens, + }); + + expect(result).toContain('# "stored.txt"'); + expect(result).toContain('stored extracted text'); + }); +}); diff --git a/packages/api/src/files/context.ts b/packages/api/src/files/context.ts index b16b498b35..735bb63ab1 100644 --- a/packages/api/src/files/context.ts +++ b/packages/api/src/files/context.ts @@ -64,7 +64,12 @@ export async function extractFileContext({ for (const file of attachments) { const source = file.source ?? FileSources.local; - if (source === FileSources.text && file.text) { + if (file.llmDeliveryPath === 'none') { + continue; + } + + const hasTextDelivery = file.llmDeliveryPath === 'text' || source === FileSources.text; + if (hasTextDelivery && file.text) { const { text: limitedText, wasTruncated } = await processTextWithTokenLimit({ text: file.text, tokenLimit: fileTokenLimit, diff --git a/packages/api/src/files/encode/processAttachments.spec.ts b/packages/api/src/files/encode/processAttachments.spec.ts index f9af6fd1e7..01855dbf67 100644 --- a/packages/api/src/files/encode/processAttachments.spec.ts +++ b/packages/api/src/files/encode/processAttachments.spec.ts @@ -17,6 +17,7 @@ function categorizeFile( source?: string; embedded?: boolean; metadata?: { fileIdentifier?: string; codeEnvRef?: unknown }; + llmDeliveryPath?: 'provider' | 'text' | 'none'; }, isBedrock: boolean, mergedFileConfig: FileConfig | undefined, @@ -26,6 +27,9 @@ function categorizeFile( if (source === FileSources.text) { return 'skipped'; } + if (file.llmDeliveryPath === 'text' || file.llmDeliveryPath === 'none') { + return 'skipped'; + } if ( file.embedded === true || file.metadata?.codeEnvRef != null || @@ -154,6 +158,28 @@ describe('processAttachments — supportedMimeTypes routing logic', () => { expect(result).toBe('skipped'); }); + it('should skip text-delivery markdown files regardless of config', () => { + const { merged, epConfig } = resolveConfig(['.*']); + const result = categorizeFile( + { type: 'text/markdown', llmDeliveryPath: 'text' }, + false, + merged, + epConfig, + ); + expect(result).toBe('skipped'); + }); + + it('should skip none-delivery markdown files regardless of config', () => { + const { merged, epConfig } = resolveConfig(['.*']); + const result = categorizeFile( + { type: 'text/markdown', llmDeliveryPath: 'none' }, + false, + merged, + epConfig, + ); + expect(result).toBe('skipped'); + }); + it('should skip embedded files regardless of config', () => { const { merged, epConfig } = resolveConfig(['.*']); const result = categorizeFile({ type: 'text/csv', embedded: true }, false, merged, epConfig); diff --git a/packages/data-provider/src/file-config.spec.ts b/packages/data-provider/src/file-config.spec.ts index 3bd44a6a1a..52bf09c6b3 100644 --- a/packages/data-provider/src/file-config.spec.ts +++ b/packages/data-provider/src/file-config.spec.ts @@ -1,4 +1,4 @@ -import type { MimeUploadCapability } from './file-config'; +import type { MimeUploadCapability, TDefaultLLMDeliveryPathConfig } from './file-config'; import type { FileConfig } from './types/files'; import { fileConfig as baseFileConfig, @@ -1736,3 +1736,90 @@ describe('fileConfigSchema clientImageResize', () => { expect(result.success).toBe(false); }); }); + +describe('defaultLLMDeliveryPath config merging', () => { + it('should include defaultLLMDeliveryPath and legacyFileUploadUX in merged endpoint config', () => { + const merged = mergeFileConfig({ + endpoints: { + [EModelEndpoint.agents]: { + defaultLLMDeliveryPath: { + fallback: 'none', + overrides: { 'image/*': 'text' }, + }, + legacyFileUploadUX: true, + }, + }, + }); + const endpointConfig = getEndpointFileConfig({ + fileConfig: merged, + endpoint: EModelEndpoint.agents, + }); + expect(endpointConfig.defaultLLMDeliveryPath).toEqual({ + fallback: 'none', + overrides: { 'image/*': 'text' }, + }); + expect(endpointConfig.legacyFileUploadUX).toBe(true); + }); + + it('should merge global defaultLLMDeliveryPath into mergedConfig', () => { + const merged = mergeFileConfig({ + defaultLLMDeliveryPath: { + fallback: 'provider', + overrides: { 'audio/*': 'none' }, + }, + legacyFileUploadUX: true, + }); + expect(merged.defaultLLMDeliveryPath).toEqual({ + fallback: 'provider', + overrides: { 'audio/*': 'none' }, + }); + expect(merged.legacyFileUploadUX).toBe(true); + }); + + it('should inherit global legacyFileUploadUX into endpoint config', () => { + const merged = mergeFileConfig({ + legacyFileUploadUX: true, + }); + const endpointConfig = getEndpointFileConfig({ + fileConfig: merged, + endpoint: EModelEndpoint.openAI, + }); + expect(endpointConfig.legacyFileUploadUX).toBe(true); + }); + + it('should allow endpoint legacyFileUploadUX to override global legacyFileUploadUX', () => { + const merged = mergeFileConfig({ + legacyFileUploadUX: true, + endpoints: { + [EModelEndpoint.openAI]: { + legacyFileUploadUX: false, + }, + }, + }); + const endpointConfig = getEndpointFileConfig({ + fileConfig: merged, + endpoint: EModelEndpoint.openAI, + }); + expect(endpointConfig.legacyFileUploadUX).toBe(false); + }); + + it('should pass through endpoint defaultLLMDeliveryPath in mergeWithDefault', () => { + const merged = mergeFileConfig({ + endpoints: { + [EModelEndpoint.openAI]: { + defaultLLMDeliveryPath: { overrides: { 'application/pdf': 'text' } }, + }, + }, + }); + const endpointConfig = getEndpointFileConfig({ + fileConfig: merged, + endpoint: EModelEndpoint.openAI, + }); + expect(endpointConfig.defaultLLMDeliveryPath?.overrides?.['application/pdf']).toBe('text'); + }); + + it('should default legacyFileUploadUX to undefined when not set', () => { + const merged = mergeFileConfig(undefined); + expect(merged.legacyFileUploadUX).toBeUndefined(); + }); +}); diff --git a/packages/data-provider/src/file-config.ts b/packages/data-provider/src/file-config.ts index 392a8e4e85..f983aaf4cc 100644 --- a/packages/data-provider/src/file-config.ts +++ b/packages/data-provider/src/file-config.ts @@ -507,8 +507,14 @@ export const fileConfig = { const supportedMimeTypesSchema = z.array(z.string()).optional(); -export const FileInteractionMode = z.enum(['text', 'provider', 'deferred', 'legacy']); -export type TFileInteractionMode = z.infer; +export const DefaultLLMDeliveryPath = z.enum(['provider', 'text', 'none']); +export type TDefaultLLMDeliveryPath = z.infer; + +export const defaultLLMDeliveryPathSchema = z.object({ + fallback: DefaultLLMDeliveryPath.optional(), + overrides: z.record(DefaultLLMDeliveryPath).optional(), +}); +export type TDefaultLLMDeliveryPathConfig = z.infer; export const endpointFileConfigSchema = z.object({ disabled: z.boolean().optional(), @@ -516,7 +522,8 @@ export const endpointFileConfigSchema = z.object({ fileSizeLimit: z.number().min(0).optional(), totalSizeLimit: z.number().min(0).optional(), supportedMimeTypes: supportedMimeTypesSchema.optional(), - defaultFileInteraction: FileInteractionMode.optional(), + defaultLLMDeliveryPath: defaultLLMDeliveryPathSchema.optional(), + legacyFileUploadUX: z.boolean().optional(), }); const skillFileConfigSchema = z.object({ @@ -553,7 +560,8 @@ export const fileConfigSchema = z.object({ supportedMimeTypes: supportedMimeTypesSchema.optional(), }) .optional(), - defaultFileInteraction: FileInteractionMode.optional(), + defaultLLMDeliveryPath: defaultLLMDeliveryPathSchema.optional(), + legacyFileUploadUX: z.boolean().optional(), }); export type TFileConfig = z.infer; @@ -876,8 +884,9 @@ function mergeWithDefault( fileSizeLimit: endpointConfig.fileSizeLimit ?? defaultConfig.fileSizeLimit, totalSizeLimit: endpointConfig.totalSizeLimit ?? defaultConfig.totalSizeLimit, supportedMimeTypes: endpointConfig.supportedMimeTypes ?? defaultMimeTypes, - defaultFileInteraction: - endpointConfig.defaultFileInteraction ?? defaultConfig.defaultFileInteraction, + defaultLLMDeliveryPath: + endpointConfig.defaultLLMDeliveryPath ?? defaultConfig.defaultLLMDeliveryPath, + legacyFileUploadUX: endpointConfig.legacyFileUploadUX ?? defaultConfig.legacyFileUploadUX, }; } @@ -893,11 +902,17 @@ export function getEndpointFileConfig(params: { } /** Compute an effective default by merging user-configured default over the base default */ - const baseDefaultConfig = fileConfig.endpoints.default; + const baseDefaultConfig: EndpointFileConfig = fileConfig.endpoints.default; + const globalDefaultConfig: EndpointFileConfig = { + ...baseDefaultConfig, + defaultLLMDeliveryPath: + mergedFileConfig.defaultLLMDeliveryPath ?? baseDefaultConfig.defaultLLMDeliveryPath, + legacyFileUploadUX: mergedFileConfig.legacyFileUploadUX ?? baseDefaultConfig.legacyFileUploadUX, + }; const userDefaultConfig = mergedFileConfig.endpoints.default; const defaultConfig = userDefaultConfig - ? mergeWithDefault(userDefaultConfig, baseDefaultConfig, 'default') - : baseDefaultConfig; + ? mergeWithDefault(userDefaultConfig, globalDefaultConfig, 'default') + : globalDefaultConfig; const normalizedEndpoint = normalizeEndpointName(endpoint ?? ''); const standardEndpoints = new Set([ @@ -1009,8 +1024,12 @@ export function mergeFileConfig(dynamic: z.infer | unde return mergedConfig; } - if (dynamic.defaultFileInteraction !== undefined) { - mergedConfig.defaultFileInteraction = dynamic.defaultFileInteraction; + if (dynamic.defaultLLMDeliveryPath !== undefined) { + mergedConfig.defaultLLMDeliveryPath = dynamic.defaultLLMDeliveryPath; + } + + if (dynamic.legacyFileUploadUX !== undefined) { + mergedConfig.legacyFileUploadUX = dynamic.legacyFileUploadUX; } if (dynamic.serverFileSizeLimit !== undefined) { @@ -1113,8 +1132,12 @@ export function mergeFileConfig(dynamic: z.infer | unde ); } - if (dynamicEndpoint.defaultFileInteraction !== undefined) { - mergedEndpoint.defaultFileInteraction = dynamicEndpoint.defaultFileInteraction; + if (dynamicEndpoint.defaultLLMDeliveryPath !== undefined) { + mergedEndpoint.defaultLLMDeliveryPath = dynamicEndpoint.defaultLLMDeliveryPath; + } + + if (dynamicEndpoint.legacyFileUploadUX !== undefined) { + mergedEndpoint.legacyFileUploadUX = dynamicEndpoint.legacyFileUploadUX; } } diff --git a/packages/data-provider/src/index.ts b/packages/data-provider/src/index.ts index 42c5336c6d..f2c70e41cf 100644 --- a/packages/data-provider/src/index.ts +++ b/packages/data-provider/src/index.ts @@ -5,6 +5,7 @@ export * from './balance'; export * from './config'; export * from './filters'; export * from './file-config'; +export * from './resolve-llm-delivery-path'; /* messages */ export * from './messages'; /* run steps */ diff --git a/packages/data-provider/src/resolve-llm-delivery-path.spec.ts b/packages/data-provider/src/resolve-llm-delivery-path.spec.ts new file mode 100644 index 0000000000..c69c676472 --- /dev/null +++ b/packages/data-provider/src/resolve-llm-delivery-path.spec.ts @@ -0,0 +1,103 @@ +import { resolveDefaultLLMDeliveryPath, SYSTEM_LLM_DELIVERY_DEFAULTS } from './resolve-llm-delivery-path'; +import type { TDefaultLLMDeliveryPathConfig } from './file-config'; + +describe('resolveDefaultLLMDeliveryPath', () => { + it('should return system default for images when no config provided', () => { + expect(resolveDefaultLLMDeliveryPath('image/png')).toBe('provider'); + }); + + it('should return system default for PDFs when no config provided', () => { + expect(resolveDefaultLLMDeliveryPath('application/pdf')).toBe('provider'); + }); + + it('should return system fallback for unknown mime types', () => { + expect(resolveDefaultLLMDeliveryPath('text/plain')).toBe('text'); + }); + + it('should match exact mime type before wildcard', () => { + const config: TDefaultLLMDeliveryPathConfig = { + overrides: { 'image/png': 'text', 'image/*': 'provider' }, + }; + expect(resolveDefaultLLMDeliveryPath('image/png', config)).toBe('text'); + }); + + it('should match wildcard when no exact match', () => { + const config: TDefaultLLMDeliveryPathConfig = { + overrides: { 'image/*': 'none' }, + }; + expect(resolveDefaultLLMDeliveryPath('image/jpeg', config)).toBe('none'); + }); + + it('should use config fallback when no override matches', () => { + const config: TDefaultLLMDeliveryPathConfig = { + fallback: 'none', + overrides: { 'image/*': 'provider' }, + }; + expect(resolveDefaultLLMDeliveryPath('text/plain', config)).toBe('none'); + }); + + it('should resolve endpoint config before global config', () => { + const endpointConfig: TDefaultLLMDeliveryPathConfig = { + overrides: { 'image/*': 'text' }, + }; + const globalConfig: TDefaultLLMDeliveryPathConfig = { + overrides: { 'image/*': 'provider' }, + }; + expect(resolveDefaultLLMDeliveryPath('image/png', endpointConfig, globalConfig)).toBe('text'); + }); + + it('should fall through to global config when endpoint has no match', () => { + const endpointConfig: TDefaultLLMDeliveryPathConfig = { + overrides: { 'audio/*': 'none' }, + }; + const globalConfig: TDefaultLLMDeliveryPathConfig = { + overrides: { 'image/*': 'text' }, + }; + expect(resolveDefaultLLMDeliveryPath('image/png', endpointConfig, globalConfig)).toBe('text'); + }); + + it('should use endpoint fallback before global overrides', () => { + const endpointConfig: TDefaultLLMDeliveryPathConfig = { + fallback: 'none', + }; + const globalConfig: TDefaultLLMDeliveryPathConfig = { + overrides: { 'text/*': 'provider' }, + }; + expect(resolveDefaultLLMDeliveryPath('text/plain', endpointConfig, globalConfig)).toBe('none'); + }); + + it('should fall through entire chain to system defaults', () => { + const endpointConfig: TDefaultLLMDeliveryPathConfig = {}; + const globalConfig: TDefaultLLMDeliveryPathConfig = {}; + expect(resolveDefaultLLMDeliveryPath('image/png', endpointConfig, globalConfig)).toBe('provider'); + expect(resolveDefaultLLMDeliveryPath('application/pdf', endpointConfig, globalConfig)).toBe('provider'); + expect(resolveDefaultLLMDeliveryPath('text/csv', endpointConfig, globalConfig)).toBe('text'); + }); + + it('should resolve none destination correctly', () => { + const config: TDefaultLLMDeliveryPathConfig = { + overrides: { 'audio/*': 'none' }, + }; + expect(resolveDefaultLLMDeliveryPath('audio/mpeg', config)).toBe('none'); + }); + + it('should prefer exact match over wildcard in the same config', () => { + const config: TDefaultLLMDeliveryPathConfig = { + overrides: { 'image/*': 'provider', 'image/svg+xml': 'text' }, + }; + expect(resolveDefaultLLMDeliveryPath('image/svg+xml', config)).toBe('text'); + expect(resolveDefaultLLMDeliveryPath('image/png', config)).toBe('provider'); + }); + + it('should handle undefined configs gracefully', () => { + expect(resolveDefaultLLMDeliveryPath('text/plain', undefined, undefined)).toBe('text'); + }); + + it('should export SYSTEM_LLM_DELIVERY_DEFAULTS with correct shape', () => { + expect(SYSTEM_LLM_DELIVERY_DEFAULTS.fallback).toBe('text'); + expect(SYSTEM_LLM_DELIVERY_DEFAULTS.overrides).toEqual({ + 'image/*': 'provider', + 'application/pdf': 'provider', + }); + }); +}); diff --git a/packages/data-provider/src/resolve-llm-delivery-path.ts b/packages/data-provider/src/resolve-llm-delivery-path.ts new file mode 100644 index 0000000000..5d6e9d47af --- /dev/null +++ b/packages/data-provider/src/resolve-llm-delivery-path.ts @@ -0,0 +1,56 @@ +import type { TDefaultLLMDeliveryPath, TDefaultLLMDeliveryPathConfig } from './file-config'; + +export const SYSTEM_LLM_DELIVERY_DEFAULTS: Required = { + fallback: 'text', + overrides: { + 'image/*': 'provider', + 'application/pdf': 'provider', + }, +}; + +/** + * Resolves the default file path destination for a given mime type. + * Resolution chain: endpoint overrides -> endpoint fallback -> global overrides -> global fallback -> system defaults. + */ +export function resolveDefaultLLMDeliveryPath( + mimeType: string, + endpointConfig?: TDefaultLLMDeliveryPathConfig, + globalConfig?: TDefaultLLMDeliveryPathConfig, +): TDefaultLLMDeliveryPath { + const wildcard = mimeType.split('/')[0] + '/*'; + + if (endpointConfig?.overrides) { + if (endpointConfig.overrides[mimeType]) { + return endpointConfig.overrides[mimeType] as TDefaultLLMDeliveryPath; + } + if (endpointConfig.overrides[wildcard]) { + return endpointConfig.overrides[wildcard] as TDefaultLLMDeliveryPath; + } + } + + if (endpointConfig?.fallback) { + return endpointConfig.fallback; + } + + if (globalConfig?.overrides) { + if (globalConfig.overrides[mimeType]) { + return globalConfig.overrides[mimeType] as TDefaultLLMDeliveryPath; + } + if (globalConfig.overrides[wildcard]) { + return globalConfig.overrides[wildcard] as TDefaultLLMDeliveryPath; + } + } + + if (globalConfig?.fallback) { + return globalConfig.fallback; + } + + if (SYSTEM_LLM_DELIVERY_DEFAULTS.overrides[mimeType]) { + return SYSTEM_LLM_DELIVERY_DEFAULTS.overrides[mimeType] as TDefaultLLMDeliveryPath; + } + if (SYSTEM_LLM_DELIVERY_DEFAULTS.overrides[wildcard]) { + return SYSTEM_LLM_DELIVERY_DEFAULTS.overrides[wildcard] as TDefaultLLMDeliveryPath; + } + + return SYSTEM_LLM_DELIVERY_DEFAULTS.fallback; +} diff --git a/packages/data-provider/src/types/files.ts b/packages/data-provider/src/types/files.ts index fe9a4b1a35..5268042d91 100644 --- a/packages/data-provider/src/types/files.ts +++ b/packages/data-provider/src/types/files.ts @@ -1,3 +1,4 @@ +import type { TDefaultLLMDeliveryPathConfig } from '../file-config'; import type { CodeEnvRef, CodeEnvRefMap } from '../codeEnvRef'; import { EToolResources } from './assistants'; @@ -48,7 +49,8 @@ export type EndpointFileConfig = { fileSizeLimit?: number; totalSizeLimit?: number; supportedMimeTypes?: RegexLike[]; - defaultFileInteraction?: 'text' | 'provider' | 'deferred' | 'legacy'; + defaultLLMDeliveryPath?: TDefaultLLMDeliveryPathConfig; + legacyFileUploadUX?: boolean; }; export type FileConfig = { @@ -79,7 +81,8 @@ export type FileConfig = { supportedMimeTypes?: RegexLike[]; }; checkType?: (fileType: string, supportedTypes: RegexLike[]) => boolean; - defaultFileInteraction?: 'text' | 'provider' | 'deferred' | 'legacy'; + defaultLLMDeliveryPath?: TDefaultLLMDeliveryPathConfig; + legacyFileUploadUX?: boolean; }; export type FileConfigInput = { @@ -107,7 +110,8 @@ export type FileConfigInput = { supportedMimeTypes?: string[]; }; checkType?: (fileType: string, supportedTypes: RegexLike[]) => boolean; - defaultFileInteraction?: 'text' | 'provider' | 'deferred' | 'legacy'; + defaultLLMDeliveryPath?: TDefaultLLMDeliveryPathConfig; + legacyFileUploadUX?: boolean; }; export type TFile = { @@ -171,6 +175,7 @@ export type TFile = { /** Dispatch-order stamp for the current source artifact generation. */ sourceDispatchedAt?: number; }; + llmDeliveryPath?: 'provider' | 'text' | 'none'; createdAt?: string | Date; updatedAt?: string | Date; }; diff --git a/packages/data-schemas/src/schema/file.ts b/packages/data-schemas/src/schema/file.ts index 208ad67bdf..db1a6600d3 100644 --- a/packages/data-schemas/src/schema/file.ts +++ b/packages/data-schemas/src/schema/file.ts @@ -136,6 +136,10 @@ const file: Schema = new Schema( default: undefined, }, }, + llmDeliveryPath: { + type: String, + enum: ['provider', 'text', 'none'], + }, expiresAt: { /* Short-lived upload TTL managed by MongoDB. This is separate from * retention-scoped `expiredAt`, which is swept by application code diff --git a/packages/data-schemas/src/types/file.ts b/packages/data-schemas/src/types/file.ts index 8038512144..52617ac323 100644 --- a/packages/data-schemas/src/types/file.ts +++ b/packages/data-schemas/src/types/file.ts @@ -74,6 +74,7 @@ export interface IMongoFile extends Omit { /** Dispatch-order stamp for the current source artifact generation. */ sourceDispatchedAt?: number; }; + llmDeliveryPath?: string; expiresAt?: Date; expiredAt?: Date | null; createdAt?: Date;