diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index 72b716a9b4..05d1b7e714 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -38,6 +38,9 @@ const { HITL_MESSAGE_FILTER_FIELDS, getEndpointFileConfig, stripReasoningLabelMetadata, + resolveUploadLLMDeliveryPath, + isSpeechProviderConfigured, + resolveUseResponsesApi, } = require('librechat-data-provider'); const { getStrategyFunctions } = require('~/server/services/Files/strategies'); const { logViolation } = require('~/cache'); @@ -1720,6 +1723,16 @@ class BaseClient { return await this.sendCompletion(payload, opts); } + /** Whether this turn talks to the Responses API, which is what lets Azure carry a + * document natively. A saved agent holds it in its parameters and a plain conversation + * in its model options, and both readers of it have been wrong by consulting one. */ + usesResponsesApi() { + return resolveUseResponsesApi( + this.options.agent?.model_parameters?.useResponsesApi, + this.modelOptions?.useResponsesApi, + ); + } + async addDocuments(message, attachments) { const documentResult = await encodeAndFormatDocuments( this.options.req, @@ -1727,7 +1740,7 @@ class BaseClient { { provider: this.options.agent?.provider ?? this.options.endpoint, endpoint: this.options.agent?.endpoint ?? this.options.endpoint, - useResponsesApi: this.options.agent?.model_parameters?.useResponsesApi, + useResponsesApi: this.usesResponsesApi(), model: this.modelOptions?.model ?? this.model, }, getStrategyFunctions, @@ -1777,8 +1790,9 @@ class BaseClient { * @returns {Promise} */ async addFileContextToMessage(message, attachments) { + const textAttachments = this.getTextContextAttachments(attachments); const fileContext = await extractFileContext({ - attachments, + attachments: textAttachments, req: this.options?.req, tokenCountFn: (text) => countTokens(text), }); @@ -1788,6 +1802,44 @@ class BaseClient { } } + getTextContextAttachments(attachments) { + return attachments.filter((file) => { + const deliveryPath = this.getAttachmentDeliveryPath(file); + /* Records predating delivery paths keep legacy extraction. Current routing is + * authoritative for inferred uploads, so native provider bytes are not also + * injected as extracted text after a provider handoff. */ + return deliveryPath == null || deliveryPath === 'text'; + }); + } + + /** Re-resolves an inferred upload route against the provider handling this turn. */ + getAttachmentDeliveryPath(file) { + if (!this._mergedFileConfig) { + this._mergedFileConfig = mergeFileConfig(this.options.req?.config?.fileConfig); + /* Agent file policy is configured under the endpoint it names, not the client + * family initialization may rewrite it to. */ + const agentEndpoint = this.options.agent?.endpoint ?? this.options.agent?.provider; + this._deliveryEndpoint = agentEndpoint ?? this.options.endpoint; + this._endpointFileConfig = getEndpointFileConfig({ + fileConfig: this._mergedFileConfig, + endpoint: this._deliveryEndpoint, + endpointType: agentEndpoint != null ? undefined : this.options.endpointType, + }); + } + + return file.llmDeliveryPath == null || file.metadata?.destinationChosen === true + ? file.llmDeliveryPath + : resolveUploadLLMDeliveryPath({ + /* Conversion changes the stored type, so use the type routing originally saw. */ + mimeType: file.metadata?.routingMimeType ?? file.type, + endpointConfig: this._endpointFileConfig, + fileConfig: this._mergedFileConfig, + endpoint: this._deliveryEndpoint, + useResponsesApi: this.usesResponsesApi(), + sttConfigured: isSpeechProviderConfigured(this.options.req?.config?.speech?.stt), + }); + } + async processAttachments(message, attachments) { const categorizedAttachments = { images: [], @@ -1797,20 +1849,14 @@ class BaseClient { }; const allFiles = []; - const provider = this.options.agent?.provider ?? this.options.endpoint; const isBedrock = provider === EModelEndpoint.bedrock; - if (!this._mergedFileConfig) { - this._mergedFileConfig = mergeFileConfig(this.options.req?.config?.fileConfig); - const endpoint = this.options.agent?.endpoint ?? this.options.endpoint; - this._endpointFileConfig = getEndpointFileConfig({ - fileConfig: this._mergedFileConfig, - endpoint, - endpointType: this.options.endpointType, - }); - } - + /* The stored path records what upload time inferred from the endpoint it saw, and this + * turn may be running somewhere else: audio stored as `provider` under Google reaches + * an encoder that emits nothing for OpenAI, delivering neither media nor text. An + * explicit chooser decision is the user's and survives, and a record predating the + * field keeps its legacy handling. */ for (const file of attachments) { /** @type {FileSources} */ const source = file.source ?? FileSources.local; @@ -1818,11 +1864,20 @@ class BaseClient { allFiles.push(file); continue; } + const deliveryPath = this.getAttachmentDeliveryPath(file); + if (deliveryPath === 'text' || deliveryPath === 'none') { + allFiles.push(file); + continue; + } + /* An explicit `provider` path is authoritative: lazy provisioning stamps + * `embedded`/`codeEnvRef` on files that are still meant for the model, so the + * legacy tool-provisioning exclusion only applies to records without one. */ if ( - file.embedded === true || - file.metadata?.codeEnvRef != null || - file.metadata?.codeEnvRefs != null || - file.metadata?.fileIdentifier != null + deliveryPath !== 'provider' && + (file.embedded === true || + file.metadata?.codeEnvRef != null || + file.metadata?.codeEnvRefs != null || + file.metadata?.fileIdentifier != null) ) { allFiles.push(file); continue; diff --git a/api/app/clients/specs/BaseClient.test.js b/api/app/clients/specs/BaseClient.test.js index 9caa0a06b0..8f2bbbafee 100644 --- a/api/app/clients/specs/BaseClient.test.js +++ b/api/app/clients/specs/BaseClient.test.js @@ -1,4 +1,4 @@ -const { Constants, ContentTypes } = require('librechat-data-provider'); +const { Constants, ContentTypes, EModelEndpoint } = require('librechat-data-provider'); const BaseClientClass = require('../BaseClient'); const { ContentFilterError } = require('@librechat/api'); const { FakeClient, initializeFakeClient } = require('./FakeClient'); @@ -3106,6 +3106,400 @@ describe('BaseClient', () => { ]); }); }); + + describe('processAttachments llmDeliveryPath handling', () => { + beforeEach(() => { + TestClient.options = { + endpoint: EModelEndpoint.openAI, + }; + TestClient._mergedFileConfig = undefined; + TestClient._endpointFileConfig = undefined; + TestClient.addImageURLs = jest.fn(async (message, files) => { + message.image_urls = ['encoded-image']; + return files; + }); + TestClient.addDocuments = jest.fn(async (message, files) => { + message.documents = [{ type: 'file' }]; + return files; + }); + TestClient.addVideos = jest.fn(async (_message, files) => files); + TestClient.modelOptions = undefined; + TestClient.addAudios = jest.fn(async (_message, files) => files); + }); + + /* The stored path is an upload-time inference, so delivery re-resolves it for the + * endpoint running the turn. A test asserting a route has to configure that route + * rather than rely on the stored value alone. */ + const routeTo = (path, ...mimeTypes) => { + TestClient.options.req = { + config: { + fileConfig: { + endpoints: { + [EModelEndpoint.openAI]: { + defaultLLMDeliveryPath: { + overrides: Object.fromEntries(mimeTypes.map((mime) => [mime, path])), + }, + }, + }, + }, + }, + }; + TestClient._mergedFileConfig = undefined; + TestClient._endpointFileConfig = undefined; + }; + + test('keeps a none image in returned files without adding image URLs', async () => { + routeTo('none', 'image/*'); + const message = {}; + const file = { + user: 'user1', + file_id: 'none-image', + filename: 'image.png', + filepath: '/uploads/image.png', + type: 'image/png', + bytes: 100, + source: 'local', + llmDeliveryPath: 'none', + }; + + const result = await TestClient.processAttachments(message, [file]); + + expect(result).toEqual([file]); + expect(message.image_urls).toBeUndefined(); + expect(TestClient.addImageURLs).not.toHaveBeenCalled(); + }); + + test('does not inject extracted text after the current provider resolves none', () => { + routeTo('none', 'application/pdf'); + const file = { + file_id: 'none-pdf', + filename: 'report.pdf', + type: 'application/pdf', + llmDeliveryPath: 'text', + metadata: { destinationChosen: false }, + }; + + expect(TestClient.getTextContextAttachments([file])).toEqual([]); + }); + + test('does not inject extracted text when the current provider resolves native delivery', () => { + routeTo('provider', 'application/pdf'); + const file = { + file_id: 'provider-pdf', + filename: 'report.pdf', + type: 'application/pdf', + llmDeliveryPath: 'text', + metadata: { destinationChosen: false }, + }; + + expect(TestClient.getTextContextAttachments([file])).toEqual([]); + }); + + test('re-resolves a path stored under a different provider', async () => { + /* Audio uploaded under Google stores `provider`, and the OpenAI encoder emits no + * audio payload for it, so the inference is not this endpoint's to honor. + * + * What this does not do is produce a transcript: the record holds raw media and no + * extracted text, and extraction at delivery is Phase 2 work. So the model receives + * nothing here either way, which the assertions state rather than imply, and the + * change is limited to not downloading and encoding a file to no purpose. */ + routeTo('text', 'audio/*'); + const message = {}; + const file = { + user: 'user1', + file_id: 'foreign-audio', + filename: 'note.mp3', + filepath: '/uploads/note.mp3', + type: 'audio/mpeg', + bytes: 100, + source: 'local', + llmDeliveryPath: 'provider', + }; + + const result = await TestClient.processAttachments(message, [file]); + + expect(result).toEqual([file]); + expect(TestClient.addAudios).not.toHaveBeenCalled(); + expect(message.audios).toBeUndefined(); + expect(file.text).toBeUndefined(); + }); + + test('re-resolves a converted image against the type it was routed on', async () => { + /* Conversion rewrote the stored type, so resolving against that asks about a format + * the administrator never configured a route for and delivers what they excluded. */ + routeTo('none', 'image/png'); + const message = {}; + const file = { + user: 'user1', + file_id: 'converted-image', + filename: 'photo.png', + filepath: '/uploads/photo.webp', + type: 'image/webp', + bytes: 100, + source: 'local', + llmDeliveryPath: 'none', + metadata: { routingMimeType: 'image/png' }, + }; + + const result = await TestClient.processAttachments(message, [file]); + + expect(result).toEqual([file]); + expect(TestClient.addImageURLs).not.toHaveBeenCalled(); + }); + + test('reads the Responses setting from a plain conversation too', async () => { + /* A non-agent Azure chat carries it in model options, and reading only the agent + * parameters re-resolves a natively supported PDF to text, which the record has + * none of, so the model receives nothing. */ + TestClient.options = { + endpoint: EModelEndpoint.azureOpenAI, + req: { config: { fileConfig: undefined } }, + }; + TestClient.modelOptions = { useResponsesApi: true }; + TestClient._mergedFileConfig = undefined; + TestClient._endpointFileConfig = undefined; + const message = {}; + const file = { + user: 'user1', + file_id: 'azure-pdf', + filename: 'doc.pdf', + filepath: '/uploads/doc.pdf', + type: 'application/pdf', + bytes: 100, + source: 'local', + llmDeliveryPath: 'provider', + }; + + await TestClient.processAttachments(message, [file]); + + expect(TestClient.addDocuments).toHaveBeenCalled(); + TestClient.modelOptions = undefined; + }); + + test('resolves a custom endpoint policy by the name the admin configured', async () => { + /* `initializeAgent` rewrites `agent.provider` to the client family a custom endpoint + * runs on, so resolving by it looks up `openAI` and silently loses every override + * written against the endpoint's own name. Upload routed under that name, and + * delivery has to agree or the file is stored and never sent. */ + TestClient.options = { + endpoint: EModelEndpoint.agents, + endpointType: EModelEndpoint.agents, + agent: { provider: EModelEndpoint.openAI, endpoint: 'Mock Provider B' }, + req: { + config: { + fileConfig: { + endpoints: { + 'Mock Provider B': { + defaultLLMDeliveryPath: { overrides: { 'image/*': 'none' } }, + }, + }, + }, + }, + }, + }; + TestClient._mergedFileConfig = undefined; + TestClient._endpointFileConfig = undefined; + const message = {}; + const file = { + user: 'user1', + file_id: 'custom-image', + filename: 'photo.png', + filepath: '/uploads/photo.png', + type: 'image/png', + bytes: 100, + source: 'local', + llmDeliveryPath: 'provider', + }; + + const result = await TestClient.processAttachments(message, [file]); + + expect(result).toEqual([file]); + expect(TestClient.addImageURLs).not.toHaveBeenCalled(); + }); + + test('resolves an agent policy by its own endpoint, not the agents container', async () => { + /* getEndpointFileConfig prefers endpointType, and an agents chat carries `agents`, + * so supplying it answers with the generic entry rather than the agent's. + * `initializeAgent` sets `agent.endpoint` from the agent's provider, so it names a + * configurable entry and never the container. */ + TestClient.options = { + endpoint: EModelEndpoint.agents, + endpointType: EModelEndpoint.agents, + agent: { provider: EModelEndpoint.openAI, endpoint: EModelEndpoint.openAI }, + req: { + config: { + fileConfig: { + endpoints: { + [EModelEndpoint.openAI]: { + defaultLLMDeliveryPath: { overrides: { 'image/*': 'none' } }, + }, + }, + }, + }, + }, + }; + TestClient._mergedFileConfig = undefined; + TestClient._endpointFileConfig = undefined; + const message = {}; + const file = { + user: 'user1', + file_id: 'agent-image', + filename: 'photo.png', + filepath: '/uploads/photo.png', + type: 'image/png', + bytes: 100, + source: 'local', + llmDeliveryPath: 'provider', + }; + + const result = await TestClient.processAttachments(message, [file]); + + expect(result).toEqual([file]); + expect(TestClient.addImageURLs).not.toHaveBeenCalled(); + }); + + test('keeps an explicitly named destination even under a different provider', async () => { + /* The user named this one, through the chooser or by requesting a tool resource, + * and that decision is not this endpoint's to re-derive. */ + routeTo('text', 'audio/*'); + const message = {}; + const file = { + user: 'user1', + file_id: 'chosen-audio', + filename: 'note.mp3', + filepath: '/uploads/note.mp3', + type: 'audio/mpeg', + bytes: 100, + source: 'local', + llmDeliveryPath: 'provider', + metadata: { destinationChosen: true }, + }; + + await TestClient.processAttachments(message, [file]); + + expect(TestClient.addAudios).toHaveBeenCalled(); + }); + + test('keeps a none PDF in returned files without adding documents', async () => { + routeTo('none', 'application/pdf'); + const message = {}; + const file = { + user: 'user1', + file_id: 'none-pdf', + filename: 'document.pdf', + filepath: '/uploads/document.pdf', + type: 'application/pdf', + bytes: 100, + source: 'local', + llmDeliveryPath: 'none', + }; + + const result = await TestClient.processAttachments(message, [file]); + + expect(result).toEqual([file]); + expect(message.documents).toBeUndefined(); + expect(TestClient.addDocuments).not.toHaveBeenCalled(); + }); + + test('keeps a text-delivery markdown file in returned files without adding documents', async () => { + const message = {}; + const file = { + user: 'user1', + file_id: 'text-markdown', + filename: 'notes.md', + filepath: '/uploads/notes.md', + type: 'text/markdown', + bytes: 100, + source: 'local', + text: 'extracted markdown', + llmDeliveryPath: 'text', + }; + + const result = await TestClient.processAttachments(message, [file]); + + expect(result).toEqual([file]); + expect(message.documents).toBeUndefined(); + expect(TestClient.addDocuments).not.toHaveBeenCalled(); + }); + + test('still delivers a provider PDF that lazy provisioning marked embedded', async () => { + const message = {}; + const file = { + user: 'user1', + file_id: 'provisioned-pdf', + filename: 'report.pdf', + filepath: '/uploads/report.pdf', + type: 'application/pdf', + bytes: 100, + source: 'local', + embedded: true, + llmDeliveryPath: 'provider', + }; + + await TestClient.processAttachments(message, [file]); + + expect(TestClient.addDocuments).toHaveBeenCalled(); + expect(message.documents).toEqual([{ type: 'file' }]); + }); + + test('still delivers a provider image that carries a codeEnvRef', async () => { + const message = {}; + const file = { + user: 'user1', + file_id: 'provisioned-image', + filename: 'chart.png', + filepath: '/uploads/chart.png', + type: 'image/png', + bytes: 100, + source: 'local', + llmDeliveryPath: 'provider', + metadata: { codeEnvRef: { kind: 'user', id: 'u1' } }, + }; + + await TestClient.processAttachments(message, [file]); + + expect(TestClient.addImageURLs).toHaveBeenCalled(); + expect(message.image_urls).toEqual(['encoded-image']); + }); + + test('keeps excluding embedded legacy files that have no delivery path', async () => { + const message = {}; + const file = { + user: 'user1', + file_id: 'legacy-embedded', + filename: 'legacy.pdf', + filepath: '/uploads/legacy.pdf', + type: 'application/pdf', + bytes: 100, + source: 'local', + embedded: true, + }; + + const result = await TestClient.processAttachments(message, [file]); + + expect(result).toEqual([file]); + expect(TestClient.addDocuments).not.toHaveBeenCalled(); + }); + + test('routes legacy files without llmDeliveryPath normally', async () => { + const message = {}; + const file = { + user: 'user1', + file_id: 'legacy-pdf', + filename: 'document.pdf', + filepath: '/uploads/document.pdf', + type: 'application/pdf', + bytes: 100, + source: 'local', + }; + + const result = await TestClient.processAttachments(message, [file]); + + expect(result).toEqual([file]); + expect(message.documents).toEqual([{ type: 'file' }]); + expect(TestClient.addDocuments).toHaveBeenCalledWith(message, [file]); + }); + }); }); describe('BaseClient compaction turns', () => { diff --git a/api/server/controllers/agents/__tests__/openai.spec.js b/api/server/controllers/agents/__tests__/openai.spec.js index 42be10e7f4..9986276e20 100644 --- a/api/server/controllers/agents/__tests__/openai.spec.js +++ b/api/server/controllers/agents/__tests__/openai.spec.js @@ -173,6 +173,8 @@ jest.mock('@librechat/agents', () => ({ })); jest.mock('@librechat/api', () => ({ + /* Provisioning moved into this package; the controllers build the callback from it. */ + createProvisionFilesCallback: () => async () => {}, createAgentExecutionContext: (context) => context, /** Grants both by default; the capability set is what these specs vary. */ resolveToolRoleGrants: jest.fn(async () => ({ runCode: true, fileSearch: true })), diff --git a/api/server/controllers/agents/__tests__/responses.unit.spec.js b/api/server/controllers/agents/__tests__/responses.unit.spec.js index 42c56f763e..a088e484bd 100644 --- a/api/server/controllers/agents/__tests__/responses.unit.spec.js +++ b/api/server/controllers/agents/__tests__/responses.unit.spec.js @@ -196,6 +196,8 @@ jest.mock('@librechat/agents', () => ({ })); jest.mock('@librechat/api', () => ({ + /* Provisioning moved into this package; the controllers build the callback from it. */ + createProvisionFilesCallback: () => async () => {}, createAgentExecutionContext: (context) => context, /** Grants both by default; the capability set is what these specs vary. */ resolveToolRoleGrants: jest.fn(async () => ({ runCode: true, fileSearch: true })), diff --git a/api/server/controllers/agents/openai.js b/api/server/controllers/agents/openai.js index 2a6c920fca..3fb6a213da 100644 --- a/api/server/controllers/agents/openai.js +++ b/api/server/controllers/agents/openai.js @@ -92,6 +92,8 @@ const { resolveMemoryAvailability, enrichLoadedToolsWithAgentContext, } = require('~/server/services/Endpoints/agents/skillDeps'); +const { createProvisionFilesCallback } = require('~/server/services/Files/provisionCallback'); +const { checkSessionsAlive, loadCodeApiKey } = require('~/server/services/Files/provision'); const { getModelsConfig } = require('~/server/controllers/ModelController'); const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions'); const { resolveConfigServers } = require('~/server/services/MCP'); @@ -482,6 +484,9 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { updateFilesUsage: db.updateFilesUsage, getUserKeyValues: db.getUserKeyValues, getUserCodeFiles: db.getUserCodeFiles, + getDeferredProvisionFiles: db.getDeferredProvisionFiles, + checkSessionsAlive, + loadCodeApiKey, getToolFilesByIds: db.getToolFilesByIds, getCodeGeneratedFiles: db.getCodeGeneratedFiles, listSkillsByAccess: skillDbMethods.listSkillsByAccess, @@ -793,6 +798,11 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { agent never gains sandbox access even if the admin enabled the capability globally. */ const toolExecuteOptions = { + provisionFiles: createProvisionFilesCallback({ + req, + agentToolContexts, + resolvePrimaryAgentId: () => primaryConfig.id, + }), loadTools: async (toolNames, agentId, _configurable, callerCapabilityProjection) => { const ctx = agentToolContexts.get(agentId) ?? agentToolContexts.get(primaryConfig.id) ?? {}; diff --git a/api/server/controllers/agents/responses.js b/api/server/controllers/agents/responses.js index 7b8ff642f6..fdcd93a372 100644 --- a/api/server/controllers/agents/responses.js +++ b/api/server/controllers/agents/responses.js @@ -107,6 +107,8 @@ const { resolveMemoryAvailability, enrichLoadedToolsWithAgentContext, } = require('~/server/services/Endpoints/agents/skillDeps'); +const { createProvisionFilesCallback } = require('~/server/services/Files/provisionCallback'); +const { checkSessionsAlive, loadCodeApiKey } = require('~/server/services/Files/provision'); const { getModelsConfig } = require('~/server/controllers/ModelController'); const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions'); const { resolveConfigServers, getAccessibleMcpServerNames } = require('~/server/services/MCP'); @@ -730,6 +732,9 @@ const executeResponse = async (envelope, { req, res }) => { updateFilesUsage: db.updateFilesUsage, getUserKeyValues: db.getUserKeyValues, getUserCodeFiles: db.getUserCodeFiles, + getDeferredProvisionFiles: db.getDeferredProvisionFiles, + checkSessionsAlive, + loadCodeApiKey, getToolFilesByIds: db.getToolFilesByIds, getCodeGeneratedFiles: db.getCodeGeneratedFiles, listSkillsByAccess: skillDbMethods.listSkillsByAccess, @@ -1134,6 +1139,11 @@ const executeResponse = async (envelope, { req, res }) => { // Create tool execute options for event-driven tool execution const toolExecuteOptions = { + provisionFiles: createProvisionFilesCallback({ + req, + agentToolContexts, + resolvePrimaryAgentId: () => primaryConfig.id, + }), loadTools: async (toolNames, agentId, _configurable, callerCapabilityProjection) => { const ctx = agentToolContexts.get(agentId) ?? agentToolContexts.get(primaryConfig.id) ?? {}; @@ -1349,6 +1359,11 @@ const executeResponse = async (envelope, { req, res }) => { }); const toolExecuteOptions = { + provisionFiles: createProvisionFilesCallback({ + req, + agentToolContexts, + resolvePrimaryAgentId: () => primaryConfig.id, + }), loadTools: async (toolNames, agentId, _configurable, callerCapabilityProjection) => { const ctx = agentToolContexts.get(agentId) ?? agentToolContexts.get(primaryConfig.id) ?? {}; diff --git a/api/server/index.js b/api/server/index.js index 046004252d..a6be1e68ae 100644 --- a/api/server/index.js +++ b/api/server/index.js @@ -57,6 +57,7 @@ const { createScheduleWriteGate, startCodeEnvironmentLifecycleReconciler, waitForKeyvRedisClient, + warnOnUnreachableDeliveryPaths, } = require('@librechat/api'); const { connectDb, indexSync } = require('~/db'); const { @@ -228,6 +229,7 @@ const startServer = async () => { }); const appConfig = await getAppConfig({ baseOnly: true }); configureAgentEventRuntime(appConfig?.endpoints?.agents?.eventDriven); + warnOnUnreachableDeliveryPaths(appConfig); initializeFileStorage(appConfig); const projectRoot = path.resolve(__dirname, '../..'); // Plugin hooks execute only when the operator opts in via DEPLOYMENT_PLUGIN_HOOKS; diff --git a/api/server/routes/files/files.agents.test.js b/api/server/routes/files/files.agents.test.js index 86ff18025f..4323a28e81 100644 --- a/api/server/routes/files/files.agents.test.js +++ b/api/server/routes/files/files.agents.test.js @@ -428,6 +428,13 @@ describe('File Routes - Agent Files Endpoint', () => { }; it('inspects the canonical sanitized filename used by upload processing', async () => { + await createAgent({ + id: agentCustomId, + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: authorId, + }); const testApp = createAppWithUser( authorId, SystemRoles.USER, @@ -463,6 +470,49 @@ describe('File Routes - Agent Files Endpoint', () => { expect(processAgentFileUpload).not.toHaveBeenCalled(); }); + it('denies an unauthorized caller before validating against the agent', async () => { + /* Size and content validation run under the target agent's provider policy, so + * reaching them first answers an unauthorized caller with that agent's + * configuration instead of a 403. */ + await createAgent({ + id: agentCustomId, + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: authorId, + }); + + const readFile = jest.spyOn(fs, 'readFile'); + const testApp = createAppWithUser( + otherUserId, + SystemRoles.USER, + { + filters: { + files: { + pii: { + fields: ['content'], + uninspectable: 'block', + }, + }, + }, + }, + { mimetype: 'application/octet-stream', size: 15 * 1024 * 1024 + 1 }, + ); + + const response = await request(testApp).post('/files').send({ + endpoint: 'agents', + agent_id: agentCustomId, + tool_resource: 'context', + file_id: uuidv4(), + }); + + expect(response.status).toBe(403); + expect(response.body.error).toBe('Forbidden'); + expect(readFile).not.toHaveBeenCalled(); + expect(processAgentFileUpload).not.toHaveBeenCalled(); + readFile.mockRestore(); + }); + it('should deny file upload to agent when user has no permission', async () => { // Create an agent owned by authorId await createAgent({ @@ -512,6 +562,13 @@ describe('File Routes - Agent Files Endpoint', () => { }); it('inspects mislabeled text content before upload processing', async () => { + await createAgent({ + id: agentCustomId, + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: authorId, + }); const readFile = jest .spyOn(fs, 'readFile') .mockResolvedValueOnce(Buffer.from('Contains PRIVATE-1234')); @@ -554,6 +611,13 @@ describe('File Routes - Agent Files Endpoint', () => { }); it('blocks opaque file content before upload processing when configured fail-closed', async () => { + await createAgent({ + id: agentCustomId, + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: authorId, + }); const readFile = jest .spyOn(fs, 'readFile') .mockResolvedValueOnce(Buffer.from([0, 255, 0, 137, 80, 78, 71])); @@ -709,6 +773,13 @@ describe('File Routes - Agent Files Endpoint', () => { }); it('does not defer raw content fail-close when extracted text is also selected', async () => { + await createAgent({ + id: agentCustomId, + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: authorId, + }); const readFile = jest .spyOn(fs, 'readFile') .mockResolvedValueOnce(Buffer.from([0, 255, 0, 80, 68, 70])); @@ -747,6 +818,13 @@ describe('File Routes - Agent Files Endpoint', () => { }); it('does not defer extracted-text fail-close for non-context agent resources', async () => { + await createAgent({ + id: agentCustomId, + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: authorId, + }); const readFile = jest .spyOn(fs, 'readFile') .mockResolvedValueOnce(Buffer.from([0, 255, 0, 80, 68, 70])); @@ -788,6 +866,13 @@ describe('File Routes - Agent Files Endpoint', () => { ['an unsupported binary', 'archive.bin', 'application/octet-stream'], ['audio handled as a transcript', 'recording.mp3', 'audio/mpeg'], ])('does not defer extracted-text fail-close for %s', async (_, originalname, mimetype) => { + await createAgent({ + id: agentCustomId, + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: authorId, + }); const readFile = jest .spyOn(fs, 'readFile') .mockResolvedValueOnce(Buffer.from([0, 255, 0, 80, 68, 70])); @@ -860,6 +945,13 @@ describe('File Routes - Agent Files Endpoint', () => { }); it('detects opaque content after a long printable file prefix', async () => { + await createAgent({ + id: agentCustomId, + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: authorId, + }); const readFile = jest .spyOn(fs, 'readFile') .mockResolvedValueOnce(Buffer.concat([Buffer.alloc(8192, 'a'), Buffer.from([0, 255])])); @@ -1241,7 +1333,7 @@ describe('File Routes - Agent Files Endpoint', () => { expect(JSON.stringify(response.body)).not.toContain(rawProviderDetail); }); - it('should allow file upload with agent_id but no tool_resource (message attachment)', async () => { + it('should deny a permanent agent upload with no tool_resource from a user without permission', async () => { // Create an agent owned by authorId await createAgent({ id: agentCustomId, @@ -1253,15 +1345,17 @@ describe('File Routes - Agent Files Endpoint', () => { const testApp = createAppWithUser(otherUserId); + /* Unified mode accepts an upload with no tool_resource and promotes it to a + * context resource, so the permission check has to apply here: otherwise any + * user could attach persistent context to an agent they cannot edit. */ const response = await request(testApp).post('/files').send({ endpoint: 'agents', agent_id: agentCustomId, file_id: uuidv4(), - // No tool_resource - permission check should not apply }); - expect(response.status).toBe(200); - expect(processAgentFileUpload).toHaveBeenCalled(); + expect(response.status).toBe(403); + expect(processAgentFileUpload).not.toHaveBeenCalled(); }); it('should allow message_file attachment to agent even without EDIT permission', async () => { diff --git a/api/server/routes/files/files.js b/api/server/routes/files/files.js index f0859b0fb1..de84c67c95 100644 --- a/api/server/routes/files/files.js +++ b/api/server/routes/files/files.js @@ -42,6 +42,11 @@ const { processDeleteRequest, processAgentFileUpload, } = require('~/server/services/Files/process'); +const { + resolveEffectiveToolResource, + resolveUploadEndpoint, + resolveUploadAgent, +} = require('~/server/services/Files/routing'); const { fileAccess } = require('~/server/middleware/accessResources/fileAccess'); const { getStrategyFunctions } = require('~/server/services/Files/strategies'); const { getOpenAIClient } = require('~/server/controllers/assistants/helpers'); @@ -781,7 +786,48 @@ const handleFileUpload = async (req, res) => { try { req.file.originalname = sanitizeFilename(req.file.originalname); - filterFile({ req }); + const isAssistants = isAssistantsEndpoint(metadata.endpoint); + + /* Authorization runs before anything reads the target agent. Validating against a + * record the caller cannot access answers with that agent's provider limits and + * content policy, so the rejection itself reports its configuration. */ + if (!isAssistants) { + const denied = await verifyAgentUploadPermission({ + req, + res, + metadata, + getAgent: ({ id }) => resolveUploadAgent(req, id), + checkPermission, + hasUploadBypass: () => hasCapability(req.user, SystemCapabilities.MANAGE_AGENTS), + }); + if (denied) { + return; + } + } + + /* Same configuration for validation and routing: an agent upload arrives as + * `agents` but is processed under the agent's own provider. */ + const effectiveEndpoint = await resolveUploadEndpoint({ + endpoint: metadata.endpoint, + agent_id: metadata.agent_id, + req, + }); + /* Carried so processing routes under the same configuration validation used, and so + * it can tell whether any enabled tool could consume a file kept off the model path, + * both from the one agent read this request already made. */ + metadata.effectiveEndpoint = effectiveEndpoint; + /* Left undefined when no agent record backs this upload, as for an ephemeral agent + * that exists only for the request. Processing then cannot judge what tools could + * consume the file and does not try. */ + const uploadAgent = await resolveUploadAgent(req, metadata.agent_id); + metadata.agentTools = uploadAgent?.tools; + metadata.useResponsesApi ??= uploadAgent?.model_parameters?.useResponsesApi; + filterFile({ req, endpoint: effectiveEndpoint }); + + /* Same destination the processing path will use: a unified upload routed to text + * becomes a context resource, and the preflight must account for that extraction + * before fail-closing on an uninspectable derived field. */ + const effectiveToolResource = await resolveEffectiveToolResource({ req, metadata }); /** Check the role permission before any content inspection: a forbidden upload * must be rejected without reading or embedding the file. */ @@ -803,7 +849,7 @@ const handleFileUpload = async (req, res) => { filters: req.config?.filters, file: req.file, endpoint: metadata.endpoint, - toolResource: metadata.tool_resource, + toolResource: effectiveToolResource, fileConfig: mergeFileConfig(req.config?.fileConfig), ocrConfigured: req.config?.ocr != null, ragConfigured: !!process.env.RAG_API_URL, @@ -813,7 +859,7 @@ const handleFileUpload = async (req, res) => { metadata.temp_file_id = metadata.file_id; metadata.file_id = req.file_id; - if (isAssistantsEndpoint(metadata.endpoint)) { + if (isAssistants) { openSseStreamIfRequested(); return await processFileUpload({ req, @@ -824,26 +870,6 @@ const handleFileUpload = async (req, res) => { }); } - let skipUploadAuth = false; - try { - skipUploadAuth = await hasCapability(req.user, SystemCapabilities.MANAGE_AGENTS); - } catch (err) { - logger.warn('[/files] capability check failed, denying bypass:', getSafeErrorMetadata(err)); - } - - if (!skipUploadAuth) { - const denied = await verifyAgentUploadPermission({ - req, - res, - metadata, - getAgent: db.getAgent, - checkPermission, - }); - if (denied) { - return; - } - } - openSseStreamIfRequested(); return await processAgentFileUpload({ req, res, metadata, sseStream }); } catch (error) { diff --git a/api/server/routes/files/images.agents.test.js b/api/server/routes/files/images.agents.test.js index 4bd5ffee22..e49215506d 100644 --- a/api/server/routes/files/images.agents.test.js +++ b/api/server/routes/files/images.agents.test.js @@ -22,6 +22,18 @@ jest.mock('~/server/services/Files/process', () => ({ filterFile: jest.fn(), })); +jest.mock('~/server/services/Files/routing', () => { + const actual = jest.requireActual('~/server/services/Files/routing'); + return { + ...actual, + /* Real by default so the dispatch is exercised end to end; individual tests override + * it for a single call to stand in for a routing configuration. */ + resolveEffectiveToolResource: jest.fn((...args) => + actual.resolveEffectiveToolResource(...args), + ), + }; +}); + jest.mock('fs', () => { const actualFs = jest.requireActual('fs'); return { @@ -35,6 +47,8 @@ jest.mock('fs', () => { const fs = require('fs'); const { processAgentFileUpload, processImageFile } = require('~/server/services/Files/process'); +const { resolveEffectiveToolResource } = require('~/server/services/Files/routing'); +const { filterFile } = require('~/server/services/Files/process'); const { UninspectableFileError } = require('@librechat/api'); const router = require('~/server/routes/files/images'); @@ -134,6 +148,13 @@ describe('POST /images - Agent Upload Permission Check (Integration)', () => { }; it('inspects the canonical sanitized image filename used by upload processing', async () => { + await createAgent({ + id: agentCustomId, + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: authorId, + }); const app = createAppWithUser( authorId, SystemRoles.USER, @@ -193,6 +214,41 @@ describe('POST /images - Agent Upload Permission Check (Integration)', () => { expect(fs.promises.unlink).toHaveBeenCalledWith('/tmp/t.png'); }); + it('denies an unauthorized caller before validating against the agent', async () => { + /* Image validation applies the target agent's provider policy, so reaching it first + * answers an unauthorized caller with that agent's configuration instead of a 403. */ + await createAgent({ + id: agentCustomId, + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: authorId, + }); + + const app = createAppWithUser(otherUserId, SystemRoles.USER, { + filters: { + files: { + pii: { + fields: ['content'], + starterPatterns: [], + customPatterns: [], + uninspectable: 'block', + }, + }, + }, + }); + const response = await request(app).post('/images').send({ + endpoint: 'agents', + agent_id: agentCustomId, + tool_resource: 'context', + file_id: uuidv4(), + }); + + expect(response.status).toBe(403); + expect(response.body.error).toBe('Forbidden'); + expect(processAgentFileUpload).not.toHaveBeenCalled(); + }); + it('should allow upload for agent owner', async () => { await createAgent({ id: agentCustomId, @@ -214,9 +270,36 @@ describe('POST /images - Agent Upload Permission Check (Integration)', () => { expect(processAgentFileUpload).toHaveBeenCalled(); }); + it('rejects a none-routed image for a standard chat', async () => { + const app = createAppWithUser(authorId, SystemRoles.USER, { + fileConfig: { + endpoints: { + openAI: { defaultLLMDeliveryPath: { overrides: { 'image/*': 'none' } } }, + }, + }, + }); + + const response = await request(app).post('/images').send({ + endpoint: 'openAI', + message_file: 'true', + file_id: uuidv4(), + }); + + expect(response.status).toBe(500); + expect(processImageFile).not.toHaveBeenCalled(); + expect(processAgentFileUpload).not.toHaveBeenCalled(); + }); + it.each(['content', 'extracted_text'])( - 'blocks opaque image %s before permission or processing side effects', + 'blocks opaque image %s before processing side effects', async (field) => { + await createAgent({ + id: agentCustomId, + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: authorId, + }); const app = createAppWithUser(authorId, SystemRoles.USER, { filters: { files: { @@ -284,6 +367,45 @@ describe('POST /images - Agent Upload Permission Check (Integration)', () => { expect(processAgentFileUpload).toHaveBeenCalledTimes(1); }); + it('defers extracted-text fail-close for a unified upload the config routes to text', async () => { + /* Same policy and file as the explicit-context case above, but with no tool_resource. + * Routing promotes it to a context resource, so the preflight has to see the same + * downstream extraction rather than fail-closing on an uninspectable derived field. */ + await createAgent({ + id: agentCustomId, + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: authorId, + }); + resolveEffectiveToolResource.mockResolvedValueOnce('context'); + + const app = createAppWithUser(authorId, SystemRoles.USER, { + filters: { + files: { + pii: { + fields: ['extracted_text'], + starterPatterns: [], + customPatterns: [], + uninspectable: 'block', + }, + }, + }, + fileConfig: { + ocr: { supportedMimeTypes: ['image/png'] }, + }, + ocr: {}, + }); + const response = await request(app).post('/images').send({ + endpoint: 'agents', + agent_id: agentCustomId, + file_id: uuidv4(), + }); + + expect(response.status).toBe(200); + expect(processAgentFileUpload).toHaveBeenCalledTimes(1); + }); + it('preserves a deferred extracted-text policy error from image processing', async () => { await createAgent({ id: agentCustomId, @@ -329,6 +451,13 @@ describe('POST /images - Agent Upload Permission Check (Integration)', () => { }); it('blocks extracted-text fail-close when configured OCR does not support the image MIME type', async () => { + await createAgent({ + id: agentCustomId, + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: authorId, + }); const app = createAppWithUser(authorId, SystemRoles.USER, { filters: { files: { @@ -361,6 +490,13 @@ describe('POST /images - Agent Upload Permission Check (Integration)', () => { }); it('preserves raw-content fail-close even when configured OCR supports the image', async () => { + await createAgent({ + id: agentCustomId, + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: authorId, + }); const app = createAppWithUser(authorId, SystemRoles.USER, { filters: { files: { @@ -487,6 +623,67 @@ describe('POST /images - Agent Upload Permission Check (Integration)', () => { expect(response.status).toBe(200); }); + it('validates an agent upload against the provider that will process it', async () => { + /* The request carries endpoint `agents`, but the agent's own provider governs + * routing, so it has to govern acceptance too or the two disagree. */ + await createAgent({ + id: agentCustomId, + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: authorId, + }); + const app = createAppWithUser(authorId); + + await request(app).post('/images').send({ + endpoint: 'agents', + agent_id: agentCustomId, + tool_resource: 'context', + file_id: uuidv4(), + }); + + expect(filterFile).toHaveBeenCalledWith( + expect.objectContaining({ endpoint: 'openai', image: true }), + ); + }); + + it('sends a permanent agent image through the agent upload path', async () => { + /* The image pipeline always stores a message attachment and never files anything + * against the agent, so a permanent upload sent there would report success while + * leaving an orphan. */ + await createAgent({ + id: agentCustomId, + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: authorId, + }); + const app = createAppWithUser(authorId); + + await request(app).post('/images').send({ + endpoint: 'agents', + agent_id: agentCustomId, + file_id: uuidv4(), + }); + + expect(processAgentFileUpload).toHaveBeenCalled(); + expect(processImageFile).not.toHaveBeenCalled(); + }); + + it('sends an image the config routes to text through the agent upload path', async () => { + resolveEffectiveToolResource.mockResolvedValueOnce('context'); + const app = createAppWithUser(otherUserId); + + const response = await request(app).post('/images').send({ + endpoint: 'agents', + file_id: uuidv4(), + }); + + expect(response.status).toBe(200); + expect(processAgentFileUpload).toHaveBeenCalled(); + expect(processImageFile).not.toHaveBeenCalled(); + }); + it('uses a normalized image error when file protection is active', async () => { const rawProviderDetail = 'PRIVATE-IMAGE echoed in provider failure'; const providerError = Object.assign(new Error(rawProviderDetail), { diff --git a/api/server/routes/files/images.js b/api/server/routes/files/images.js index 1228bddad1..7e5512f0cd 100644 --- a/api/server/routes/files/images.js +++ b/api/server/routes/files/images.js @@ -1,7 +1,7 @@ const path = require('path'); const fs = require('fs').promises; const express = require('express'); -const { logger } = require('@librechat/data-schemas'); +const { logger, SystemCapabilities } = require('@librechat/data-schemas'); const { getSafeErrorMetadata, shouldUseUploadSse, @@ -18,13 +18,24 @@ const { isAssistantsEndpoint, hasActivePiiPatterns, mergeFileConfig, + isMessageFileUpload, + getEndpointFileConfig, + resolveUploadLLMDeliveryPath, + isResponsesApiUpload, + isSpeechProviderConfigured, } = require('librechat-data-provider'); const { processAgentFileUpload, processImageFile, filterFile, } = require('~/server/services/Files/process'); +const { + resolveEffectiveToolResource, + resolveUploadEndpoint, + resolveUploadAgent, +} = require('~/server/services/Files/routing'); const { checkPermission } = require('~/server/services/PermissionService'); +const { hasCapability } = require('~/server/middleware/roles/capabilities'); const db = require('~/models'); const router = express.Router(); @@ -44,7 +55,70 @@ router.post('/', async (req, res) => { try { req.file.originalname = sanitizeFilename(req.file.originalname); - filterFile({ req, image: true }); + const isAssistants = isAssistantsEndpoint(metadata.endpoint); + + /* Authorization runs before anything reads the target agent, matching the file + * route. Validating against a record the caller cannot access answers with that + * agent's provider limits and content policy, so the rejection reports its + * configuration. Message attachments and requests naming no agent pass straight + * through without a read. */ + if (!isAssistants) { + const denied = await verifyAgentUploadPermission({ + req, + res, + metadata, + getAgent: ({ id }) => resolveUploadAgent(req, id), + checkPermission, + hasUploadBypass: () => hasCapability(req.user, SystemCapabilities.MANAGE_AGENTS), + }); + if (denied) { + return; + } + } + + /* Agent uploads arrive as `agents` but route by the agent's own provider, so the + * provider's configuration has to govern acceptance too. Resolved once here and + * reused by routing, authorization and processing below. */ + const effectiveEndpoint = await resolveUploadEndpoint({ + endpoint: metadata.endpoint, + agent_id: metadata.agent_id, + req, + }); + /* Carried so processing routes under the same configuration validation used, and so + * it can tell whether any enabled tool could consume a file kept off the model path, + * both from the one agent read this request already made. */ + metadata.effectiveEndpoint = effectiveEndpoint; + /* Left undefined when no agent record backs this upload, as for an ephemeral agent + * that exists only for the request. Processing then cannot judge what tools could + * consume the file and does not try. */ + const uploadAgent = await resolveUploadAgent(req, metadata.agent_id); + metadata.agentTools = uploadAgent?.tools; + metadata.useResponsesApi ??= uploadAgent?.model_parameters?.useResponsesApi; + filterFile({ req, image: true, endpoint: effectiveEndpoint }); + + /* A unified upload the config routes to text is processed as a context resource, so + * the preflight has to judge that destination. Told only the request's empty tool + * resource, it cannot see the extraction step and fail-closes on a derived field it + * would in fact be able to inspect. */ + const effectiveToolResource = await resolveEffectiveToolResource({ req, metadata }); + const fileConfig = mergeFileConfig(req.config?.fileConfig); + const deliveryPath = resolveUploadLLMDeliveryPath({ + mimeType: req.file.mimetype, + endpointConfig: getEndpointFileConfig({ fileConfig, endpoint: effectiveEndpoint }), + fileConfig, + endpoint: effectiveEndpoint, + useResponsesApi: isResponsesApiUpload(metadata.useResponsesApi), + sttConfigured: isSpeechProviderConfigured(req.config?.speech?.stt), + }); + if ( + deliveryPath === 'none' && + effectiveToolResource == null && + metadata.endpoint !== 'agents' + ) { + throw new Error( + `Files of type ${req.file.mimetype} are not sent to the model here, and this conversation has no agent whose tools could read them.`, + ); + } /** The Code Files UI routes image uploads here instead of `/files`, so the * same role gate has to run before any content inspection — otherwise an @@ -62,8 +136,8 @@ router.post('/', async (req, res) => { filters: req.config?.filters, file: req.file, endpoint: metadata.endpoint, - toolResource: metadata.tool_resource, - fileConfig: mergeFileConfig(req.config?.fileConfig), + toolResource: effectiveToolResource, + fileConfig, ocrConfigured: req.config?.ocr != null, ragConfigured: !!process.env.RAG_API_URL, rawFileMode: 'opaque', @@ -72,17 +146,20 @@ router.post('/', async (req, res) => { metadata.temp_file_id = metadata.file_id; metadata.file_id = req.file_id; - if (!isAssistantsEndpoint(metadata.endpoint) && metadata.tool_resource != null) { - const denied = await verifyAgentUploadPermission({ - req, - res, - metadata, - getAgent: db.getAgent, - checkPermission, - }); - if (denied) { - return; - } + /* An image the config routes to text delivery has to go through the agent upload + * path, which extracts and stores the text. The image pipeline would persist the + * routing without any text, leaving the file out of provider delivery and out of + * the text context both. + * + * A permanent upload against an agent takes that path regardless of what the routing + * inferred, because the image pipeline always stores a message attachment and never + * files anything against the agent. Sent here it would report success while leaving + * an orphan, so it goes where that is decided rather than assumed. */ + const isPermanentAgentUpload = + metadata.agent_id != null && !isMessageFileUpload(metadata.message_file); + const takesAgentUploadPath = effectiveToolResource != null || isPermanentAgentUpload; + + if (!isAssistants && takesAgentUploadPath) { openSseStreamIfRequested(); return await processAgentFileUpload({ req, res, metadata, sseStream }); } diff --git a/api/server/routes/files/multer.js b/api/server/routes/files/multer.js index 023619c579..5f375f39cd 100644 --- a/api/server/routes/files/multer.js +++ b/api/server/routes/files/multer.js @@ -7,6 +7,7 @@ const { logger } = require('@librechat/data-schemas'); const { mergeFileConfig, inferMimeType, + isAgentsEndpoint, getEndpointFileConfig, fileConfig: defaultFileConfig, } = require('librechat-data-provider'); @@ -58,6 +59,18 @@ const importFileFilter = (req, file, cb) => { } }; +/** Every type some configured endpoint accepts, for a request whose real endpoint is only + * known after an agent read this filter cannot make. */ +const collectSupportedMimeTypes = (customFileConfig, endpointFileConfig) => { + const merged = [...(endpointFileConfig.supportedMimeTypes ?? [])]; + for (const config of Object.values(customFileConfig?.endpoints ?? {})) { + for (const mimeType of config?.supportedMimeTypes ?? []) { + merged.push(mimeType); + } + } + return merged; +}; + const normalizeUploadMimeType = (file) => { const mimeType = inferMimeType(file.originalname || '', file.mimetype || ''); if (mimeType && file.mimetype !== mimeType) { @@ -96,7 +109,16 @@ const createFileFilter = (customFileConfig, resolveEndpoint) => { endpointType, }); - if (!defaultFileConfig.checkType(mimeType, endpointFileConfig.supportedMimeTypes)) { + /* An agent upload is validated again under the agent's own provider once the route + * has resolved and authorized it. That provider's allowlist can be wider than the + * `agents` entry, and this filter is synchronous so it cannot resolve it, so here the + * question is only whether any configured endpoint accepts the type. Narrowing to + * `agents` would make the later provider check able to reject but never to permit. */ + const supportedMimeTypes = isAgentsEndpoint(endpoint) + ? collectSupportedMimeTypes(customFileConfig, endpointFileConfig) + : endpointFileConfig.supportedMimeTypes; + + if (!defaultFileConfig.checkType(mimeType, supportedMimeTypes)) { return cb( createCustomError(415, 'Unsupported file type: ' + (file.mimetype || mimeType)), false, diff --git a/api/server/routes/files/multer.spec.js b/api/server/routes/files/multer.spec.js index 1c186dc745..351ed34a02 100644 --- a/api/server/routes/files/multer.spec.js +++ b/api/server/routes/files/multer.spec.js @@ -739,4 +739,64 @@ describe('Multer Configuration', () => { expect(getAppConfig).toHaveBeenCalled(); }); }); + + describe('agent uploads and provider allowlists', () => { + const configWithWiderProvider = () => { + const { mergeFileConfig } = require('librechat-data-provider'); + return mergeFileConfig({ + endpoints: { + agents: { supportedMimeTypes: ['^application/pdf$'] }, + 'Custom Provider': { supportedMimeTypes: ['^application/pdf$', '^video/mp4$'] }, + }, + }); + }; + + it('accepts a type only the agent provider allows', (done) => { + /* This filter is synchronous and runs before the agent read, so narrowing to the + * agents entry would make the later provider check able to reject but never + * permit. The route validates again under the resolved provider. */ + const fileFilter = createFileFilter(configWithWiderProvider()); + mockReq.body = { endpoint: 'agents' }; + + fileFilter( + mockReq, + { ...mockFile, originalname: 'clip.mp4', mimetype: 'video/mp4' }, + (err, accepted) => { + expect(err).toBeNull(); + expect(accepted).toBe(true); + done(); + }, + ); + }); + + it('still refuses a type no configured endpoint allows', (done) => { + const fileFilter = createFileFilter(configWithWiderProvider()); + mockReq.body = { endpoint: 'agents' }; + + fileFilter( + mockReq, + { ...mockFile, originalname: 'installer.exe', mimetype: 'application/x-msdownload' }, + (err, accepted) => { + expect(err).toBeTruthy(); + expect(accepted).toBe(false); + done(); + }, + ); + }); + + it('keeps a non-agent endpoint judged by its own allowlist alone', (done) => { + const fileFilter = createFileFilter(configWithWiderProvider()); + mockReq.body = { endpoint: 'agents-lookalike' }; + + fileFilter( + mockReq, + { ...mockFile, originalname: 'clip.mp4', mimetype: 'video/mp4' }, + (err, accepted) => { + expect(err).toBeTruthy(); + expect(accepted).toBe(false); + done(); + }, + ); + }); + }); }); diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js index 9d73e6d711..053f809e4a 100644 --- a/api/server/services/Endpoints/agents/initialize.js +++ b/api/server/services/Endpoints/agents/initialize.js @@ -64,6 +64,13 @@ const { resolveMemoryAvailability, enrichLoadedToolsWithAgentContext, } = require('./skillDeps'); +const { + loadCodeApiKey, + provisionToCodeEnv, + provisionToVectorDB, + checkSessionsAlive, +} = require('~/server/services/Files/provision'); +const { createProvisionFilesCallback } = require('~/server/services/Files/provisionCallback'); const { getModelsConfig } = require('~/server/controllers/ModelController'); const { checkPermission, findAccessibleResources } = require('~/server/services/PermissionService'); const AgentClient = require('~/server/controllers/agents/client'); @@ -476,6 +483,11 @@ const initializeClient = async ({ } }, ...getSkillToolDeps(), + provisionFiles: createProvisionFilesCallback({ + req, + agentToolContexts, + resolvePrimaryAgentId: () => primaryConfig?.id, + }), }; const summarizationOptions = @@ -647,12 +659,18 @@ const initializeClient = async ({ updateFilesUsage: db.updateFilesUsage, getUserKeyValues: db.getUserKeyValues, getUserCodeFiles: db.getUserCodeFiles, + getDeferredProvisionFiles: db.getDeferredProvisionFiles, getToolFilesByIds: db.getToolFilesByIds, getCodeGeneratedFiles: db.getCodeGeneratedFiles, filterFilesByAgentAccess, listSkillsByAccess: skillDbMethods.listSkillsByAccess, listAlwaysApplySkills: skillDbMethods.listAlwaysApplySkills, getSkillByName: skillDbMethods.getSkillByName, + provisionToCodeEnv, + provisionToVectorDB, + checkSessionsAlive, + loadCodeApiKey, + updateFile: db.updateFile, }, ); @@ -731,12 +749,18 @@ const initializeClient = async ({ updateFilesUsage: db.updateFilesUsage, getUserKeyValues: db.getUserKeyValues, getUserCodeFiles: db.getUserCodeFiles, + getDeferredProvisionFiles: db.getDeferredProvisionFiles, getToolFilesByIds: db.getToolFilesByIds, getCodeGeneratedFiles: db.getCodeGeneratedFiles, filterFilesByAgentAccess, listSkillsByAccess: skillDbMethods.listSkillsByAccess, listAlwaysApplySkills: skillDbMethods.listAlwaysApplySkills, getSkillByName: skillDbMethods.getSkillByName, + provisionToCodeEnv, + provisionToVectorDB, + checkSessionsAlive, + loadCodeApiKey, + updateFile: db.updateFile, }, // The callback fires during BFS, before the helper prunes agents // whose edges end up filtered. Don't populate `agentConfigs` here — @@ -1177,12 +1201,18 @@ const initializeClient = async ({ updateFilesUsage: db.updateFilesUsage, getUserKeyValues: db.getUserKeyValues, getUserCodeFiles: db.getUserCodeFiles, + getDeferredProvisionFiles: db.getDeferredProvisionFiles, getToolFilesByIds: db.getToolFilesByIds, getCodeGeneratedFiles: db.getCodeGeneratedFiles, filterFilesByAgentAccess, listSkillsByAccess: skillDbMethods.listSkillsByAccess, listAlwaysApplySkills: skillDbMethods.listAlwaysApplySkills, getSkillByName: skillDbMethods.getSkillByName, + provisionToCodeEnv, + provisionToVectorDB, + checkSessionsAlive, + loadCodeApiKey, + updateFile: db.updateFile, }, ), context.signal, diff --git a/api/server/services/Endpoints/agents/skillDeps.js b/api/server/services/Endpoints/agents/skillDeps.js index 8fe49aedc3..2558ba021c 100644 --- a/api/server/services/Endpoints/agents/skillDeps.js +++ b/api/server/services/Endpoints/agents/skillDeps.js @@ -311,6 +311,7 @@ function buildAgentToolContext({ agent, config }) { fileAuthoringToolNames: config.fileAuthoringToolNames, skillPrimedIdsByName: buildSkillPrimedIdsByName(config.manualSkillPrimes, config.alwaysApplySkillPrimes) ?? {}, + provisionState: config.provisionState, }; } diff --git a/api/server/services/Files/Audio/STTService.js b/api/server/services/Files/Audio/STTService.js index 0f52a277e2..1b0f6f5942 100644 --- a/api/server/services/Files/Audio/STTService.js +++ b/api/server/services/Files/Audio/STTService.js @@ -16,7 +16,11 @@ const { contentFilterUninspectableResponse, getBlockedUninspectableFileField, } = require('@librechat/api'); -const { extractEnvVariable, STTProviders } = require('librechat-data-provider'); +const { + STTProviders, + extractEnvVariable, + listConfiguredSpeechProviders, +} = require('librechat-data-provider'); const { getAppConfig } = require('~/server/services/Config'); /** @@ -163,9 +167,7 @@ class STTService { ); } - const providers = Object.entries(sttSchema).filter( - ([key, value]) => key !== 'allowedAddresses' && Object.keys(value).length > 0, - ); + const providers = listConfiguredSpeechProviders(sttSchema); if (providers.length !== 1) { throw new Error( diff --git a/api/server/services/Files/Audio/STTService.spec.js b/api/server/services/Files/Audio/STTService.spec.js index d8bcf364bc..d1a37c4327 100644 --- a/api/server/services/Files/Audio/STTService.spec.js +++ b/api/server/services/Files/Audio/STTService.spec.js @@ -52,6 +52,9 @@ jest.mock('@librechat/api', () => ({ })); jest.mock('librechat-data-provider', () => ({ extractEnvVariable: jest.fn(), + /* Real selection logic: these tests are about which provider a schema resolves to. */ + listConfiguredSpeechProviders: + jest.requireActual('librechat-data-provider').listConfiguredSpeechProviders, STTProviders: {}, })); jest.mock('~/server/services/Config', () => ({ getAppConfig: jest.fn() })); diff --git a/api/server/services/Files/Audio/TTSService.js b/api/server/services/Files/Audio/TTSService.js index bace395504..99b3cdb428 100644 --- a/api/server/services/Files/Audio/TTSService.js +++ b/api/server/services/Files/Audio/TTSService.js @@ -7,7 +7,11 @@ const { resolveConfigSecret, applySSRFSafeAgentIfDirect, } = require('@librechat/api'); -const { extractEnvVariable, TTSProviders } = require('librechat-data-provider'); +const { + TTSProviders, + extractEnvVariable, + listConfiguredSpeechProviders, +} = require('librechat-data-provider'); const { getRandomVoiceId, createChunkProcessor, splitTextIntoChunks } = require('./streamAudio'); const { getAppConfig } = require('~/server/services/Config'); @@ -52,9 +56,7 @@ class TTSService { 'No TTS schema is set. Did you configure TTS in the custom config (librechat.yaml)?', ); } - const providers = Object.entries(ttsSchema).filter( - ([key, value]) => key !== 'allowedAddresses' && Object.keys(value).length > 0, - ); + const providers = listConfiguredSpeechProviders(ttsSchema); if (providers.length !== 1) { throw new Error( diff --git a/api/server/services/Files/Audio/TTSService.spec.js b/api/server/services/Files/Audio/TTSService.spec.js index 1f9fb79dc0..8273d8b4e5 100644 --- a/api/server/services/Files/Audio/TTSService.spec.js +++ b/api/server/services/Files/Audio/TTSService.spec.js @@ -8,6 +8,9 @@ jest.mock('@librechat/api', () => ({ })); jest.mock('librechat-data-provider', () => ({ extractEnvVariable: jest.fn((value) => value), + /* Real selection logic: these tests are about which provider a schema resolves to. */ + listConfiguredSpeechProviders: + jest.requireActual('librechat-data-provider').listConfiguredSpeechProviders, TTSProviders: { OPENAI: 'openai', AZURE_OPENAI: 'azureOpenAI', diff --git a/api/server/services/Files/Azure/crud.js b/api/server/services/Files/Azure/crud.js index 59ffed9891..5944f40ab7 100644 --- a/api/server/services/Files/Azure/crud.js +++ b/api/server/services/Files/Azure/crud.js @@ -1,7 +1,6 @@ const fs = require('fs'); const path = require('path'); const mime = require('mime'); -const axios = require('axios'); const fetch = require('node-fetch'); const { logger } = require('@librechat/data-schemas'); const { @@ -250,14 +249,43 @@ async function uploadFileToAzure({ * @param {string} fileURL - The URL of the blob. * @returns {Promise} A readable stream of the blob. */ -async function getAzureFileStream(_req, fileURL) { +async function getAzureFileStream(_req, fileURL, { signal } = {}) { try { - const response = await axios({ - method: 'get', - url: fileURL, - responseType: 'stream', + const url = new URL(fileURL); + const configuredClient = await getAzureContainerClient(); + const configuredURL = configuredClient.url ? new URL(configuredClient.url) : undefined; + const configuredPrefix = configuredURL?.pathname.replace(/\/$/, ''); + let containerClient = configuredClient; + let blobPath; + + if ( + configuredURL && + configuredPrefix && + url.origin === configuredURL.origin && + url.pathname.startsWith(`${configuredPrefix}/`) + ) { + /* Azurite puts the account name before the container in the path. The configured + * container URL already includes both, so resolve the blob relative to it. */ + blobPath = url.pathname.slice(configuredPrefix.length + 1); + } else { + const pathSegments = url.pathname.split('/').filter(Boolean); + const containerName = pathSegments.shift(); + blobPath = pathSegments.join('/'); + if (containerName) { + containerClient = await getAzureContainerClient(decodeURIComponent(containerName)); + } + } + blobPath = blobPath?.split('/').map(decodeURIComponent).join('/'); + if (!blobPath) { + throw new Error('Invalid Azure Blob URL'); + } + const response = await containerClient.getBlockBlobClient(blobPath).download(0, undefined, { + abortSignal: signal, }); - return response.data; + if (!response.readableStreamBody) { + throw new Error('Azure Blob download returned no readable stream'); + } + return response.readableStreamBody; } catch (error) { logger.error('[getAzureFileStream] Error getting blob stream:', error); throw error; diff --git a/api/server/services/Files/Azure/crud.spec.js b/api/server/services/Files/Azure/crud.spec.js new file mode 100644 index 0000000000..c7e87b1d9b --- /dev/null +++ b/api/server/services/Files/Azure/crud.spec.js @@ -0,0 +1,57 @@ +const mockDownload = jest.fn(); +const mockGetBlockBlobClient = jest.fn(() => ({ download: mockDownload })); +const mockGetAzureContainerClient = jest.fn(async () => ({ + url: 'https://account.blob.core.windows.net/files', + getBlockBlobClient: mockGetBlockBlobClient, +})); + +jest.mock('@librechat/data-schemas', () => ({ + logger: { error: jest.fn() }, +})); + +jest.mock('@librechat/api', () => ({ + deleteRagFile: jest.fn(), + assertRemoteFileURL: jest.fn((url) => url), + getAzureContainerClient: (...args) => mockGetAzureContainerClient(...args), + getRemoteFileFetchMaxBytes: jest.fn(() => 1024), + getRemoteFileFetchTimeoutMs: jest.fn(() => 1000), + assertRemoteFileContentLength: jest.fn(), +})); + +const { getAzureFileStream } = require('./crud'); + +describe('getAzureFileStream', () => { + it('downloads private blobs through the authenticated Azure client', async () => { + const stream = { pipe: jest.fn() }; + mockDownload.mockResolvedValue({ readableStreamBody: stream }); + + await expect( + getAzureFileStream( + {}, + 'https://account.blob.core.windows.net/private-files/uploads/user/report%20one.pdf', + ), + ).resolves.toBe(stream); + + expect(mockGetAzureContainerClient).toHaveBeenCalledWith('private-files'); + expect(mockGetBlockBlobClient).toHaveBeenCalledWith('uploads/user/report one.pdf'); + }); + + it('resolves Azurite blobs relative to the configured account and container path', async () => { + const stream = { pipe: jest.fn() }; + mockDownload.mockResolvedValue({ readableStreamBody: stream }); + mockGetAzureContainerClient.mockResolvedValueOnce({ + url: 'http://127.0.0.1:10000/devstoreaccount1/files', + getBlockBlobClient: mockGetBlockBlobClient, + }); + + await expect( + getAzureFileStream( + {}, + 'http://127.0.0.1:10000/devstoreaccount1/files/uploads/user/report%20one.pdf', + ), + ).resolves.toBe(stream); + + expect(mockGetAzureContainerClient).toHaveBeenCalledWith(); + expect(mockGetBlockBlobClient).toHaveBeenCalledWith('uploads/user/report one.pdf'); + }); +}); diff --git a/api/server/services/Files/Code/crud.js b/api/server/services/Files/Code/crud.js index aaa53a82d8..3732e4f6b7 100644 --- a/api/server/services/Files/Code/crud.js +++ b/api/server/services/Files/Code/crud.js @@ -197,6 +197,7 @@ async function uploadCodeEnvFile({ codeApiBaseUrl, executionProfile, bridgeWorkerId, + signal, }) { try { const form = new FormData(); @@ -220,6 +221,7 @@ async function uploadCodeEnvFile({ timeout: 120000, maxContentLength: MAX_FILE_SIZE, maxBodyLength: MAX_FILE_SIZE, + ...(signal ? { signal } : {}), }; const response = await axios.post(`${baseURL}/upload`, form, options); diff --git a/api/server/services/Files/Code/process.js b/api/server/services/Files/Code/process.js index ab76eaf3aa..001a43423e 100644 --- a/api/server/services/Files/Code/process.js +++ b/api/server/services/Files/Code/process.js @@ -53,6 +53,7 @@ const { mergeCodeEnvRef, getCodeEnvRefForProfile, getEndpointFileConfig, + resolveSandboxFilename, } = require('librechat-data-provider'); const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions'); const { createFile, getFiles, updateFile, claimCodeFile } = require('~/models'); @@ -1395,11 +1396,20 @@ const primeFiles = async (options) => { * codeapi can resolve sessionKey per-file (kind switch + * tenant prefix from auth context). */ + const sandboxName = resolveSandboxFilename(file.filename, file.type); + let claimedDestination; + const getDestination = () => { + claimedDestination ??= claimCodeDestination(destinations, sandboxName, file.file_id); + return claimedDestination; + }; + const pushFile = (overrideSessionId, overrideId) => { /* Claimed here rather than up front so files that never reach the * sandbox — no code-env ref, or a failed re-upload — do not reserve a * name and push a file that does reach it onto a counter. */ - const destination = claimCodeDestination(destinations, file.filename, file.file_id); + /* The sandbox holds the converted name, not the record's, so the mount path has + * to follow the same rule provisioning uploaded under. */ + const destination = getDestination(); if (destination !== file.filename) { logger.debug( `[primeCodeFiles] file=${file.file_id} destination=${destination} ` + @@ -1441,7 +1451,7 @@ const primeFiles = async (options) => { const uploaded = await uploadCodeEnvFile({ req: options.req, stream, - filename: file.filename, + filename: getDestination(), kind: sourceRef.kind, id: sourceRef.id, ...(sourceRef.kind === 'skill' ? { version: sourceRef.version } : {}), diff --git a/api/server/services/Files/Code/process.spec.js b/api/server/services/Files/Code/process.spec.js index 91473eb74f..f678feefab 100644 --- a/api/server/services/Files/Code/process.spec.js +++ b/api/server/services/Files/Code/process.spec.js @@ -2760,6 +2760,42 @@ describe('Code Process', () => { ]); }); + it('reuploads a recovered converted image under its advertised sandbox name', async () => { + getFiles.mockResolvedValue([ + { + file_id: 'converted-image', + filename: 'photo.png', + filepath: '/uploads/photo.webp', + type: 'image/webp', + source: 'local', + context: 'execute_code', + metadata: { + codeEnvRef: { + kind: 'user', + id: 'user-123', + storage_session_id: 'OLD_SESSION', + file_id: 'OLD_ID', + }, + }, + }, + ]); + const { handleFileUpload } = setupReuploadMocks({ + storage_session_id: 'NEW_SESSION', + file_id: 'NEW_ID', + }); + + const result = await primeFiles({ + req: { user: { id: 'user-123', role: 'USER' } }, + tool_resources: { execute_code: { file_ids: ['converted-image'], files: [] } }, + agentId: 'agent-id', + }); + + expect(handleFileUpload).toHaveBeenCalledWith( + expect.objectContaining({ filename: 'photo.webp' }), + ); + expect(result.files?.[0]?.name).toBe('photo.webp'); + }); + /* Phase C / option α (codeapi #1455): reupload preserves the * resource identity from the existing ref so codeapi re-buckets * under the same sessionKey shape. Without this, a skill-cache-miss @@ -3112,6 +3148,28 @@ describe('Code Process', () => { ]); }); + it('mounts a converted image under the name the sandbox actually holds', async () => { + /* Image uploads are converted to the configured output type while the record keeps + * the original filename. Provisioning uploaded the converted name, so rebuilding + * the mount path from the record advertises a file the sandbox does not have. */ + setupSessionInfoOk(); + getFiles.mockResolvedValue([ + makeFile({ + file_id: 'fid-ready', + filename: 'photo.png', + type: 'image/webp', + }), + ]); + + const result = await primeFiles({ + req: { user: { id: 'user-123', role: 'USER' } }, + tool_resources: { execute_code: { file_ids: ['fid-ready'], files: [] } }, + agentId: 'agent-id', + }); + + expect(result.files?.[0]?.name).toBe('photo.webp'); + }); + it('annotates a pending file with "(preview not yet generated)"', async () => { setupSessionInfoOk(); getFiles.mockResolvedValue([makeFile({ status: 'pending' })]); diff --git a/api/server/services/Files/Firebase/crud.js b/api/server/services/Files/Firebase/crud.js index 9fbdb91dbe..cf6d4c74e3 100644 --- a/api/server/services/Files/Firebase/crud.js +++ b/api/server/services/Files/Firebase/crud.js @@ -249,7 +249,7 @@ async function uploadFileToFirebase({ req, file, file_id }) { * @param {string} filepath - The filepath. * @returns {Promise} A readable stream of the file. */ -async function getFirebaseFileStream(_req, filepath) { +async function getFirebaseFileStream(_req, filepath, { signal } = {}) { try { const storage = getFirebaseStorage(); if (!storage) { @@ -260,6 +260,7 @@ async function getFirebaseFileStream(_req, filepath) { method: 'get', url: filepath, responseType: 'stream', + signal, }); return response.data; diff --git a/api/server/services/Files/VectorDB/crud.js b/api/server/services/Files/VectorDB/crud.js index 4eea1169c1..9f43f0e4ae 100644 --- a/api/server/services/Files/VectorDB/crud.js +++ b/api/server/services/Files/VectorDB/crud.js @@ -64,7 +64,7 @@ const deleteVectors = async (req, file) => { * - filepath: The path where the file is saved. * - bytes: The size of the file in bytes. */ -async function uploadVectors({ req, file, file_id, entity_id, storageMetadata }) { +async function uploadVectors({ req, file, file_id, entity_id, storageMetadata, signal }) { if (!process.env.RAG_API_URL) { throw new Error('RAG_API_URL not defined'); } @@ -86,6 +86,7 @@ async function uploadVectors({ req, file, file_id, entity_id, storageMetadata }) const formHeaders = formData.getHeaders(); const response = await axios.post(`${process.env.RAG_API_URL}/embed`, formData, { + ...(signal ? { signal } : {}), headers: { Authorization: `Bearer ${jwtToken}`, accept: 'application/json', diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js index 517d90c466..c888746bea 100644 --- a/api/server/services/Files/process.js +++ b/api/server/services/Files/process.js @@ -17,6 +17,14 @@ const { removeNullishValues, isAssistantsEndpoint, getEndpointFileConfig, + resolveUploadLLMDeliveryPath, + isNativelyReadableText, + resolveUploadDestination, + resolveSandboxFilename, + canToolResourceConsume, + isMessageFileUpload, + isResponsesApiUpload, + isSpeechProviderConfigured, } = require('librechat-data-provider'); const { logger, runAsSystem } = require('@librechat/data-schemas'); const { @@ -460,7 +468,7 @@ const processFileURL = async ({ * @param {ServerRequest} params.req - The Express request object. * @param {Express.Response} [params.res] - The Express response object. * @param {ImageMetadata} params.metadata - Additional metadata for the file. - * @param {boolean} params.returnFile - Whether to return the file metadata or return response as normal. + * @param {boolean} params.returnFile - Return the converted file's metadata without persisting it, for a caller that creates its own record. * @param {import('@librechat/api').UploadSseStream | null} [params.sseStream] - Active upload SSE stream, if enabled. * @returns {Promise} */ @@ -471,6 +479,19 @@ const processImageFile = async ({ req, res, metadata, returnFile = false, sseStr const source = getFileStrategy(appConfig, { isImage: true }); const { handleImageUpload } = getStrategyFunctions(source); const { file_id, temp_file_id, endpoint } = metadata; + const fileConfig = mergeFileConfig(appConfig?.fileConfig); + /* The route resolved the agent's provider before validating, so the same endpoint + * governs delivery routing here. */ + const configEndpoint = metadata.effectiveEndpoint ?? endpoint; + const endpointConfig = getEndpointFileConfig({ fileConfig, endpoint: configEndpoint }); + const llmDeliveryPath = resolveUploadLLMDeliveryPath({ + mimeType: file.mimetype, + endpointConfig, + fileConfig, + endpoint: configEndpoint, + useResponsesApi: isResponsesApiUpload(metadata.useResponsesApi ?? req.body?.useResponsesApi), + sttConfigured: isSpeechProviderConfigured(appConfig?.speech?.stt), + }); const { filepath, bytes, width, height, storageKey, storageRegion } = await handleImageUpload({ req, @@ -480,29 +501,41 @@ const processImageFile = async ({ req, res, metadata, returnFile = false, sseStr }); const storageMetadata = getStorageMetadata({ filepath, source, storageKey, storageRegion }); - const result = await db.createFile( - { - user: req.user.id, - file_id, - temp_file_id, - bytes, - filepath, - ...storageMetadata, - filename: file.originalname, - context: FileContext.message_attachment, - source, - type: `image/${appConfig.imageOutputType}`, - ...(await retentionExpiryPromise), - width, - height, - tenantId: req.user.tenantId, + const fileInfo = { + user: req.user.id, + file_id, + temp_file_id, + bytes, + filepath, + ...storageMetadata, + filename: file.originalname, + context: FileContext.message_attachment, + source, + type: `image/${appConfig.imageOutputType}`, + ...(await retentionExpiryPromise), + width, + height, + tenantId: req.user.tenantId, + llmDeliveryPath, + /* The image route persists through here directly, so the choice has to be recorded + * on this path too. Absent, a later turn substitutes its own endpoint's mode. */ + metadata: { + destinationChosen: + endpointConfig?.legacyFileUploadUX === true || metadata.tool_resource != null, + ...(`image/${appConfig.imageOutputType}` !== file.mimetype + ? { routingMimeType: file.mimetype } + : {}), }, - true, - ); + }; + /* Callers asking for the file are converting an image for a record of their own, under + * a different id. Persisting here would leave that row referenced by nothing while the + * converted object it points at is the one they go on to use. */ if (returnFile) { - return result; + return fileInfo; } + + const result = await db.createFile(fileInfo, true); sendUploadSuccess(res, sseStream, 'File uploaded and processed successfully', result); }; @@ -690,19 +723,150 @@ const processFileUpload = async ({ req, res, metadata, sseStream, openai: provid * @param {import('@librechat/api').UploadSseStream | null} [params.sseStream] - Active upload SSE stream, if enabled. * @returns {Promise} */ +/** Reader-facing names for the destinations an upload can be rejected against. */ +const TOOL_RESOURCE_LABELS = { + [EToolResources.execute_code]: 'the code interpreter', + [EToolResources.code_interpreter]: 'the code interpreter', + [EToolResources.file_search]: 'file search', + [EToolResources.context]: 'text context', + [EToolResources.image_edit]: 'image editing', + [EToolResources.ocr]: 'OCR', +}; + +/** Capability gate for each tool that can consume a file kept off the model path. */ +const CONSUMER_CAPABILITIES = [ + [EToolResources.execute_code, AgentCapabilities.execute_code], + [EToolResources.file_search, AgentCapabilities.file_search], +]; + +/** + * Narrows an agent's tools to those this deployment will actually honor. Filing a file + * under a disabled capability fails the upload on a rule the agent's tool order picked. + */ +const filterEnabledConsumers = async (req, agentTools, roleGrants) => { + if (!agentTools?.length) { + return agentTools; + } + const candidates = CONSUMER_CAPABILITIES.filter(([resource]) => agentTools.includes(resource)); + if (candidates.length === 0) { + return agentTools; + } + const enabled = await Promise.all( + candidates.map(([, capability]) => checkCapability(req, capability)), + ); + const disabled = new Set( + candidates + .filter(([resource], index) => { + if (!enabled[index]) { + return true; + } + if (resource === EToolResources.execute_code) { + return roleGrants?.runCode === false; + } + return resource === EToolResources.file_search && roleGrants?.fileSearch === false; + }) + .map(([resource]) => resource), + ); + return disabled.size > 0 ? agentTools.filter((tool) => !disabled.has(tool)) : agentTools; +}; + const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { const { file } = req; const appConfig = req.config; const { agent_id, tool_resource, file_id, temp_file_id = null } = metadata; - let messageAttachment = !!metadata.message_file; + let messageAttachment = isMessageFileUpload(metadata.message_file); + + let effectiveToolResource; + + const fileConfig = mergeFileConfig(appConfig?.fileConfig); + const endpoint = metadata.effectiveEndpoint ?? req.body?.endpoint; + const endpointConfig = getEndpointFileConfig({ fileConfig, endpoint }); + + /* Recorded on the file below, both ways: the endpoint setting can differ on a later + * turn, but the user's decision about this file does not change with it. An absent + * marker means a record written before this was tracked, not an inferred destination. + * + * A destination is the user's whenever they named one, which the chooser always does + * and a request naming a tool resource does too. Recording the endpoint mode instead + * would treat an explicitly sandbox-only upload in unified mode as inferred. */ + const legacyUploadUX = endpointConfig?.legacyFileUploadUX === true; + const uploadChoiceMetadata = { destinationChosen: legacyUploadUX || tool_resource != null }; if (agent_id && !tool_resource && !messageAttachment) { - throw new Error('No tool resource provided for agent file upload'); + if (legacyUploadUX) { + throw new Error('No tool resource provided for agent file upload'); + } } - if (tool_resource === EToolResources.file_search && file.mimetype.startsWith('image')) { - throw new Error('Image uploads are not supported for file search tool resources'); + const llmDeliveryPath = resolveUploadLLMDeliveryPath({ + toolResource: tool_resource, + mimeType: file.mimetype, + endpointConfig, + fileConfig, + endpoint, + useResponsesApi: isResponsesApiUpload(metadata.useResponsesApi ?? req.body?.useResponsesApi), + sttConfigured: isSpeechProviderConfigured(appConfig?.speech?.stt), + }); + + /* Destination and acceptability are one decision, made by shared policy rather than + * rebuilt here. `agentTools` is undefined when no agent record backs the upload. */ + /* Only a permanent agent upload can land on a context resource, so the capability is + * looked up only there and the common attachment path pays nothing for it. */ + const contextEnabled = + messageAttachment || agent_id == null + ? undefined + : await checkCapability(req, AgentCapabilities.context); + const hasRoleGatedConsumer = + tool_resource === EToolResources.execute_code || + tool_resource === EToolResources.file_search || + metadata.agentTools?.some( + (tool) => tool === EToolResources.execute_code || tool === EToolResources.file_search, + ); + const toolRoleGrants = hasRoleGatedConsumer + ? await resolveToolRoleGrants({ + req, + getRoleByName: db.getRoleByName, + context: 'fileUpload', + }) + : undefined; + const destination = resolveUploadDestination({ + toolResource: tool_resource, + deliveryPath: llmDeliveryPath, + mimeType: file.mimetype, + agentTools: await filterEnabledConsumers(req, metadata.agentTools, toolRoleGrants), + hasAgent: agent_id != null, + isMessageAttachment: messageAttachment, + /* A message attachment can acquire a file-tool consumer later in the same draft or + * on a later turn. This deliberately includes saved agents whose persisted tools are + * empty: skills and per-turn tool selection are not represented by agent.tools. + * Audio has no deferred file-tool contract and still requires STT. Permanent agent + * uploads must be filed under a durable resource before returning. */ + allowUnknownMessageConsumer: messageAttachment && !file.mimetype.startsWith('audio/'), + contextEnabled, + }); + + if (destination.rejection === 'no-consumer') { + throw new Error( + `Files of type ${file.mimetype} are not sent to the model here, and this conversation has no agent whose tools could read them. Attach it to an agent with the code interpreter or file search enabled, or upload a supported file type.`, + ); + } + if (destination.rejection === 'context-disabled') { + throw new Error( + `Files of type ${file.mimetype} are saved to an agent as extracted text, and the context capability is disabled. Enable it for Agents, or attach the file to a message instead.`, + ); + } + if (destination.rejection === 'no-agent-resource') { + throw new Error( + `Files of type ${file.mimetype} cannot be saved to an agent on their own. Attach the file to a message, or enable the code interpreter or file search so the agent has somewhere to keep it.`, + ); + } + effectiveToolResource = destination.toolResource; + + if (effectiveToolResource && !canToolResourceConsume(effectiveToolResource, file.mimetype)) { + throw new Error( + `Files of type ${file.mimetype} cannot be read by ${TOOL_RESOURCE_LABELS[effectiveToolResource] ?? effectiveToolResource}.`, + ); } if (!messageAttachment && !agent_id) { @@ -712,70 +876,40 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { const retentionExpiryPromise = getAgentFileRetentionExpiry({ req, messageAttachment, - tool_resource, + tool_resource: effectiveToolResource, }); const isImage = file.mimetype.startsWith('image'); let fileInfoMetadata; const entity_id = messageAttachment === true ? undefined : agent_id; const basePath = mime.getType(file.originalname)?.startsWith('image') ? 'images' : 'uploads'; - if (tool_resource === EToolResources.execute_code) { + let shouldUploadToCodeEnv = effectiveToolResource === EToolResources.execute_code; + if (effectiveToolResource === EToolResources.execute_code) { const isCodeEnabled = (await checkCapability(req, AgentCapabilities.execute_code)) && - (await resolveToolRoleGrants({ req, getRoleByName: db.getRoleByName, context: 'fileUpload' })) - .runCode; + toolRoleGrants?.runCode === true; if (!isCodeEnabled) { throw new Error('Code execution is not enabled for Agents'); } - const { handleFileUpload: uploadCodeEnvFile } = getStrategyFunctions(FileSources.execute_code); - const stream = fs.createReadStream(file.path); - /* Resource identity for codeapi's sessionKey: - * - chat attachments (messageAttachment=true): `kind: 'user'`, codeapi - * buckets under `:user:` regardless of `id`. - * - agent setup files (messageAttachment=false): `kind: 'agent'`, shared - * per agent identity. `id` carries the agent id. */ - const codeKind = messageAttachment === true ? 'user' : 'agent'; - const codeId = messageAttachment === true ? req.user.id : agent_id; - /* Upload under the same sanitized filename LC stores in its DB - * (`fileInfo.filename` below uses `sanitizeFilename(originalname)`). - * Codeapi/file_server use this as the on-disk name in the sandbox - * — `/mnt/data/` — and `primeFiles`'s `toolContext` text - * + `_injected_files.name` both reference `file.filename`. Sending - * the unsanitized `file.originalname` here makes the sandbox path - * (with spaces / special chars) drift from what LC tells the model - * is available, causing FileNotFoundError on the first reference. */ - const sandboxFilename = sanitizeFilename(file.originalname); - const uploaded = await uploadCodeEnvFile({ - req, - stream, - filename: sandboxFilename, - kind: codeKind, - id: codeId, - }); - /* Persist under the structured `codeEnvRef` shape — the only key the - * post-cutover schema (`metadata.codeEnvRef`) and downstream readers - * (`primeFiles`, `getCodeFilesByIds`, `categorizeFileForToolResources`, - * controller filtering) accept. Storing under the legacy - * `fileIdentifier` key would be silently dropped by mongoose strict - * mode and the file would lose its sandbox reference on subsequent - * priming turns. */ - fileInfoMetadata = mergeCodeEnvRef(undefined, { - kind: codeKind, - id: codeId, - storage_session_id: uploaded.storage_session_id, - file_id: uploaded.file_id, - executionProfile: 'default', - }); - } else if (tool_resource === EToolResources.file_search) { + /* Only an explicit choice uploads here. A promoted destination has no user decision + * behind it and the agent's code deployment is resolved per turn, so uploading now + * would name the default route and be uploaded again at execution, or fail outright + * where only a stateful deployment exists. Deferred provisioning does it with the + * route the turn actually runs on. */ + if (tool_resource == null) { + shouldUploadToCodeEnv = false; + } + } + + if (effectiveToolResource === EToolResources.file_search) { const isFileSearchEnabled = (await checkCapability(req, AgentCapabilities.file_search)) && - (await resolveToolRoleGrants({ req, getRoleByName: db.getRoleByName, context: 'fileUpload' })) - .fileSearch; + toolRoleGrants?.fileSearch === true; if (!isFileSearchEnabled) { throw new Error('File search is not enabled for Agents'); } // Note: File search processing continues to dual storage logic below - } else if (tool_resource === EToolResources.context) { + } else if (effectiveToolResource === EToolResources.context) { const { file_id, temp_file_id = null } = metadata; const getExtractionLogDetails = (error) => getFileExtractionLogDetails({ @@ -789,19 +923,10 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { /** * @param {object} params * @param {string} params.text - * @param {number} params.bytes - * @param {string} params.filepath - * @param {string} params.type * @param {boolean} params.isTranscript * @return {Promise} */ - const createTextFile = async ({ - text, - bytes, - filepath, - type = 'text/plain', - isTranscript = false, - }) => { + const createTextFile = async ({ text, isTranscript = false }) => { if (!isTranscript) { assertExtractedTextInspectable({ filters: appConfig?.filters, @@ -839,6 +964,19 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { return; } } + const isImageFile = file.mimetype.startsWith('image'); + const source = getFileStrategy(appConfig, { isImage: isImageFile }); + const { handleFileUpload } = getStrategyFunctions(source); + const sanitizedUploadFn = createSanitizedUploadWrapper(handleFileUpload); + const storageResult = await sanitizedUploadFn({ + req, + file, + file_id, + basePath, + entity_id, + }); + const { bytes, filename, filepath, embedded, height, width } = storageResult; + const retentionExpiry = await retentionExpiryPromise; const fileInfo = { ...removeNullishValues({ @@ -847,22 +985,27 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { file_id, temp_file_id, user: req.user.id, - type, - filepath: filepath ?? file.path, - source: FileSources.text, - filename: file.originalname, + type: file.mimetype, + filepath, + source, + filename: filename ?? sanitizeFilename(file.originalname), model: messageAttachment ? undefined : req.body.model, context: messageAttachment ? FileContext.message_attachment : FileContext.agents, tenantId: req.user.tenantId, + embedded, + height, + width, + llmDeliveryPath: 'text', }), + metadata: uploadChoiceMetadata, ...retentionExpiry, }; - if (!messageAttachment && tool_resource) { + if (!messageAttachment && effectiveToolResource) { await db.addAgentResourceFile({ file_id, agent_id, - tool_resource, + tool_resource: effectiveToolResource, updatingUserId: req?.user?.id, }); } @@ -873,7 +1016,7 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { const fileConfig = mergeFileConfig(appConfig.fileConfig); const extractedTextPlan = getUploadExtractedTextPlan({ endpoint: metadata.endpoint, - toolResource: tool_resource, + toolResource: effectiveToolResource, mimeType: file.mimetype, fileConfig, ocrConfigured: appConfig?.ocr != null, @@ -923,8 +1066,8 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { extract: resolveDocumentText, }); if (ocrResult) { - const { text, bytes, filepath: ocrFileURL } = ocrResult; - return await createTextFile({ text, bytes, filepath: ocrFileURL }); + const { text } = ocrResult; + return await createTextFile({ text }); } throw new Error( `Unable to extract text from "${file.originalname}". The document may be image-based and requires an OCR service to process.`, @@ -938,8 +1081,8 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { if (shouldUseSTT) { const sttService = await STTService.getInstance(); - const { text, bytes } = await processAudioFile({ req, file, sttService }); - return await createTextFile({ text, bytes, type: file.mimetype, isTranscript: true }); + const { text } = await processAudioFile({ req, file, sttService }); + return await createTextFile({ text, isTranscript: true }); } const shouldUseText = fileConfig.checkType( @@ -977,29 +1120,40 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { `Unable to extract text from "${file.originalname}". RAG text extraction was unavailable and the built-in parser produced no result.`, ); } - const { text, bytes, filepath: docFileURL } = documentText; - return await createTextFile({ text, bytes, filepath: docFileURL }); + const { text } = documentText; + return await createTextFile({ text }); } - return await createTextFile({ - text: configuredText.text, - bytes: configuredText.bytes, - type: file.mimetype, - }); + return await createTextFile({ text: configuredText.text }); } - const { text, bytes } = await extractInspectableFileText({ + /* The native reader decodes whatever bytes it is given as UTF-8, which is meaningful + * only for types that are already text. For anything else, a raster image on a + * deployment without OCR being the case in point, it would store mojibake as the + * file's text, so a real extractor is required and its absence surfaces as an error + * rather than as nonsense content. */ + const { text } = await extractInspectableFileText({ filters: appConfig?.filters, - extract: () => parseText({ req, file, file_id }), + extract: () => + parseText({ + req, + file, + file_id, + allowNativeFallback: isNativelyReadableText(file.mimetype), + }), }); - return await createTextFile({ text, bytes, type: file.mimetype }); + return await createTextFile({ text }); } // Dual storage pattern for RAG files: Storage + Vector DB let storageResult, embeddingResult; + let storedType = file.mimetype; const isImageFile = file.mimetype.startsWith('image'); const source = getFileStrategy(appConfig, { isImage: isImageFile }); - if (tool_resource === EToolResources.file_search) { + if ( + effectiveToolResource === EToolResources.file_search && + tool_resource === EToolResources.file_search + ) { // FIRST: Upload to Storage for permanent backup (S3/local/etc.) const { handleFileUpload } = getStrategyFunctions(source); const sanitizedUploadFn = createSanitizedUploadWrapper(handleFileUpload); @@ -1021,8 +1175,30 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { entity_id, }); - // Vector status will be stored at root level, no need for metadata - fileInfoMetadata = {}; + /* Vectors live under the entity that embedded them, and priming asks which namespaces + * hold them rather than reading the root flag. Omitting it here re-embeds the file on + * the first search, and aborts that search if RAG is briefly unavailable. */ + fileInfoMetadata = entity_id != null ? { embeddedEntities: [entity_id] } : {}; + } else if (isImage) { + /* The conversion is this file's storage step. Uploading the original first left a + * second object nothing references, and the record's size and dimensions describing + * bytes that were replaced. Only the storage fields are kept: the record below is + * built here, and its filename goes through the sanitizer. */ + const converted = await processImageFile({ + req, + file, + metadata: { file_id }, + returnFile: true, + }); + storedType = converted.type ?? storedType; + storageResult = { + bytes: converted.bytes, + filepath: converted.filepath, + storageKey: converted.storageKey, + storageRegion: converted.storageRegion, + height: converted.height, + width: converted.width, + }; } else { // Standard single storage for non-RAG files const { handleFileUpload } = getStrategyFunctions(source); @@ -1036,6 +1212,57 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { }); } + if (shouldUploadToCodeEnv) { + const { handleFileUpload: uploadCodeEnvFile } = getStrategyFunctions(FileSources.execute_code); + const { getDownloadStream } = getStrategyFunctions(source); + if (!getDownloadStream && source !== FileSources.local) { + throw new Error(`No download stream available for ${source} storage`); + } + /* Upload from the persisted representation. Images have already been converted here, + * so both the bytes and extension advertised to the sandbox describe the same file. */ + const downloadPath = storageResult.storageKey ?? storageResult.filepath; + const stream = getDownloadStream + ? await getDownloadStream(req, downloadPath) + : fs.createReadStream(file.path); + const codeKind = messageAttachment === true ? 'user' : 'agent'; + const codeId = messageAttachment === true ? req.user.id : agent_id; + const sandboxFilename = resolveSandboxFilename(sanitizeFilename(file.originalname), storedType); + let uploaded; + try { + uploaded = await uploadCodeEnvFile({ + req, + stream, + filename: sandboxFilename, + kind: codeKind, + id: codeId, + }); + } catch (error) { + const { deleteFile } = getStrategyFunctions(source); + if (deleteFile) { + try { + await deleteFile(req, { + file_id, + filepath: storageResult.filepath, + storageKey: storageResult.storageKey, + storageRegion: storageResult.storageRegion, + source, + }); + } catch (cleanupError) { + logger.error('[processAgentFileUpload] Failed to clean up stored file', cleanupError); + } + } + throw error; + } + fileInfoMetadata = mergeCodeEnvRef(undefined, { + kind: codeKind, + id: codeId, + storage_session_id: uploaded.storage_session_id, + file_id: uploaded.file_id, + executionProfile: 'default', + provisionedAt: Date.now(), + }); + } + let { bytes, filename, @@ -1047,7 +1274,10 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { } = storageResult; // For RAG files, use embedding result; for others, use storage result let embedded = storageResult.embedded; - if (tool_resource === EToolResources.file_search) { + if ( + effectiveToolResource === EToolResources.file_search && + tool_resource === EToolResources.file_search + ) { embedded = embeddingResult?.embedded; filename = embeddingResult?.filename || filename; } @@ -1060,31 +1290,15 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { storageRegion: _storageRegion, }); - if (!messageAttachment && tool_resource) { + if (!messageAttachment && effectiveToolResource) { await db.addAgentResourceFile({ file_id, agent_id, - tool_resource, + tool_resource: effectiveToolResource, updatingUserId: req?.user?.id, }); } - if (isImage) { - const result = await processImageFile({ - req, - file, - metadata: { file_id: v4() }, - returnFile: true, - }); - filepath = result.filepath; - storageMetadata = getStorageMetadata({ - filepath, - source: result.source, - storageKey: result.storageKey, - storageRegion: result.storageRegion, - }); - } - const retentionExpiry = await retentionExpiryPromise; const fileInfo = { ...removeNullishValues({ @@ -1097,13 +1311,21 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { filename: filename ?? sanitizeFilename(file.originalname), context: messageAttachment ? FileContext.message_attachment : FileContext.agents, model: messageAttachment ? undefined : req.body.model, - metadata: fileInfoMetadata, - type: file.mimetype, + metadata: { + ...(fileInfoMetadata ?? {}), + ...uploadChoiceMetadata, + /* The route was resolved against the upload's own type, and delivery re-resolves + * it later. Conversion changes `type`, so without this the second answer is drawn + * from a format the administrator never configured a route for. */ + ...(storedType !== file.mimetype ? { routingMimeType: file.mimetype } : {}), + }, + type: storedType, embedded, source, height, width, tenantId: req.user.tenantId, + llmDeliveryPath, }), ...retentionExpiry, }; @@ -1403,9 +1625,30 @@ async function saveBase64Image( * * @throws {Error} If a file exception is caught (invalid file size or type, lack of metadata). */ -function filterFile({ req, image, isAvatar }) { +/** + * @param {object} params + * @param {ServerRequest} params.req + * @param {boolean} [params.image] + * @param {boolean} [params.isAvatar] + * @param {string} [params.endpoint] Effective endpoint for this upload. Agent uploads + * arrive as `agents` but route by the agent's own provider, so validation has to be + * told which configuration governs, or it admits files the provider rejects and + * rejects files the provider allows. + */ +function filterFile({ req, image, isAvatar, endpoint: endpointOverride }) { const { file } = req; - const { endpoint, endpointType, file_id, width, height } = req.body; + const { + endpoint: requestEndpoint, + endpointType: requestEndpointType, + file_id, + width, + height, + } = req.body; + const endpoint = endpointOverride ?? requestEndpoint; + /* getEndpointFileConfig consults endpointType ahead of endpoint, so a composer upload + * carrying `agents` would keep the Agents policy and shadow the provider the override + * names. The override replaces both or neither. */ + const endpointType = endpointOverride != null ? undefined : requestEndpointType; if (!file_id && !isAvatar) { throw new Error('No file_id provided'); @@ -1432,6 +1675,9 @@ function filterFile({ req, image, isAvatar }) { fileConfig, endpointType, }); + if (isAvatar !== true && endpointFileConfig?.disabled === true) { + throw new Error(`File uploads are disabled for ${endpoint} endpoint`); + } const fileSizeLimit = isAvatar === true ? fileConfig.avatarSizeLimit : endpointFileConfig.fileSizeLimit; diff --git a/api/server/services/Files/process.spec.js b/api/server/services/Files/process.spec.js index 9a54b3a3d3..76c0e2044e 100644 --- a/api/server/services/Files/process.spec.js +++ b/api/server/services/Files/process.spec.js @@ -174,6 +174,7 @@ jest.mock('~/models', () => ({ findFileById: jest.fn(), getConvo: jest.fn(), getExpiredFiles: jest.fn(), + getAgent: jest.fn().mockResolvedValue(null), incrementFileDeletionAttempts: jest.fn(), deferExpiredFile: jest.fn(), addAgentResourceFile: jest.fn().mockResolvedValue({}), @@ -215,6 +216,10 @@ jest.mock('~/server/services/Files/Audio/STTService', () => ({ STTService: { getInstance: jest.fn() }, })); +jest.mock('./VectorDB/crud', () => ({ + uploadVectors: jest.fn().mockResolvedValue({ embedded: true, filename: 'embedded-upload.bin' }), +})); + const { getRetentionExpiry, getAgentFileRetentionExpiry, @@ -222,6 +227,7 @@ const { startExpiredFileSweep: startExpiredFileSweepWithDeps, } = require('@librechat/api'); const { + EModelEndpoint, EToolResources, FileSources, FileContext, @@ -230,19 +236,21 @@ const { } = require('librechat-data-provider'); const { mergeFileConfig } = require('librechat-data-provider'); const { checkCapability } = require('~/server/services/Config'); +const { loadAuthValues } = require('~/server/services/Tools/credentials'); const { getStrategyFunctions } = require('~/server/services/Files/strategies'); const { uploadVectors } = require('./VectorDB/crud'); const { logger } = require('@librechat/data-schemas'); const db = require('~/models'); const { processAgentFileUpload, + processImageFile, processDeleteRequest, processFileUpload, processFileURL, - processImageFile, retrieveAndProcessFile, sweepExpiredFiles, startExpiredFileSweep, + filterFile, } = require('./process'); const { inspectContent, @@ -279,6 +287,7 @@ const makeReq = ({ ocrConfig = null, interfaceConfig, filters, + speech, body, } = {}) => ({ user: { id: 'user-123', tenantId: 'tenant-a' }, @@ -292,7 +301,9 @@ const makeReq = ({ config: { fileConfig: {}, fileStrategy: 'local', + imageOutputType: 'webp', ocr: ocrConfig, + ...(speech ? { speech } : {}), ...(filters ? { filters } : {}), ...(interfaceConfig ? { interfaceConfig } : {}), }, @@ -422,6 +433,8 @@ describe('processAgentFileUpload', () => { mockRes.status.mockReturnThis(); mockRes.json.mockReturnValue({}); checkCapability.mockResolvedValue(true); + loadAuthValues.mockResolvedValue({ CODE_API_KEY: 'code-key' }); + uploadVectors.mockResolvedValue({ embedded: true, filename: 'embedded-upload.bin' }); getStrategyFunctions.mockReturnValue({ handleFileUpload: jest .fn() @@ -527,7 +540,7 @@ describe('processAgentFileUpload', () => { expect(db.createFile).toHaveBeenCalledWith( expect.objectContaining({ type: 'audio/webm', - source: FileSources.text, + llmDeliveryPath: 'text', text: 'submitted transcript', }), true, @@ -640,7 +653,11 @@ describe('processAgentFileUpload', () => { test('throws when configured OCR capability is not enabled for the agent', async () => { mergeFileConfig.mockReturnValue(makeFileConfig({ ocrSupportedMimeTypes: [PDF_MIME] })); - checkCapability.mockResolvedValue(false); + /* Only OCR is under test here; disabling every capability would trip the + * separate context guard first. */ + checkCapability.mockImplementation( + async (_req, capability) => capability !== AgentCapabilities.ocr, + ); const req = makeReq({ mimetype: PDF_MIME, ocrConfig: { strategy: FileSources.mistral_ocr }, @@ -652,7 +669,11 @@ describe('processAgentFileUpload', () => { }); test('uses document_parser (no capability check) when OCR capability returns false but no OCR config', async () => { - checkCapability.mockResolvedValue(false); + /* Only OCR is under test here; disabling every capability would trip the + * separate context guard first. */ + checkCapability.mockImplementation( + async (_req, capability) => capability !== AgentCapabilities.ocr, + ); const req = makeReq({ mimetype: PDF_MIME, ocrConfig: null }); await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); @@ -1271,11 +1292,165 @@ describe('processAgentFileUpload', () => { getStrategyFunctions.mockImplementation((src) => src === FileSources.execute_code ? { handleFileUpload: codeEnvUpload } - : { handleFileUpload: localUpload, saveBuffer: jest.fn() }, + : { + handleFileUpload: localUpload, + getDownloadStream: jest.fn(async () => Readable.from(Buffer.from('stored'))), + saveBuffer: jest.fn(), + }, ); return codeEnvUpload; }; + const agentsZipReq = () => { + const req = makeReq({ mimetype: 'application/zip', ocrConfig: null }); + req.body.endpoint = EModelEndpoint.agents; + return req; + }; + + it('leaves a promoted code destination for deferred provisioning', async () => { + /* No explicit choice stands behind a promotion, and the agent's code deployment is + * resolved per turn, so uploading now names the default route and has to be + * uploaded again where the turn actually runs. */ + const codeEnvUpload = setupCodeEnvUpload({ storage_session_id: 'sess-x', file_id: 'fid-x' }); + + await processAgentFileUpload({ + req: agentsZipReq(), + res: mockRes, + metadata: { + agent_id: 'agent-abc', + file_id: 'file-promoted', + agentTools: [EToolResources.execute_code], + }, + }).catch(() => {}); + + expect(codeEnvUpload).not.toHaveBeenCalled(); + expect(db.createFile).toHaveBeenCalledWith( + expect.objectContaining({ + metadata: expect.not.objectContaining({ codeEnvRef: expect.anything() }), + }), + true, + ); + }); + + it('still uploads eagerly when the user chose the code destination', async () => { + const codeEnvUpload = setupCodeEnvUpload({ storage_session_id: 'sess-y', file_id: 'fid-y' }); + + await processAgentFileUpload({ + req: agentsZipReq(), + res: mockRes, + metadata: { + agent_id: 'agent-abc', + tool_resource: EToolResources.execute_code, + file_id: 'file-chosen', + }, + }).catch(() => {}); + + expect(codeEnvUpload).toHaveBeenCalled(); + }); + + it('defers an inferred file-search destination until tool execution', async () => { + const { uploadVectors } = require('~/server/services/Files/VectorDB/crud'); + setupStoredFileUpload(); + const req = makeReq({ + mimetype: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + ocrConfig: null, + }); + req.body.endpoint = EModelEndpoint.agents; + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + agent_id: 'agent-abc', + file_id: 'file-inferred-search', + agentTools: [EToolResources.file_search], + }, + }); + + expect(uploadVectors).not.toHaveBeenCalled(); + expect(db.createFile).toHaveBeenCalledWith( + expect.objectContaining({ + llmDeliveryPath: 'none', + metadata: expect.not.objectContaining({ embeddedEntities: expect.anything() }), + }), + true, + ); + }); + + it('uploads converted image bytes under the converted extension', async () => { + const codeEnvUpload = jest + .fn() + .mockResolvedValue({ storage_session_id: 'sess-image', file_id: 'fid-image' }); + const convertedStream = Readable.from(Buffer.from('webp')); + getStrategyFunctions.mockImplementation((src) => + src === FileSources.execute_code + ? { handleFileUpload: codeEnvUpload } + : { + handleImageUpload: jest.fn().mockResolvedValue({ + filepath: '/images/photo.webp', + bytes: 4, + width: 10, + height: 10, + }), + getDownloadStream: jest.fn().mockResolvedValue(convertedStream), + }, + ); + const req = makeReq({ mimetype: 'image/jpeg', ocrConfig: null }); + req.file.originalname = 'photo.jpg'; + req.body.endpoint = EModelEndpoint.agents; + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + agent_id: 'agent-abc', + tool_resource: EToolResources.execute_code, + file_id: 'file-image', + }, + }); + + expect(codeEnvUpload).toHaveBeenCalledWith( + expect.objectContaining({ filename: 'photo.webp', stream: convertedStream }), + ); + }); + + it('removes durable storage when an eager code upload fails', async () => { + const deleteFile = jest.fn().mockResolvedValue(undefined); + getStrategyFunctions.mockImplementation((src) => + src === FileSources.execute_code + ? { handleFileUpload: jest.fn().mockRejectedValue(new Error('code unavailable')) } + : { + handleFileUpload: jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/stored.bin', + bytes: 4, + }), + getDownloadStream: jest.fn(async () => Readable.from(Buffer.from('stored'))), + deleteFile, + }, + ); + + await expect( + processAgentFileUpload({ + req: agentsZipReq(), + res: mockRes, + metadata: { + agent_id: 'agent-abc', + tool_resource: EToolResources.execute_code, + file_id: 'file-cleanup', + }, + }), + ).rejects.toThrow('code unavailable'); + + expect(deleteFile).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + file_id: 'file-cleanup', + filepath: '/uploads/user-123/stored.bin', + }), + ); + expect(db.createFile).not.toHaveBeenCalled(); + }); + it('persists kind:user codeEnvRef for chat attachments (messageAttachment=true)', async () => { setupCodeEnvUpload({ storage_session_id: 'sess-1', file_id: 'fid-1' }); const req = makeReq(); @@ -1292,13 +1467,14 @@ describe('processAgentFileUpload', () => { expect(db.createFile).toHaveBeenCalledWith( expect.objectContaining({ - metadata: { + metadata: expect.objectContaining({ codeEnvRef: { kind: 'user', id: 'user-123', storage_session_id: 'sess-1', file_id: 'fid-1', executionProfile: 'default', + provisionedAt: expect.any(Number), }, codeEnvRefs: { default: { @@ -1307,9 +1483,10 @@ describe('processAgentFileUpload', () => { storage_session_id: 'sess-1', file_id: 'fid-1', executionProfile: 'default', + provisionedAt: expect.any(Number), }, }, - }, + }), }), true, ); @@ -1330,13 +1507,14 @@ describe('processAgentFileUpload', () => { expect(db.createFile).toHaveBeenCalledWith( expect.objectContaining({ - metadata: { + metadata: expect.objectContaining({ codeEnvRef: { kind: 'agent', id: 'agent-abc', storage_session_id: 'sess-2', file_id: 'fid-2', executionProfile: 'default', + provisionedAt: expect.any(Number), }, codeEnvRefs: { default: { @@ -1345,9 +1523,10 @@ describe('processAgentFileUpload', () => { storage_session_id: 'sess-2', file_id: 'fid-2', executionProfile: 'default', + provisionedAt: expect.any(Number), }, }, - }, + }), }), true, ); @@ -1400,13 +1579,14 @@ describe('processAgentFileUpload', () => { expect.objectContaining({ expiredAt, context: FileContext.agents, - metadata: { + metadata: expect.objectContaining({ codeEnvRef: { kind: 'agent', id: 'agent-abc', storage_session_id: 'sess-5', file_id: 'fid-5', executionProfile: 'default', + provisionedAt: expect.any(Number), }, codeEnvRefs: { default: { @@ -1415,9 +1595,10 @@ describe('processAgentFileUpload', () => { storage_session_id: 'sess-5', file_id: 'fid-5', executionProfile: 'default', + provisionedAt: expect.any(Number), }, }, - }, + }), }), true, ); @@ -1447,6 +1628,555 @@ describe('processAgentFileUpload', () => { expect(persisted.metadata).not.toHaveProperty('fileIdentifier'); }); }); + + describe('text delivery storage', () => { + test('stores the original file durably for plain text delivery records', async () => { + const { parseText } = require('@librechat/api'); + const { createFile } = require('~/models'); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 128, + filename: 'upload.bin', + embedded: false, + }); + mergeFileConfig.mockReturnValue(makeFileConfig({ textSupportedMimeTypes: ['text/plain'] })); + parseText.mockResolvedValueOnce({ text: 'plain extracted text', bytes: 20 }); + getStrategyFunctions.mockReturnValue({ handleFileUpload: storageUpload }); + const req = makeReq({ mimetype: 'text/plain', ocrConfig: null }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(storageUpload).toHaveBeenCalledWith( + expect.objectContaining({ + file_id: 'file-uuid-123', + file: expect.objectContaining({ originalname: 'upload.bin' }), + }), + ); + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'plain extracted text', + bytes: 128, + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + source: FileSources.local, + filename: 'upload.bin', + type: 'text/plain', + llmDeliveryPath: 'text', + }), + true, + ); + }); + + test('stores the original file durably for OCR delivery records', async () => { + const { createFile } = require('~/models'); + const documentUpload = jest.fn().mockResolvedValue({ + text: 'ocr extracted text', + bytes: 42, + filepath: 'document_parser', + }); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 4096, + filename: 'upload.bin', + embedded: false, + }); + getStrategyFunctions.mockImplementation((source) => { + if (source === FileSources.document_parser) { + return { handleFileUpload: documentUpload }; + } + return { handleFileUpload: storageUpload }; + }); + const req = makeReq({ mimetype: PDF_MIME, ocrConfig: null }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(documentUpload).toHaveBeenCalled(); + expect(storageUpload).toHaveBeenCalled(); + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'ocr extracted text', + bytes: 4096, + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + source: FileSources.local, + filename: 'upload.bin', + type: PDF_MIME, + llmDeliveryPath: 'text', + }), + true, + ); + }); + }); + + describe('explicit legacy tool delivery path', () => { + test('persists llmDeliveryPath none for explicit file_search uploads', async () => { + const { createFile } = require('~/models'); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 128, + filename: 'upload.bin', + embedded: false, + }); + getStrategyFunctions.mockReturnValue({ handleFileUpload: storageUpload }); + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + defaultLLMDeliveryPath: { + fallback: 'text', + }, + }); + const req = makeReq({ mimetype: 'text/markdown', ocrConfig: null }); + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + ...makeMetadata(), + tool_resource: EToolResources.file_search, + }, + }); + + expect(checkCapability).toHaveBeenCalledWith( + expect.anything(), + AgentCapabilities.file_search, + ); + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + source: FileSources.local, + type: 'text/markdown', + embedded: true, + llmDeliveryPath: 'none', + }), + true, + ); + }); + + test('persists llmDeliveryPath provider for legacy provider uploads without tool_resource', async () => { + const { createFile } = require('~/models'); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 128, + filename: 'upload.bin', + embedded: false, + }); + getStrategyFunctions.mockReturnValue({ handleFileUpload: storageUpload }); + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + endpoints: { + [EModelEndpoint.agents]: { legacyFileUploadUX: true }, + }, + defaultLLMDeliveryPath: { + fallback: 'none', + }, + }); + const req = makeReq({ mimetype: 'text/markdown', ocrConfig: null }); + req.body.endpoint = EModelEndpoint.agents; + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + agent_id: 'agent-abc', + message_file: 'true', + file_id: 'file-uuid-123', + }, + }); + + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + source: FileSources.local, + type: 'text/markdown', + llmDeliveryPath: 'provider', + }), + true, + ); + }); + + test('rejects ordinary chat audio when no speech provider can consume it', async () => { + /* Transcription needs exactly one non-empty provider block. A schema holding only + * allowedAddresses reports STT present while the service refuses it, so routing + * audio to text there sends the upload to a transcription that cannot run. */ + const { createFile } = require('~/models'); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 128, + filename: 'upload.bin', + embedded: false, + }); + getStrategyFunctions.mockReturnValue({ handleFileUpload: storageUpload }); + mergeFileConfig.mockReturnValue(makeFileConfig()); + const req = makeReq({ + mimetype: 'audio/mpeg', + ocrConfig: null, + speech: { stt: { allowedAddresses: ['127.0.0.1'] } }, + }); + req.body.endpoint = EModelEndpoint.openAI; + + const error = await processAgentFileUpload({ + req, + res: mockRes, + metadata: { agent_id: 'agent-abc', message_file: 'true', file_id: 'file-uuid-123' }, + }).catch((thrown) => thrown); + + expect(error).toEqual( + expect.objectContaining({ message: expect.stringMatching(/no agent/i) }), + ); + expect(createFile).not.toHaveBeenCalled(); + }); + + test('retains an auto-routed context upload as an agent resource', async () => { + const { getAgentFileRetentionExpiry } = require('@librechat/api'); + mergeFileConfig.mockReturnValue(makeFileConfig()); + const req = makeReq({ mimetype: DOCX_MIME, ocrConfig: null }); + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { agent_id: 'agent-abc', file_id: 'file-uuid-123' }, + }).catch(() => {}); + + expect(getAgentFileRetentionExpiry).toHaveBeenCalledWith( + expect.objectContaining({ toolResource: EToolResources.context }), + expect.any(Object), + ); + }); + + test('plans extraction with the promoted context resource for auto-routed text uploads', async () => { + const { getUploadExtractedTextPlan } = require('@librechat/api'); + mergeFileConfig.mockReturnValue(makeFileConfig()); + const req = makeReq({ mimetype: DOCX_MIME, ocrConfig: null }); + + /** The planner runs before extraction; downstream extraction is covered by the + * OCR strategy tests, so failures past this point must not mask the argument. */ + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { agent_id: 'agent-abc', file_id: 'file-uuid-123' }, + }).catch(() => {}); + + expect(getUploadExtractedTextPlan).toHaveBeenCalledWith( + expect.objectContaining({ toolResource: EToolResources.context }), + ); + }); + + test('routes under the provider endpoint the caller resolved', async () => { + /* The route resolves the agent's provider once, before validation, and hands it + * down so acceptance and routing use one configuration. */ + const { createFile } = require('~/models'); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 128, + filename: 'upload.bin', + embedded: false, + }); + getStrategyFunctions.mockReturnValue({ handleFileUpload: storageUpload }); + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + endpoints: { + 'Custom Provider': { defaultLLMDeliveryPath: { fallback: 'none' } }, + }, + }); + const req = makeReq({ mimetype: 'text/markdown', ocrConfig: null }); + req.body.endpoint = EModelEndpoint.agents; + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + agent_id: 'agent-abc', + message_file: 'true', + file_id: 'file-uuid-123', + effectiveEndpoint: 'Custom Provider', + }, + }); + + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ llmDeliveryPath: 'none' }), + true, + ); + }); + + test('persists llmDeliveryPath none for explicit execute_code uploads', async () => { + const { createFile } = require('~/models'); + const codeUpload = jest + .fn() + .mockResolvedValue({ storage_session_id: 'sess-csv', file_id: 'fid-csv' }); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 128, + filename: 'upload.bin', + embedded: false, + }); + getStrategyFunctions.mockImplementation((source) => { + if (source === FileSources.execute_code) { + return { handleFileUpload: codeUpload }; + } + return { handleFileUpload: storageUpload }; + }); + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + defaultLLMDeliveryPath: { + fallback: 'text', + }, + }); + const req = makeReq({ mimetype: 'text/csv', ocrConfig: null }); + req.file.path = __filename; + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + ...makeMetadata(), + tool_resource: EToolResources.execute_code, + }, + }); + + expect(checkCapability).toHaveBeenCalledWith( + expect.anything(), + AgentCapabilities.execute_code, + ); + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + source: FileSources.local, + type: 'text/csv', + metadata: expect.objectContaining({ + codeEnvRef: { + kind: 'agent', + id: 'agent-abc', + storage_session_id: 'sess-csv', + file_id: 'fid-csv', + executionProfile: 'default', + provisionedAt: expect.any(Number), + }, + codeEnvRefs: { + default: { + kind: 'agent', + id: 'agent-abc', + storage_session_id: 'sess-csv', + file_id: 'fid-csv', + executionProfile: 'default', + provisionedAt: expect.any(Number), + }, + }, + }), + llmDeliveryPath: 'none', + }), + true, + ); + }); + + test('persists llmDeliveryPath text for explicit context uploads', async () => { + const { parseText } = require('@librechat/api'); + const { createFile } = require('~/models'); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 128, + filename: 'upload.bin', + embedded: false, + }); + getStrategyFunctions.mockReturnValue({ handleFileUpload: storageUpload }); + mergeFileConfig.mockReturnValue( + makeFileConfig({ textSupportedMimeTypes: ['text/markdown'] }), + ); + parseText.mockResolvedValueOnce({ text: 'markdown text', bytes: 13 }); + const req = makeReq({ mimetype: 'text/markdown', ocrConfig: null }); + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + ...makeMetadata(), + tool_resource: EToolResources.context, + }, + }); + + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'markdown text', + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + source: FileSources.local, + type: 'text/markdown', + llmDeliveryPath: 'text', + }), + true, + ); + }); + + test('normalizes explicit ocr uploads to context text delivery', async () => { + const { parseText } = require('@librechat/api'); + const { createFile, addAgentResourceFile } = require('~/models'); + const storageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/user-123/file-uuid-123__upload.bin', + bytes: 128, + filename: 'upload.bin', + embedded: false, + }); + getStrategyFunctions.mockReturnValue({ handleFileUpload: storageUpload }); + mergeFileConfig.mockReturnValue( + makeFileConfig({ textSupportedMimeTypes: ['text/markdown'] }), + ); + parseText.mockResolvedValueOnce({ text: 'markdown text', bytes: 13 }); + const req = makeReq({ mimetype: 'text/markdown', ocrConfig: null }); + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + ...makeMetadata(), + tool_resource: EToolResources.ocr, + }, + }); + + expect(addAgentResourceFile).toHaveBeenCalledWith( + expect.objectContaining({ + file_id: 'file-uuid-123', + agent_id: 'agent-abc', + tool_resource: EToolResources.context, + }), + ); + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'markdown text', + source: FileSources.local, + type: 'text/markdown', + llmDeliveryPath: 'text', + }), + true, + ); + }); + }); +}); + +describe('processImageFile', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockRes.status.mockReturnThis(); + mockRes.json.mockReturnValue({}); + mergeFileConfig.mockReturnValue(makeFileConfig()); + }); + + test('persists resolved llmDeliveryPath for image uploads', async () => { + const { createFile } = require('~/models'); + const handleImageUpload = jest.fn().mockResolvedValue({ + filepath: '/images/user-123/image.webp', + bytes: 256, + width: 100, + height: 80, + }); + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + defaultLLMDeliveryPath: { + overrides: { 'image/*': 'none' }, + }, + }); + getStrategyFunctions.mockReturnValue({ handleImageUpload }); + const req = makeReq({ mimetype: 'image/png', ocrConfig: null }); + + await processImageFile({ + req, + res: mockRes, + metadata: { + file_id: 'image-file-id', + temp_file_id: 'temp-image-file-id', + endpoint: EModelEndpoint.agents, + }, + }); + + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + file_id: 'image-file-id', + temp_file_id: 'temp-image-file-id', + filepath: '/images/user-123/image.webp', + source: FileSources.local, + type: 'image/webp', + llmDeliveryPath: 'none', + }), + true, + ); + }); + + test('persists provider llmDeliveryPath for legacy image provider uploads', async () => { + const { createFile } = require('~/models'); + const handleImageUpload = jest.fn().mockResolvedValue({ + filepath: '/images/user-123/image.webp', + bytes: 256, + width: 100, + height: 80, + }); + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + endpoints: { + [EModelEndpoint.agents]: { legacyFileUploadUX: true }, + }, + defaultLLMDeliveryPath: { + overrides: { 'image/*': 'none' }, + }, + }); + getStrategyFunctions.mockReturnValue({ handleImageUpload }); + const req = makeReq({ mimetype: 'image/png', ocrConfig: null }); + req.body.endpoint = EModelEndpoint.agents; + + await processImageFile({ + req, + res: mockRes, + metadata: { + file_id: 'image-file-id', + temp_file_id: 'temp-image-file-id', + endpoint: EModelEndpoint.agents, + }, + }); + + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ + file_id: 'image-file-id', + temp_file_id: 'temp-image-file-id', + filepath: '/images/user-123/image.webp', + source: FileSources.local, + type: 'image/webp', + llmDeliveryPath: 'provider', + }), + true, + ); + }); + + test('routes an agent image under the provider endpoint the caller resolved', async () => { + const { createFile } = require('~/models'); + const handleImageUpload = jest.fn().mockResolvedValue({ + filepath: '/images/user-123/image.webp', + bytes: 256, + width: 100, + height: 80, + }); + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + endpoints: { + 'Custom Provider': { defaultLLMDeliveryPath: { overrides: { 'image/*': 'none' } } }, + }, + }); + getStrategyFunctions.mockReturnValue({ handleImageUpload }); + const req = makeReq({ mimetype: 'image/png', ocrConfig: null }); + + await processImageFile({ + req, + res: mockRes, + metadata: { + file_id: 'image-file-id', + agent_id: 'agent-abc', + endpoint: EModelEndpoint.agents, + effectiveEndpoint: 'Custom Provider', + }, + }); + + /* Storage still keys off the request endpoint; only delivery routing follows the + * resolved provider. */ + expect(handleImageUpload).toHaveBeenCalledWith( + expect.objectContaining({ endpoint: EModelEndpoint.agents }), + ); + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ llmDeliveryPath: 'none' }), + true, + ); + }); }); describe('processFileURL', () => { @@ -1977,3 +2707,431 @@ describe('startExpiredFileSweep', () => { expect(interval).toBe('sweep-interval'); }); }); + +describe('uploads with no consumer on the agent record', () => { + test('accepts a type the record shows no tool for', async () => { + /* Skills contribute file search and code execution for a turn without being written + * to agent.tools, so an empty list is not evidence that nothing will read the file. + * Reaching storage, which this suite leaves unwired, proves it was not refused. */ + const req = makeReq({ mimetype: 'application/zip', ocrConfig: null }); + req.body.endpoint = EModelEndpoint.agents; + getStrategyFunctions.mockClear(); + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + agent_id: 'agent-abc', + message_file: 'true', + file_id: 'file-uuid-zip', + agentTools: [], + }, + }).catch(() => {}); + + expect(getStrategyFunctions).toHaveBeenCalled(); + }); + + test('accepts a standard-chat attachment for a tool enabled later', async () => { + const req = makeReq({ mimetype: 'application/zip', ocrConfig: null }); + req.body.endpoint = 'Custom Provider'; + getStrategyFunctions.mockClear(); + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + endpoint: 'Custom Provider', + message_file: 'true', + file_id: 'file-uuid-zip', + }, + }).catch(() => {}); + + expect(getStrategyFunctions).toHaveBeenCalled(); + }); +}); + +describe('native text fallback', () => { + /* The text matcher has to admit the type, or processing refuses before reaching the + * reader at all. This mirrors a deployment whose text config accepts these types. */ + beforeEach(() => { + mergeFileConfig.mockReturnValue( + makeFileConfig({ textSupportedMimeTypes: [/^image\/png$/, /^text\/plain$/] }), + ); + setupStoredFileUpload(); + }); + + test('does not read a raster image as text when no extractor handles it', async () => { + /* An administrator can route images to text; without OCR nothing parses them, and + * reading the bytes directly would store mojibake as the file's text. */ + const { parseText } = require('@librechat/api'); + const req = makeReq({ mimetype: 'image/png', ocrConfig: null }); + req.body.endpoint = EModelEndpoint.agents; + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + agent_id: 'agent-abc', + message_file: 'true', + file_id: 'f-png', + tool_resource: 'context', + }, + }).catch(() => {}); + + const call = parseText.mock.calls.at(-1)?.[0]; + expect(call?.allowNativeFallback).toBe(false); + }); + + test('still reads a text file directly', async () => { + const { parseText } = require('@librechat/api'); + const req = makeReq({ mimetype: 'text/plain', ocrConfig: null }); + req.body.endpoint = EModelEndpoint.agents; + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + agent_id: 'agent-abc', + message_file: 'true', + file_id: 'f-txt', + tool_resource: 'context', + }, + }).catch(() => {}); + + const call = parseText.mock.calls.at(-1)?.[0]; + expect(call?.allowNativeFallback).toBe(true); + }); +}); + +describe('permanent unified uploads and unknown tool sets', () => { + const zipReq = () => { + const req = makeReq({ mimetype: 'application/zip', ocrConfig: null }); + req.body.endpoint = EModelEndpoint.agents; + return req; + }; + + test('does not refuse when the agent tools are unknown', async () => { + /* An ephemeral agent has no record to read, so the route reports no tool set and + * processing must not conclude that nothing can consume the file. */ + const error = await processAgentFileUpload({ + req: zipReq(), + res: mockRes, + metadata: { agent_id: 'agent-abc', message_file: 'true', file_id: 'f-eph' }, + }).catch((thrown) => thrown); + + expect(String(error?.message ?? '')).not.toMatch(/code interpreter or file search/i); + }); + + test('files a permanent upload under the tool that will consume it', async () => { + const { addAgentResourceFile } = require('~/models'); + setupStoredFileUpload(); + /* The code-env branch streams the upload from disk after this assertion resolves, and + * an unhandled stream error would take down the worker. Left in place rather than + * cleaned up, since the read happens later than the test body. */ + jest.requireActual('fs').writeFileSync('/tmp/upload.bin', 'zip'); + + await processAgentFileUpload({ + req: zipReq(), + res: mockRes, + metadata: { + agent_id: 'agent-abc', + file_id: 'f-perm', + agentTools: [EToolResources.execute_code], + }, + }).catch(() => {}); + + expect(addAgentResourceFile).toHaveBeenCalledWith( + expect.objectContaining({ tool_resource: EToolResources.execute_code }), + ); + }); + + test('records the converted image format rather than the upload type', async () => { + /* The stored bytes are the converted image. Keeping the upload's type leaves a later + * reprovision handing the sandbox webp bytes under a .jpg name, which the rename + * cannot catch because the extension already matches the stale type. */ + const handleImageUpload = jest.fn().mockResolvedValue({ + filepath: '/uploads/photo.webp', + bytes: 64, + width: 10, + height: 10, + }); + const storedFileUpload = jest.fn().mockResolvedValue({ + bytes: 4096, + filename: 'photo.jpg', + filepath: '/uploads/photo.jpg', + }); + getStrategyFunctions.mockReturnValue({ + handleImageUpload, + handleFileUpload: storedFileUpload, + }); + const req = makeReq({ mimetype: 'image/jpeg', ocrConfig: null }); + req.body.endpoint = EModelEndpoint.agents; + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { agent_id: 'agent-abc', message_file: 'true', file_id: 'f-image' }, + }).catch(() => {}); + + expect(db.createFile).toHaveBeenLastCalledWith( + expect.objectContaining({ file_id: 'f-image', type: 'image/webp' }), + true, + ); + /* The conversion runs under an id of its own, so persisting it too would leave a + * message-attachment row referenced by nothing beside the record above. */ + expect(db.createFile).toHaveBeenCalledTimes(1); + /* And the conversion is the only storage write: uploading the original first left a + * second object nothing references. */ + expect(handleImageUpload).toHaveBeenCalledTimes(1); + expect(storedFileUpload).not.toHaveBeenCalled(); + /* Size and dimensions describe the bytes actually stored, which the persistent-file + * screening later charges against the agent's allowance. */ + expect(db.createFile).toHaveBeenLastCalledWith( + expect.objectContaining({ bytes: 64, width: 10, height: 10 }), + true, + ); + }); + + test('treats an explicit message_file of "false" as a permanent upload', async () => { + /* Multipart form values arrive as strings, so the truthy reading made "false" mean + * message attachment while the route had already classified it as permanent. The + * upload reported success and filed nothing against the agent. */ + const { addAgentResourceFile } = require('~/models'); + setupStoredFileUpload(); + + await processAgentFileUpload({ + req: zipReq(), + res: mockRes, + metadata: { + agent_id: 'agent-abc', + message_file: 'false', + file_id: 'f-explicit-false', + agentTools: [EToolResources.execute_code], + }, + }).catch(() => {}); + + expect(addAgentResourceFile).toHaveBeenCalledWith( + expect.objectContaining({ tool_resource: EToolResources.execute_code }), + ); + }); + + test('skips a consumer whose capability is disabled', async () => { + /* Otherwise the choice depends on the persisted tool order: code execution listed + * first would be selected and then rejected, while file search could have kept it. */ + const { addAgentResourceFile } = require('~/models'); + setupStoredFileUpload(); + checkCapability.mockImplementation( + async (_req, capability) => capability !== AgentCapabilities.execute_code, + ); + + /* A searchable type routed off the model path, since a consumer is only chosen for a + * file kept off it, and only one that can read the file. */ + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + defaultLLMDeliveryPath: { overrides: { [PDF_MIME]: 'none' } }, + }); + const req = makeReq({ mimetype: PDF_MIME, ocrConfig: null }); + req.body.endpoint = EModelEndpoint.agents; + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + agent_id: 'agent-abc', + file_id: 'f-disabled', + agentTools: [EToolResources.execute_code, EToolResources.file_search], + }, + }).catch(() => {}); + + expect(addAgentResourceFile).toHaveBeenCalledWith( + expect.objectContaining({ tool_resource: EToolResources.file_search }), + ); + checkCapability.mockResolvedValue(true); + }); + + test('chooses an authorized consumer before routing an inferred destination', async () => { + const { addAgentResourceFile } = require('~/models'); + const { resolveToolRoleGrants } = require('@librechat/api'); + setupStoredFileUpload(); + resolveToolRoleGrants.mockResolvedValueOnce({ runCode: false, fileSearch: true }); + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + defaultLLMDeliveryPath: { overrides: { [PDF_MIME]: 'none' } }, + }); + const req = makeReq({ mimetype: PDF_MIME, ocrConfig: null }); + req.body.endpoint = EModelEndpoint.agents; + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + agent_id: 'agent-abc', + file_id: 'f-role-filtered', + agentTools: [EToolResources.execute_code, EToolResources.file_search], + }, + }); + + expect(addAgentResourceFile).toHaveBeenCalledWith( + expect.objectContaining({ tool_resource: EToolResources.file_search }), + ); + expect(resolveToolRoleGrants).toHaveBeenCalledTimes(1); + }); + + test('records a named destination as chosen in unified mode', async () => { + /* The marker asks whether the user named a destination, not which endpoint mode was + * on. Recording the mode treats an explicitly sandbox-only upload as inferred, and + * delivery then re-resolves it onto the model path. */ + setupStoredFileUpload(); + const req = makeReq({ mimetype: PDF_MIME, ocrConfig: null }); + req.body.endpoint = EModelEndpoint.agents; + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + agent_id: 'agent-abc', + tool_resource: EToolResources.file_search, + file_id: 'f-named', + }, + }).catch(() => {}); + + expect(db.createFile).toHaveBeenCalledWith( + expect.objectContaining({ + metadata: expect.objectContaining({ destinationChosen: true }), + }), + true, + ); + }); + + test('records the agent namespace on an eagerly embedded file', async () => { + /* Vectors go in under entity_id, and priming asks which namespaces hold them rather + * than reading the root flag, so omitting this re-embeds on the first search and + * aborts that search when RAG is briefly unavailable. */ + setupStoredFileUpload(); + const req = makeReq({ mimetype: PDF_MIME, ocrConfig: null }); + req.body.endpoint = EModelEndpoint.agents; + + await processAgentFileUpload({ + req, + res: mockRes, + metadata: { + agent_id: 'agent-abc', + tool_resource: EToolResources.file_search, + file_id: 'f-embedded', + }, + }).catch(() => {}); + + expect(db.createFile).toHaveBeenCalledWith( + expect.objectContaining({ + metadata: expect.objectContaining({ embeddedEntities: ['agent-abc'] }), + }), + true, + ); + }); + + test('names the destination that refused the file, not file search', async () => { + /* The message is the only thing telling the reader which tool rejected the upload, + * and reporting an image sent to file search for an audio file sent to the code + * interpreter describes neither the file nor the destination. */ + const req = makeReq({ mimetype: 'audio/mpeg', ocrConfig: null }); + req.body.endpoint = EModelEndpoint.agents; + + await expect( + processAgentFileUpload({ + req, + res: mockRes, + metadata: { + agent_id: 'agent-abc', + tool_resource: EToolResources.execute_code, + file_id: 'f-audio', + }, + }), + ).rejects.toThrow(/audio\/mpeg.*code interpreter/i); + }); + + test('refuses a permanent upload that would land on no agent resource', async () => { + /* Delivered straight to the model, with no agent resource to hold it, storing it + * would report success while leaving the agent without a reference. */ + const req = makeReq({ mimetype: 'image/png', ocrConfig: null }); + req.body.endpoint = EModelEndpoint.agents; + + await expect( + processAgentFileUpload({ + req, + res: mockRes, + metadata: { agent_id: 'agent-abc', file_id: 'f-orphan', agentTools: [] }, + }), + ).rejects.toThrow(/cannot be saved to an agent on their own/i); + }); +}); + +describe('filterFile endpoint resolution', () => { + /* getEndpointFileConfig consults endpointType ahead of endpoint, so a composer upload + * carrying `agents` would keep the Agents policy and shadow the provider the caller + * resolved. */ + /* mergeFileConfig is mocked here, so these are raw byte values rather than the + * megabytes an admin would write. The agents policy refuses the file on size and the + * provider policy accepts it, which makes the assertions read as which one governed. */ + const policyConfig = { + ...makeFileConfig(), + endpoints: { + default: { + disabled: false, + fileLimit: 10, + fileSizeLimit: 1_000_000, + totalSizeLimit: 1_000_000, + supportedMimeTypes: [/^image\/png$/], + }, + agents: { fileSizeLimit: 1 }, + 'Custom Provider': { fileSizeLimit: 1_000_000 }, + }, + }; + + const makeFilterReq = (endpointType) => ({ + body: { + endpoint: EModelEndpoint.agents, + ...(endpointType ? { endpointType } : {}), + file_id: '00000000-0000-4000-8000-000000000000', + width: 1, + height: 1, + }, + file: { size: 10, mimetype: 'image/png', originalname: 'a.png' }, + config: {}, + }); + + beforeEach(() => { + mergeFileConfig.mockReturnValue(policyConfig); + }); + + test('applies the resolved provider policy even when the request names an endpoint type', () => { + expect(() => + filterFile({ req: makeFilterReq('agents'), image: true, endpoint: 'Custom Provider' }), + ).not.toThrow(); + }); + + test('keeps the request endpoint type when no override is given', () => { + expect(() => filterFile({ req: makeFilterReq('agents'), image: true })).toThrow(/size limit/i); + }); + + test('rejects endpoint uploads when the resolved provider disables files', () => { + policyConfig.endpoints['Disabled Provider'] = { + ...policyConfig.endpoints.default, + disabled: true, + }; + + expect(() => + filterFile({ req: makeFilterReq('agents'), image: true, endpoint: 'Disabled Provider' }), + ).toThrow(/uploads are disabled/i); + }); + + test('does not apply an endpoint disabled flag to avatars', () => { + policyConfig.endpoints['Disabled Provider'] = { + ...policyConfig.endpoints.default, + disabled: true, + }; + const req = makeFilterReq(); + req.body.endpoint = 'Disabled Provider'; + + expect(() => filterFile({ req, image: true, isAvatar: true })).not.toThrow(); + }); +}); diff --git a/api/server/services/Files/provision.js b/api/server/services/Files/provision.js new file mode 100644 index 0000000000..32311cc500 --- /dev/null +++ b/api/server/services/Files/provision.js @@ -0,0 +1,23 @@ +const { createProvisionService } = require('@librechat/api'); +const { loadAuthValues } = require('~/server/services/Tools/credentials'); +const { uploadVectors } = require('./VectorDB/crud'); +const { getStrategyFunctions } = require('./strategies'); + +/* Wiring only: the provisioning logic lives in packages/api, where it is type checked. + * Storage strategies, vector upload, and credential lookup are api-workspace concerns, + * so they are bound here and injected. + * + * Built on first use rather than at module load: this module is reachable from the + * OpenAI-compatible controllers, whose suites partially mock @librechat/api, and a + * load-time call would throw before any provisioning is requested. */ +let service; +const getService = () => + (service ??= createProvisionService({ getStrategyFunctions, uploadVectors, loadAuthValues })); + +module.exports = { + loadCodeApiKey: (...args) => getService().loadCodeApiKey(...args), + provisionToCodeEnv: (...args) => getService().provisionToCodeEnv(...args), + provisionToVectorDB: (...args) => getService().provisionToVectorDB(...args), + checkCodeEnvFileAlive: (...args) => getService().checkCodeEnvFileAlive(...args), + checkSessionsAlive: (...args) => getService().checkSessionsAlive(...args), +}; diff --git a/api/server/services/Files/provisionCallback.js b/api/server/services/Files/provisionCallback.js new file mode 100644 index 0000000000..328cca2d9b --- /dev/null +++ b/api/server/services/Files/provisionCallback.js @@ -0,0 +1,20 @@ +const { createProvisionFilesCallback: createCallback } = require('@librechat/api'); +const { provisionToCodeEnv, provisionToVectorDB } = require('~/server/services/Files/provision'); +const db = require('~/models'); + +/* Wiring only: binds this workspace's provisioning service and file model to the + * ON_TOOL_EXECUTE callback implemented in packages/api. */ +function createProvisionFilesCallback({ req, agentToolContexts, resolvePrimaryAgentId }) { + return createCallback({ + req, + agentToolContexts, + resolvePrimaryAgentId, + provisionToCodeEnv, + provisionToVectorDB, + updateFile: db.updateFile, + updateCodeEnvRef: db.updateFileCodeEnvRef, + addEmbeddedEntity: db.addFileEmbeddedEntity, + }); +} + +module.exports = { createProvisionFilesCallback }; diff --git a/api/server/services/Files/routing.js b/api/server/services/Files/routing.js new file mode 100644 index 0000000000..0a22fa880f --- /dev/null +++ b/api/server/services/Files/routing.js @@ -0,0 +1,24 @@ +const { + resolveUploadAgent: resolveAgent, + resolveEffectiveToolResource: resolveToolResource, + resolveUploadEndpoint: resolveEndpoint, + resolveUploadLLMDeliveryPath, +} = require('@librechat/api'); +const db = require('~/models'); + +/* Wiring only: binds this workspace's agent model to the upload routing implemented in + * packages/api. */ +const resolveUploadAgent = (req, agent_id) => resolveAgent(req, agent_id, db.getAgent); + +const resolveUploadEndpoint = ({ endpoint, agent_id, req }) => + resolveEndpoint({ req, metadata: { endpoint, agent_id }, getAgent: db.getAgent }); + +const resolveEffectiveToolResource = ({ req, metadata }) => + resolveToolResource({ req, metadata, getAgent: db.getAgent }); + +module.exports = { + resolveUploadAgent, + resolveUploadEndpoint, + resolveUploadLLMDeliveryPath, + resolveEffectiveToolResource, +}; diff --git a/api/server/services/__tests__/toolCapabilityGates.spec.js b/api/server/services/__tests__/toolCapabilityGates.spec.js index d50f1ba1e7..56820fec4b 100644 --- a/api/server/services/__tests__/toolCapabilityGates.spec.js +++ b/api/server/services/__tests__/toolCapabilityGates.spec.js @@ -37,8 +37,10 @@ const CLASSIFIED = { 'api/server/controllers/agents/responses.js': 2, /** `codeEnvAvailable` for the memory-agent initializer. */ 'api/server/controllers/agents/client.js': 1, - /** Upload processing: the code-environment and file-search branches. */ - 'api/server/services/Files/process.js': 2, + /** Upload processing: two consumer-map entries, the role-gated consumer checks, + * and the code-environment/file-search branches. All consumer selection shares + * the role grant resolved before routing. */ + 'api/server/services/Files/process.js': 5, /** Agent-management upload purposes. */ 'api/server/routes/agents/management.js': 2, /** Two gates, each paired when the embedder wires `getRoleByName`; one in a diff --git a/client/src/Providers/DragDropContext.tsx b/client/src/Providers/DragDropContext.tsx index eae5f0165f..c7c5fc076f 100644 --- a/client/src/Providers/DragDropContext.tsx +++ b/client/src/Providers/DragDropContext.tsx @@ -1,8 +1,6 @@ import React, { createContext, useContext, useMemo } from 'react'; -import { isAgentsEndpoint, resolveEndpointType } from 'librechat-data-provider'; import type { EModelEndpoint } from 'librechat-data-provider'; -import { useGetEndpointsQuery, useGetAgentByIdQuery } from '~/data-provider'; -import { useAgentsMapContext } from './AgentsMapContext'; +import useAgentUploadTarget from '~/hooks/Agents/useAgentUploadTarget'; import { useChatContext } from './ChatContext'; interface DragDropContextValue { @@ -17,51 +15,7 @@ const DragDropContext = createContext(undefine export function DragDropProvider({ children }: { children: React.ReactNode }) { const { conversation } = useChatContext(); - const { data: endpointsConfig } = useGetEndpointsQuery(); - const agentsMap = useAgentsMapContext(); - - const needsAgentFetch = useMemo(() => { - const isAgents = isAgentsEndpoint(conversation?.endpoint); - if (!isAgents || !conversation?.agent_id) { - return false; - } - const agent = agentsMap?.[conversation.agent_id]; - return !agent?.model_parameters; - }, [conversation?.endpoint, conversation?.agent_id, agentsMap]); - - const { data: agentData } = useGetAgentByIdQuery(conversation?.agent_id, { - enabled: needsAgentFetch, - }); - - const agentProvider = useMemo(() => { - const isAgents = isAgentsEndpoint(conversation?.endpoint); - if (!isAgents || !conversation?.agent_id) { - return undefined; - } - return agentData?.provider ?? agentsMap?.[conversation.agent_id]?.provider; - }, [conversation?.endpoint, conversation?.agent_id, agentData, agentsMap]); - - const endpointType = useMemo( - () => resolveEndpointType(endpointsConfig, conversation?.endpoint, agentProvider), - [endpointsConfig, conversation?.endpoint, agentProvider], - ); - - const useResponsesApi = useMemo(() => { - const isAgents = isAgentsEndpoint(conversation?.endpoint); - if (!isAgents || !conversation?.agent_id || conversation?.useResponsesApi !== undefined) { - return conversation?.useResponsesApi; - } - return ( - agentData?.model_parameters?.useResponsesApi ?? - agentsMap?.[conversation.agent_id]?.model_parameters?.useResponsesApi - ); - }, [ - conversation?.endpoint, - conversation?.agent_id, - conversation?.useResponsesApi, - agentData, - agentsMap, - ]); + const { endpointType, useResponsesApi } = useAgentUploadTarget(conversation); /** Context value only created when conversation fields change */ const contextValue = useMemo( diff --git a/client/src/Providers/__tests__/DragDropContext.spec.tsx b/client/src/Providers/__tests__/DragDropContext.spec.tsx index 12a5edbb0a..946ced7de2 100644 --- a/client/src/Providers/__tests__/DragDropContext.spec.tsx +++ b/client/src/Providers/__tests__/DragDropContext.spec.tsx @@ -136,7 +136,9 @@ describe('DragDropContext endpointType resolution', () => { expect(result.current.useResponsesApi).toBe(true); }); - it('preserves an explicit conversation useResponsesApi false override', () => { + it('lets the saved agent decide over the conversation', () => { + /* Execution reads the agent's own model parameters, so predicting the conversation + * here routes an upload the turn then resolves the other way. */ mockConversation = { endpoint: EModelEndpoint.agents, agent_id: 'agent-1', @@ -147,7 +149,21 @@ describe('DragDropContext endpointType resolution', () => { model_parameters: { useResponsesApi: true }, } as Partial; const { result } = renderHook(() => useDragDropContext(), { wrapper }); - expect(result.current.useResponsesApi).toBe(false); + expect(result.current.useResponsesApi).toBe(true); + }); + + it('keeps the conversation setting when the agent states none', () => { + mockConversation = { + endpoint: EModelEndpoint.agents, + agent_id: 'agent-1', + useResponsesApi: true, + }; + mockAgentQueryData = { + provider: EModelEndpoint.azureOpenAI, + model_parameters: {}, + } as Partial; + const { result } = renderHook(() => useDragDropContext(), { wrapper }); + expect(result.current.useResponsesApi).toBe(true); }); }); diff --git a/client/src/components/Chat/Input/Files/AttachFileChat.tsx b/client/src/components/Chat/Input/Files/AttachFileChat.tsx index 89bd49f0ae..5395b8351b 100644 --- a/client/src/components/Chat/Input/Files/AttachFileChat.tsx +++ b/client/src/components/Chat/Input/Files/AttachFileChat.tsx @@ -4,14 +4,15 @@ import { supportsFiles, mergeFileConfig, isAgentsEndpoint, - resolveEndpointType, + isEphemeralAgentId, isAssistantsEndpoint, getEndpointFileConfig, } from 'librechat-data-provider'; import type { TConversation } from 'librechat-data-provider'; import type { ExtendedFile, FileSetter } from '~/common'; -import { useGetFileConfig, useGetEndpointsQuery, useGetAgentByIdQuery } from '~/data-provider'; -import { useAgentsMapContext } from '~/Providers'; +import useAgentUploadTarget from '~/hooks/Agents/useAgentUploadTarget'; +import { useGetFileConfig } from '~/data-provider'; +import { isUnifiedUploadMode } from '~/utils'; import AttachFileMenu from './AttachFileMenu'; import AttachFile from './AttachFile'; @@ -33,48 +34,14 @@ function AttachFileChat({ const isAgents = useMemo(() => isAgentsEndpoint(endpoint), [endpoint]); const isAssistants = useMemo(() => isAssistantsEndpoint(endpoint), [endpoint]); - const agentsMap = useAgentsMapContext(); + const { agentProvider, endpointType, useResponsesApi } = useAgentUploadTarget(conversation); - const needsAgentFetch = useMemo(() => { - if (!isAgents || !conversation?.agent_id) { - return false; - } - const agent = agentsMap?.[conversation.agent_id]; - return !agent?.model_parameters; - }, [isAgents, conversation?.agent_id, agentsMap]); - - const { data: agentData } = useGetAgentByIdQuery(conversation?.agent_id, { - enabled: needsAgentFetch, - }); - - const useResponsesApi = useMemo(() => { - if (!isAgents || !conversation?.agent_id || conversation?.useResponsesApi !== undefined) { - return conversation?.useResponsesApi; - } - return ( - agentData?.model_parameters?.useResponsesApi ?? - agentsMap?.[conversation.agent_id]?.model_parameters?.useResponsesApi - ); - }, [isAgents, conversation?.agent_id, conversation?.useResponsesApi, agentData, agentsMap]); - - const { data: fileConfig = null } = useGetFileConfig({ + /* Success, not merely settled: a failed or paused fetch leaves the built-in defaults in + * place, where the absent opt-out reads as unified. */ + const { data: fileConfig = null, isSuccess: isFileConfigLoaded } = useGetFileConfig({ select: (data) => mergeFileConfig(data), }); - const { data: endpointsConfig } = useGetEndpointsQuery(); - - const agentProvider = useMemo(() => { - if (!isAgents || !conversation?.agent_id) { - return undefined; - } - return agentData?.provider ?? agentsMap?.[conversation.agent_id]?.provider; - }, [isAgents, conversation?.agent_id, agentData, agentsMap]); - - const endpointType = useMemo( - () => resolveEndpointType(endpointsConfig, endpoint, agentProvider), - [endpointsConfig, endpoint, agentProvider], - ); - const fileConfigEndpoint = useMemo( () => (isAgents && agentProvider ? agentProvider : endpoint), [isAgents, agentProvider, endpoint], @@ -96,6 +63,19 @@ function AttachFileChat({ () => (disableInputs || endpointFileConfig?.disabled) ?? false, [disableInputs, endpointFileConfig?.disabled], ); + /* A saved agent's policy lives under its provider, so nothing is resolved for it until + * that provider is known; until then endpointFileConfig is the generic agents entry. + * An ephemeral agent has no record to fetch, so waiting on one never ends. */ + const isSavedAgent = + isAgents && conversation?.agent_id != null && !isEphemeralAgentId(conversation.agent_id); + const isPolicyResolved = isFileConfigLoaded && (!isSavedAgent || agentProvider != null); + + /* Resolved here rather than in the menu: an unresolved config reads as unified, which + * would show the wrong uploader on a legacy deployment. */ + const isUnifiedMode = useMemo( + () => isUnifiedUploadMode(endpointFileConfig, isPolicyResolved), + [endpointFileConfig, isPolicyResolved], + ); if (isAssistants && endpointSupportsFiles && !isUploadDisabled) { return ( @@ -111,11 +91,15 @@ function AttachFileChat({ return ( ; setFiles: FileSetter; @@ -90,6 +93,7 @@ const AttachFileMenu = ({ endpointType, conversationId, endpointFileConfig, + isUnifiedMode, useResponsesApi, files, setFiles, @@ -164,11 +168,38 @@ const AttachFileMenu = ({ inputRef.current.accept = ''; } inputRef.current.click(); - inputRef.current.accept = ''; }, [endpointFileConfig?.supportedMimeTypes], ); + /** Unified mode: single click triggers file upload with no tool_resource */ + const handleUnifiedUpload = useCallback(() => { + toolResourceRef.current = undefined; + handleUploadClick(); + }, [handleUploadClick]); + + /** Unified mode removed the destination chooser, not the source chooser. SharePoint has + * no trigger of its own, so without this the picker becomes unreachable whenever the + * composer is in unified mode. Destination stays implicit on both sources. */ + const unifiedSourceItems = useMemo( + () => [ + { + label: localize('com_files_upload_local_machine'), + onClick: handleUnifiedUpload, + icon: , + }, + { + label: localize('com_files_upload_sharepoint'), + onClick: () => { + toolResourceRef.current = undefined; + setIsSharePointDialogOpen(true); + }, + icon: , + }, + ], + [localize, handleUnifiedUpload, setIsSharePointDialogOpen], + ); + const dropdownItems = useMemo(() => { const setToolResource = (value: EToolResources | undefined) => { toolResourceRef.current = value; @@ -330,6 +361,63 @@ const AttachFileMenu = ({ } }; + if (isUnifiedMode) { + return ( + <> + { + handleFileChange(e, toolResourceRef.current); + }} + > + {sharePointEnabled === true ? ( + + ) : ( + + + + } + id="attach-file-button" + description={uploadFileTooltip} + disabled={isUploadDisabled} + /> + )} + + + + ); + } + return ( <> > = {}; let mockAgentQueryData: Partial | undefined; +let mockFileConfigLoaded = true; jest.mock('~/data-provider', () => ({ useGetEndpointsQuery: () => ({ data: mockEndpointsConfig }), useGetFileConfig: ({ select }: { select?: (data: unknown) => unknown }) => ({ data: select != null ? select(mockFileConfig) : mockFileConfig, + isSuccess: mockFileConfigLoaded, }), useGetAgentByIdQuery: () => ({ data: mockAgentQueryData }), })); @@ -38,6 +40,12 @@ jest.mock('~/Providers', () => ({ useAgentsMapContext: () => mockAgentsMap, })); +/* The shared upload-target hook reads the module directly rather than the barrel, so + * the barrel mock alone leaves it with no agents map. */ +jest.mock('~/Providers/AgentsMapContext', () => ({ + useAgentsMapContext: () => mockAgentsMap, +})); + /** Capture the props passed to AttachFileMenu */ let mockAttachFileMenuProps: Record = {}; jest.mock('../AttachFileMenu', () => { @@ -77,6 +85,7 @@ describe('AttachFileChat', () => { mockAgentsMap = {}; mockAgentQueryData = undefined; mockAttachFileMenuProps = {}; + mockFileConfigLoaded = true; }); describe('rendering decisions', () => { @@ -161,7 +170,9 @@ describe('AttachFileChat', () => { expect(mockAttachFileMenuProps.useResponsesApi).toBe(true); }); - it('preserves an explicit conversation useResponsesApi false override', () => { + it('lets the saved agent decide over the conversation', () => { + /* Execution reads the agent's own model parameters, so the attach menu must offer + * what the turn will actually do rather than what the conversation says. */ mockAgentQueryData = { provider: EModelEndpoint.azureOpenAI, model_parameters: { useResponsesApi: true }, @@ -171,7 +182,20 @@ describe('AttachFileChat', () => { agent_id: 'agent-1', useResponsesApi: false, }); - expect(mockAttachFileMenuProps.useResponsesApi).toBe(false); + expect(mockAttachFileMenuProps.useResponsesApi).toBe(true); + }); + + it('keeps the conversation setting when the agent states none', () => { + mockAgentQueryData = { + provider: EModelEndpoint.azureOpenAI, + model_parameters: {}, + } as Partial; + renderComponent({ + endpoint: EModelEndpoint.agents, + agent_id: 'agent-1', + useResponsesApi: true, + }); + expect(mockAttachFileMenuProps.useResponsesApi).toBe(true); }); }); @@ -282,3 +306,64 @@ describe('AttachFileChat', () => { }); }); }); + +describe('AttachFileChat upload config gating', () => { + beforeEach(() => { + mockFileConfig = defaultFileConfig; + mockAgentsMap = {}; + mockAgentQueryData = undefined; + mockAttachFileMenuProps = {}; + }); + + it('leaves the attach menu inert until the file config resolves', () => { + /* The menu is an action, not just a display: offering the chooser here submits an + * explicit destination a unified deployment would have inferred, and after a failed + * fetch it would keep offering it. */ + mockFileConfigLoaded = false; + + renderComponent({ endpoint: EModelEndpoint.agents }); + + expect(mockAttachFileMenuProps.disabled).toBe(true); + }); + + it('enables it once the config lands', () => { + mockFileConfigLoaded = true; + + renderComponent({ endpoint: EModelEndpoint.agents }); + + expect(mockAttachFileMenuProps.disabled).toBe(false); + }); + + it('keeps it inert while a saved agent provider is still being fetched', () => { + /* The config can succeed before the agent does, and until then endpointFileConfig is + * the generic agents entry rather than the provider's. */ + mockFileConfigLoaded = true; + mockAgentsMap = {}; + mockAgentQueryData = undefined; + + renderComponent({ endpoint: EModelEndpoint.agents, agent_id: 'agent_saved01' }); + + expect(mockAttachFileMenuProps.disabled).toBe(true); + }); + + it('does not wait on a provider an ephemeral agent will never have', () => { + /* There is no record to fetch for one, so gating on its provider never lifts and the + * attach control stays disabled for the whole conversation. */ + mockFileConfigLoaded = true; + mockAgentsMap = {}; + mockAgentQueryData = undefined; + + renderComponent({ endpoint: EModelEndpoint.agents, agent_id: 'ephemeral-convo-1' }); + + expect(mockAttachFileMenuProps.disabled).toBe(false); + }); + + it('enables it once that provider resolves', () => { + mockFileConfigLoaded = true; + mockAgentQueryData = { provider: 'openAI' } as Partial; + + renderComponent({ endpoint: EModelEndpoint.agents, agent_id: 'agent_saved01' }); + + expect(mockAttachFileMenuProps.disabled).toBe(false); + }); +}); diff --git a/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx b/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx index 4393f81918..13d60564e4 100644 --- a/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx +++ b/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx @@ -46,6 +46,12 @@ jest.mock('@librechat/client', () => { ), ), TooltipAnchor: (props) => props.render, + /* Mirrors the real primitive closely enough for these assertions: it applies + * `label` as the accessible name, which is how the trigger is queried. */ + IconButton: R.forwardRef((props, ref) => { + const { label, size: _size, shape: _shape, variant: _variant, children, ...rest } = props; + return R.createElement('button', { ref, 'aria-label': label, ...rest }, children); + }), DropdownPopup: (props) => R.createElement( 'div', @@ -99,6 +105,7 @@ const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false function setupMocks(overrides: { provider?: string } = {}) { const translations: Record = { + com_files_upload_local_machine: 'From Local Computer', com_files_upload_sharepoint: 'Upload from SharePoint', com_sidepanel_attach_files: 'Attach Files', com_ui_upload_code_environment: 'Upload to Code Environment', @@ -141,6 +148,7 @@ function renderMenu(props: Record = {}) { setFiles={() => {}} setFilesLoading={() => {}} conversation={null} + isUnifiedMode={false} {...props} /> @@ -155,6 +163,51 @@ function openMenu() { describe('AttachFileMenu', () => { beforeEach(jest.clearAllMocks); + describe('unified mode upload sources', () => { + it('uses a single upload button when SharePoint is disabled', () => { + setupMocks(); + renderMenu({ isUnifiedMode: true }); + + expect(screen.getByRole('button', { name: /attach files/i })).toBeInTheDocument(); + expect( + screen.queryByRole('button', { name: /attach file options/i }), + ).not.toBeInTheDocument(); + }); + + it('offers SharePoint alongside local upload when SharePoint is enabled', () => { + setupMocks(); + mockUseGetStartupConfig.mockReturnValue({ data: { sharePointFilePickerEnabled: true } }); + renderMenu({ isUnifiedMode: true }); + + openMenu(); + + expect(screen.getByText('From Local Computer')).toBeInTheDocument(); + expect(screen.getByText('Upload from SharePoint')).toBeInTheDocument(); + }); + + it('does not offer a destination choice on either source', () => { + setupMocks(); + mockUseAgentCapabilities.mockReturnValue({ + contextEnabled: true, + fileSearchEnabled: true, + codeEnabled: true, + }); + mockUseAgentToolPermissions.mockReturnValue({ + fileSearchAllowedByAgent: true, + codeAllowedByAgent: true, + provider: undefined, + }); + mockUseGetStartupConfig.mockReturnValue({ data: { sharePointFilePickerEnabled: true } }); + renderMenu({ isUnifiedMode: true }); + + openMenu(); + + expect(screen.queryByText('Upload to Code Environment')).not.toBeInTheDocument(); + expect(screen.queryByText('Upload for File Search')).not.toBeInTheDocument(); + expect(screen.queryByText('Upload as Text')).not.toBeInTheDocument(); + }); + }); + describe('Upload to Provider vs Upload Image', () => { it('shows "Upload to Provider" when endpointType is custom (resolved from agent provider)', () => { setupMocks({ provider: 'Moonshot' }); @@ -273,6 +326,15 @@ describe('AttachFileMenu', () => { expect(screen.getByTestId('dropdown-popup')).toHaveAttribute('data-modal', 'false'); expect(screen.getByTestId('dropdown-popup')).toHaveAttribute('data-portal', 'true'); }); + + it('renders the unified upload button when legacyFileUploadUX is not true', () => { + setupMocks(); + renderMenu({ isUnifiedMode: true }); + expect(screen.getByRole('button', { name: /attach files/i })).toBeInTheDocument(); + expect( + screen.queryByRole('button', { name: /attach file options/i }), + ).not.toBeInTheDocument(); + }); }); describe('Agent Capabilities', () => { diff --git a/client/src/components/Chat/Input/ToolsDropdown.tsx b/client/src/components/Chat/Input/ToolsDropdown.tsx index 66d5684ddc..64fc779bba 100644 --- a/client/src/components/Chat/Input/ToolsDropdown.tsx +++ b/client/src/components/Chat/Input/ToolsDropdown.tsx @@ -167,7 +167,7 @@ const ToolsDropdown = ({ disabled }: ToolsDropdownProps) => { onClick: handleFileSearchToggle, hideOnClick: false, render: (props) => ( -
+
{localize('com_assistants_file_search')} @@ -317,7 +317,7 @@ const ToolsDropdown = ({ disabled }: ToolsDropdownProps) => { onClick: handleCodeInterpreterToggle, hideOnClick: false, render: (props) => ( -
+