diff --git a/CONTEXT.md b/CONTEXT.md index b390f49505..62ad355845 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -9,6 +9,7 @@ - **Agent execution host**: the protocol-neutral module that owns run admission, disconnect cancellation, provider-start fencing, and terminal settlement. Protocol implementations execute behind its callback interface; HTTP adapters retain validation and final stream rendering. - **Agent execution enrollment**: the durable, protocol-neutral lifecycle authority for an admitted Agent run. It is created under the authenticated user and tenant before user-owned initialization, rechecks the shared owner-deletion admission fence after registration, exposes the only provider abort signal, fences exact provider start, terminalizes the run, waits for every trailing usage, artifact, and stored-response write, and acknowledges provider drain last. A transient terminalization failure is reconciled after trailing writes; provider drain is never acknowledged while the exact job remains nonterminal. Delete-all holds the owner fence, drains every owner run before selecting its first persistence snapshot, and repeats both the drain and an idempotent owner-persistence sweep after any recovered fence lapse before releasing admission. Exact-conversation deletion additionally performs an unconditional idempotent cleanup over its immutable deleted-ID set because a fully drained run may leave the active index after racing the first delete; only the explicit empty result is benign, while storage failures remain fatal. Chat Completions, Responses, Channels, and future ingress adapters share this authority without moving LibreChat persistence policy into the Agents SDK. - **Agent turn execution plan**: the immutable, request-local decision compiled once after authentication, agent resolution, and tool initialization. It records the trusted turn origin, conversation lineage, pause capability, binding/action context, and the preferred checkpoint, history, or fresh state-loading strategy without executing the model or owning persistence. Checkpoint failure falls back to durable history within the same Agents lifecycle. +- **Turn delivery routing**: the per-agent, request-local value that decides how each attachment reaches the model on one turn (`provider`, `text`, or `none`). Initialization settles it once, after the provider swap and the Responses API decision, under the endpoint's own name and the media dialect its config declares. Every reader of a turn route consumes that one value rather than deriving it from the agent. A stored route is an upload-time inference that this value resolves again for the turn; a destination the user chose stands. - **Effective agent selection**: the resolved endpoint and agent identity after an enforced model spec is applied. Authorization and agent loading must consume this same identity before the Agent run envelope is initialized. - **MCP runtime request body**: trusted chat identifiers supplied only while an MCP server handles an agent request. It enables request-scoped header placeholders without retaining user-specific request data on a shared server definition. - **MCP direct OpenID bearer**: an operator-trusted remote MCP credential mode that resolves the logged-in user's live OpenID access token into an Authorization header. It may replace one rejected connection after a forced session refresh, but it never replays the rejected tool invocation automatically. diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index 81f8998760..4835f9ad80 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -9,6 +9,7 @@ const { sanitizeFileForTransmit, extractFileContext, getReferencedQuotes, + applyTurnDelivery, encodeAndFormatAudios, encodeAndFormatVideos, getTransactionsConfig, @@ -33,19 +34,15 @@ const { isCompactedLeaf, excludedKeys, EModelEndpoint, - mergeFileConfig, isParamEndpoint, isAgentsEndpoint, isEphemeralAgentId, supportsBalanceCheck, isBedrockDocumentType, HITL_MESSAGE_FILTER_FIELDS, - getEndpointFileConfig, stripReasoningLabelMetadata, - resolveUploadLLMDeliveryPath, - isSpeechProviderConfigured, + resolveTurnLLMDeliveryPath, resolveUseResponsesApi, - getCustomEndpointProvider, } = require('librechat-data-provider'); const { getStrategyFunctions } = require('~/server/services/Files/strategies'); const { logViolation } = require('~/cache'); @@ -242,10 +239,6 @@ class BaseClient { this.currentMessages = []; /** @type {import('librechat-data-provider').VisionModes | undefined} */ this.visionMode; - /** @type {import('librechat-data-provider').FileConfig | undefined} */ - this._mergedFileConfig; - /** @type {import('librechat-data-provider').EndpointFileConfig | undefined} */ - this._endpointFileConfig; } setOptions() { @@ -832,9 +825,8 @@ class BaseClient { if (this.options.resendFiles !== false && this.authorizedHistoricalFiles == null) { const historicalFileState = collectModelBoundHistoricalFileIdState(modelBoundStoredMessages); this.modelBoundHistoricalFileIdsOverflowed ||= historicalFileState.overflowed; - const files = await getOwnerHistoricalFiles( - historicalFileState.fileIds, - this.options.req?.user, + const files = this.resolveTurnAttachments( + await getOwnerHistoricalFiles(historicalFileState.fileIds, this.options.req?.user), ); this.authorizedHistoricalFiles = new Map( files @@ -1781,8 +1773,8 @@ class BaseClient { * @param {MongoFile[]} attachments - Array of file attachments * @returns {Promise} */ - async addFileContextToMessage(message, attachments) { - const textAttachments = this.getTextContextAttachments(attachments); + async addFileContextToMessage(message, attachments, fileConsumers) { + const textAttachments = this.getTextContextAttachments(attachments, fileConsumers); const fileContext = await extractFileContext({ attachments: textAttachments, req: this.options?.req, @@ -1794,9 +1786,9 @@ class BaseClient { } } - getTextContextAttachments(attachments) { + getTextContextAttachments(attachments, fileConsumers) { return attachments.filter((file) => { - const deliveryPath = this.getAttachmentDeliveryPath(file); + const deliveryPath = this.getAttachmentDeliveryPath(file, fileConsumers); /* Records predating delivery paths keep legacy extraction. Current routing is * authoritative for inferred uploads, so native provider bytes are not also * injected as extracted text after a provider handoff. */ @@ -1804,41 +1796,19 @@ class BaseClient { }); } - /** Re-resolves an inferred upload route against the provider handling this turn. */ - getAttachmentDeliveryPath(file) { - if (!this._mergedFileConfig) { - this._mergedFileConfig = mergeFileConfig(this.options.req?.config?.fileConfig); - /* Agent file policy is configured under the endpoint it names, not the client - * family initialization may rewrite it to. */ - const agentEndpoint = this.options.agent?.endpoint ?? this.options.agent?.provider; - this._deliveryEndpoint = agentEndpoint ?? this.options.endpoint; - this._endpointFileConfig = getEndpointFileConfig({ - fileConfig: this._mergedFileConfig, - endpoint: this._deliveryEndpoint, - endpointType: agentEndpoint != null ? undefined : this.options.endpointType, - }); - } - - return file.llmDeliveryPath == null || file.metadata?.destinationChosen === true - ? file.llmDeliveryPath - : resolveUploadLLMDeliveryPath({ - /* Conversion changes the stored type, so use the type routing originally saw. */ - mimeType: file.metadata?.routingMimeType ?? file.type, - endpointConfig: this._endpointFileConfig, - fileConfig: this._mergedFileConfig, - endpoint: this._deliveryEndpoint, - endpointProvider: - this.options.agent?.provider ?? - getCustomEndpointProvider( - this.options.req?.config?.endpoints?.custom, - this._deliveryEndpoint, - ), - useResponsesApi: this.usesResponsesApi(), - sttConfigured: isSpeechProviderConfigured(this.options.req?.config?.speech?.stt), - }); + /** The turn's view of stored records, applied before admission at every load. */ + resolveTurnAttachments(files, fileConsumers = this.options.agent?.fileConsumers) { + return applyTurnDelivery(files, { + routing: this.options.agent?.deliveryRouting, + consumers: fileConsumers, + }); } - async processAttachments(message, attachments) { + getAttachmentDeliveryPath(file, fileConsumers = this.options.agent?.fileConsumers) { + return resolveTurnLLMDeliveryPath(this.options.agent?.deliveryRouting, file, fileConsumers); + } + + async processAttachments(message, attachments, fileConsumers) { const categorizedAttachments = { images: [], videos: [], @@ -1849,6 +1819,7 @@ class BaseClient { const allFiles = []; const provider = this.options.agent?.provider ?? this.options.endpoint; const isBedrock = provider === EModelEndpoint.bedrock; + const deliveryRouting = this.options.agent?.deliveryRouting; /* The stored path records what upload time inferred from the endpoint it saw, and this * turn may be running somewhere else: audio stored as `provider` under Google reaches @@ -1862,7 +1833,7 @@ class BaseClient { allFiles.push(file); continue; } - const deliveryPath = this.getAttachmentDeliveryPath(file); + const deliveryPath = this.getAttachmentDeliveryPath(file, fileConsumers); if (deliveryPath === 'text' || deliveryPath === 'none') { allFiles.push(file); continue; @@ -1897,9 +1868,11 @@ class BaseClient { allFiles.push(file); } else if ( file.type && - this._mergedFileConfig && - this._endpointFileConfig?.supportedMimeTypes && - this._mergedFileConfig.checkType(file.type, this._endpointFileConfig.supportedMimeTypes) + deliveryRouting?.endpointConfig.supportedMimeTypes && + deliveryRouting.fileConfig.checkType( + file.type, + deliveryRouting.endpointConfig.supportedMimeTypes, + ) ) { categorizedAttachments.documents.push(file); allFiles.push(file); @@ -1961,9 +1934,8 @@ class BaseClient { const historicalFileState = collectModelBoundHistoricalFileIdState(_messages); this.modelBoundHistoricalFileIdsOverflowed ||= historicalFileState.overflowed; const authorizedFilesById = new Map(); - const files = await getOwnerHistoricalFiles( - historicalFileState.fileIds, - this.options.req?.user, + const files = this.resolveTurnAttachments( + await getOwnerHistoricalFiles(historicalFileState.fileIds, this.options.req?.user), ); const nonSteerReplayFileIds = collectModelBoundHistoricalFileIdState( _messages.map((message) => ({ diff --git a/api/app/clients/specs/BaseClient.test.js b/api/app/clients/specs/BaseClient.test.js index 26d8f3ccfb..53c22c33dc 100644 --- a/api/app/clients/specs/BaseClient.test.js +++ b/api/app/clients/specs/BaseClient.test.js @@ -1,6 +1,11 @@ const { Constants, ContentTypes, EModelEndpoint } = require('librechat-data-provider'); const BaseClientClass = require('../BaseClient'); -const { ContentFilterError } = require('@librechat/api'); +const { + ContentFilterError, + resolveTurnDeliveryRouting, + buildSteerMedia, + Tokenizer, +} = require('@librechat/api'); const { FakeClient, initializeFakeClient } = require('./FakeClient'); function deferred() { @@ -1174,6 +1179,69 @@ describe('BaseClient', () => { expect(TestClient.buildMessages).toHaveBeenCalled(); }); + test('keeps the turn view of historical files that projection and steer replay read', async () => { + const routedCsv = { + file_id: 'csv-file', + filename: 'sales.csv', + filepath: '/uploads/sales.csv', + type: 'text/csv', + text: 'region,total', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + user: 'user-1', + }; + getFiles.mockReset(); + getFiles.mockResolvedValueOnce([routedCsv]); + TestClient = initializeFakeClient( + apiKey, + { + ...options, + agent: { + provider: EModelEndpoint.openAI, + fileConsumers: { executeCode: false, fileSearch: false }, + }, + req: { + user: { id: 'user-1', tenantId: 'tenant-a' }, + config: { + fileConfig: { + endpoints: { + [EModelEndpoint.openAI]: { + defaultLLMDeliveryPath: { overrides: { 'text/csv': 'none' } }, + textFallbackWithoutTools: true, + }, + }, + }, + }, + }, + }, + [ + { + role: 'user', + isCreatedByUser: true, + text: 'Summarize my sheet', + files: [{ file_id: 'csv-file' }], + messageId: 'historical-csv-message', + parentMessageId: Constants.NO_PARENT, + }, + ], + ); + + TestClient.options.agent.deliveryRouting = resolveTurnDeliveryRouting({ + agent: TestClient.options.agent, + config: TestClient.options.req.config, + }); + await TestClient.sendMessage('And the totals?', { + conversationId: 'historical-csv-conversation', + parentMessageId: 'historical-csv-message', + }); + + expect(TestClient.authorizedHistoricalFiles.get('csv-file')).toEqual({ + ...routedCsv, + llmDeliveryPath: 'text', + }); + expect(routedCsv.llmDeliveryPath).toBe('none'); + }); + test('does not block a missing historical file omitted from the final payload', async () => { getFiles.mockReset(); getFiles.mockResolvedValueOnce([]); @@ -2580,6 +2648,110 @@ describe('BaseClient', () => { TestClient.checkVisionRequest = jest.fn(); }); + describe('tool-routed files on a later turn', () => { + const routedCsv = { + file_id: 'csv-file', + filename: 'sales.csv', + filepath: '/uploads/sales.csv', + source: 'local', + type: 'text/csv', + user: 'user-1', + text: 'region,total', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + }; + + const replayCsv = async ( + fileConsumers, + endpointConfig = { + defaultLLMDeliveryPath: { overrides: { 'text/csv': 'none' } }, + textFallbackWithoutTools: true, + }, + ) => { + getFiles.mockResolvedValueOnce([routedCsv]); + TestClient.options.req.config = { + fileConfig: { endpoints: { [EModelEndpoint.openAI]: endpointConfig } }, + }; + TestClient.options.agent = { provider: EModelEndpoint.openAI, fileConsumers }; + TestClient.options.agent.deliveryRouting = resolveTurnDeliveryRouting({ + agent: TestClient.options.agent, + config: TestClient.options.req?.config, + }); + TestClient.assertHistoricalAttachmentLimits = jest.fn(async (files) => files); + const [message] = await TestClient.addPreviousAttachments([ + { messageId: 'msg-csv', text: 'Summarize it', files: [{ file_id: 'csv-file' }] }, + ]); + return message; + }; + + test('replays the stored text when this turn runs no tool that can read the file', async () => { + const message = await replayCsv({ executeCode: false, fileSearch: false }); + const replayed = { ...routedCsv, llmDeliveryPath: 'text' }; + + expect(TestClient.assertHistoricalAttachmentLimits).toHaveBeenCalledWith([replayed]); + expect(TestClient.addFileContextToMessage).toHaveBeenCalledWith(message, [replayed]); + expect(TestClient.authorizedHistoricalFiles.get('csv-file')).toEqual(replayed); + expect(message.fileContext).toBe('region,total'); + expect(routedCsv.llmDeliveryPath).toBe('none'); + }); + + test('replays the stored text once the endpoint routes the type to text', async () => { + const message = await replayCsv( + { executeCode: true, fileSearch: false }, + { defaultLLMDeliveryPath: { overrides: { 'text/csv': 'text' } } }, + ); + const replayed = { ...routedCsv, llmDeliveryPath: 'text' }; + + expect(TestClient.assertHistoricalAttachmentLimits).toHaveBeenCalledWith([replayed]); + expect(TestClient.addFileContextToMessage).toHaveBeenCalledWith(message, [replayed]); + expect(message.fileContext).toBe('region,total'); + }); + + test('admits a historical tool-routed file this turn sends to the provider', async () => { + const routedImage = { + file_id: 'image-file', + filename: 'chart.png', + filepath: '/uploads/chart.png', + source: 'local', + type: 'image/png', + user: 'user-1', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + }; + getFiles.mockResolvedValueOnce([routedImage]); + TestClient.options.req.config = { fileConfig: { endpoints: {} } }; + TestClient.options.agent = { + provider: EModelEndpoint.openAI, + fileConsumers: { executeCode: false, fileSearch: false }, + }; + TestClient.options.agent.deliveryRouting = resolveTurnDeliveryRouting({ + agent: TestClient.options.agent, + config: TestClient.options.req?.config, + }); + TestClient.assertHistoricalAttachmentLimits = jest.fn(async (files) => files); + + await TestClient.addPreviousAttachments([ + { + messageId: 'msg-image', + text: 'What does it show?', + files: [{ file_id: 'image-file' }], + }, + ]); + + expect(TestClient.assertHistoricalAttachmentLimits).toHaveBeenCalledWith([ + { ...routedImage, llmDeliveryPath: 'provider' }, + ]); + }); + + test('keeps the file off the prompt when this turn can read it with code', async () => { + const message = await replayCsv({ executeCode: true, fileSearch: false }); + + expect(TestClient.assertHistoricalAttachmentLimits).toHaveBeenCalledWith([]); + expect(TestClient.addFileContextToMessage).not.toHaveBeenCalled(); + expect(message.fileContext).toBeUndefined(); + }); + }); + test('rehydrates historical file refs from owner-scoped DB rows only', async () => { getFiles.mockResolvedValueOnce([ownerFile]); @@ -3192,8 +3364,6 @@ describe('BaseClient', () => { TestClient.options = { endpoint: EModelEndpoint.openAI, }; - TestClient._mergedFileConfig = undefined; - TestClient._endpointFileConfig = undefined; TestClient.addImageURLs = jest.fn(async (message, files) => { message.image_urls = ['encoded-image']; return files; @@ -3207,9 +3377,18 @@ describe('BaseClient', () => { TestClient.addAudios = jest.fn(async (_message, files) => files); }); - /* The stored path is an upload-time inference, so delivery re-resolves it for the - * endpoint running the turn. A test asserting a route has to configure that route - * rather than rely on the stored value alone. */ + /** The routing initialization settles for an agent, from the request config it reads. */ + const routedAgent = (agent) => ({ + ...agent, + deliveryRouting: resolveTurnDeliveryRouting({ + agent, + config: TestClient.options.req?.config, + }), + }); + + /* The stored path is an upload-time inference, so delivery resolves it again by the + * routing settled for the agent running the turn. A test asserting a route has to + * configure that route rather than rely on the stored value alone. */ const routeTo = (path, ...mimeTypes) => { TestClient.options.req = { config: { @@ -3224,8 +3403,10 @@ describe('BaseClient', () => { }, }, }; - TestClient._mergedFileConfig = undefined; - TestClient._endpointFileConfig = undefined; + TestClient.options.agent = routedAgent({ + provider: EModelEndpoint.openAI, + endpoint: EModelEndpoint.openAI, + }); }; test('keeps a none image in returned files without adding image URLs', async () => { @@ -3262,6 +3443,144 @@ describe('BaseClient', () => { expect(TestClient.getTextContextAttachments([file])).toEqual([]); }); + const routeCsvToTools = ({ textFallbackWithoutTools = true } = {}) => { + routeTo('none', 'text/csv'); + TestClient.options.req.config.fileConfig.endpoints[ + EModelEndpoint.openAI + ].textFallbackWithoutTools = textFallbackWithoutTools; + }; + + test('injects the text stored for a tool-routed file when this turn runs no reader', () => { + routeCsvToTools(); + TestClient.options.agent = { + provider: EModelEndpoint.openAI, + fileConsumers: { executeCode: false, fileSearch: false }, + }; + TestClient.options.agent.deliveryRouting = resolveTurnDeliveryRouting({ + agent: TestClient.options.agent, + config: TestClient.options.req?.config, + }); + /* Agent initialization marks the copy it hands this client, so the stored route reads + * `text` while the configured route stays `none`. */ + const file = { + file_id: 'fallback-csv', + filename: 'sales.csv', + type: 'text/csv', + text: 'region,total', + llmDeliveryPath: 'text', + metadata: { destinationChosen: false }, + }; + + expect(TestClient.getAttachmentDeliveryPath(file)).toBe('text'); + expect(TestClient.getTextContextAttachments([file])).toEqual([file]); + }); + + test('delivers a late steer through fallback even when the initialized agent has file tools', async () => { + routeCsvToTools(); + TestClient.options.agent = routedAgent({ + provider: EModelEndpoint.openAI, + fileConsumers: { executeCode: true, fileSearch: true }, + }); + const file = { + file_id: 'late-csv', + filename: 'late.csv', + type: 'text/csv', + source: 'local', + text: 'region,total', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + }; + const assertFilesAllowed = jest.fn(); + const initEncoding = jest.spyOn(Tokenizer, 'initEncoding').mockResolvedValue(undefined); + const getTokenCount = jest.spyOn(Tokenizer, 'getTokenCount').mockReturnValue(4); + let result; + try { + result = await buildSteerMedia({ + client: { + resolveTurnAttachments: TestClient.resolveTurnAttachments.bind(TestClient), + addFileContextToMessage: + BaseClientClass.prototype.addFileContextToMessage.bind(TestClient), + processAttachments: BaseClientClass.prototype.processAttachments.bind(TestClient), + }, + user: { id: 'user-1' }, + item: { steerId: 'late', text: 'Read this file', files: [{ file_id: file.file_id }] }, + getFiles: jest.fn().mockResolvedValue([file]), + assertFilesAllowed, + }); + } finally { + initEncoding.mockRestore(); + getTokenCount.mockRestore(); + } + expect(assertFilesAllowed).toHaveBeenCalledWith([{ ...file, llmDeliveryPath: 'text' }]); + expect(JSON.stringify(result.content)).toContain('region,total'); + expect(file.llmDeliveryPath).toBe('none'); + expect(TestClient.options.agent.fileConsumers).toEqual({ + executeCode: true, + fileSearch: true, + }); + }); + + test('keeps a tool-routed file off the prompt when this turn can read it with code', () => { + routeCsvToTools(); + TestClient.options.agent = { + provider: EModelEndpoint.openAI, + fileConsumers: { executeCode: true, fileSearch: false }, + }; + TestClient.options.agent.deliveryRouting = resolveTurnDeliveryRouting({ + agent: TestClient.options.agent, + config: TestClient.options.req?.config, + }); + const file = { + file_id: 'code-csv', + filename: 'sales.csv', + type: 'text/csv', + text: 'region,total', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + }; + + expect(TestClient.getAttachmentDeliveryPath(file)).toBe('none'); + expect(TestClient.getTextContextAttachments([file])).toEqual([]); + }); + + test('does not fall back on an endpoint that has not enabled it', () => { + routeCsvToTools({ textFallbackWithoutTools: false }); + TestClient.options.agent = { + provider: EModelEndpoint.openAI, + fileConsumers: { executeCode: false, fileSearch: false }, + }; + TestClient.options.agent.deliveryRouting = resolveTurnDeliveryRouting({ + agent: TestClient.options.agent, + config: TestClient.options.req?.config, + }); + const file = { + file_id: 'disabled-csv', + filename: 'sales.csv', + type: 'text/csv', + text: 'region,total', + llmDeliveryPath: 'text', + metadata: { destinationChosen: false }, + }; + + expect(TestClient.getAttachmentDeliveryPath(file)).toBe('none'); + expect(TestClient.getTextContextAttachments([file])).toEqual([]); + }); + + test('does not fall back when the turn tools are unknown', () => { + routeCsvToTools(); + TestClient.options.agent = { provider: EModelEndpoint.openAI }; + const file = { + file_id: 'unknown-csv', + filename: 'sales.csv', + type: 'text/csv', + text: 'region,total', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + }; + + expect(TestClient.getTextContextAttachments([file])).toEqual([]); + }); + test('does not inject extracted text when the current provider resolves native delivery', () => { routeTo('provider', 'application/pdf'); const file = { @@ -3327,17 +3646,19 @@ describe('BaseClient', () => { expect(TestClient.addImageURLs).not.toHaveBeenCalled(); }); - test('reads the Responses setting from a plain conversation too', async () => { - /* A non-agent Azure chat carries it in model options, and reading only the agent - * parameters re-resolves a natively supported PDF to text, which the record has - * none of, so the model receives nothing. */ + test('reads the Responses setting the turn runs on from the settled routing', async () => { + /* Azure sends a PDF natively only under the Responses API. The routing carries the + * decision initialization made, so a record stored as `provider` is not resolved + * again to text it has none of, which would leave the model with nothing. */ TestClient.options = { - endpoint: EModelEndpoint.azureOpenAI, + endpoint: EModelEndpoint.agents, req: { config: { fileConfig: undefined } }, }; - TestClient.modelOptions = { useResponsesApi: true }; - TestClient._mergedFileConfig = undefined; - TestClient._endpointFileConfig = undefined; + TestClient.options.agent = routedAgent({ + provider: EModelEndpoint.azureOpenAI, + endpoint: EModelEndpoint.azureOpenAI, + model_parameters: { useResponsesApi: true }, + }); const message = {}; const file = { user: 'user1', @@ -3353,7 +3674,6 @@ describe('BaseClient', () => { await TestClient.processAttachments(message, [file]); expect(TestClient.addDocuments).toHaveBeenCalled(); - TestClient.modelOptions = undefined; }); test('resolves a custom endpoint policy by the name the admin configured', async () => { @@ -3377,8 +3697,7 @@ describe('BaseClient', () => { }, }, }; - TestClient._mergedFileConfig = undefined; - TestClient._endpointFileConfig = undefined; + TestClient.options.agent = routedAgent(TestClient.options.agent); const message = {}; const file = { user: 'user1', @@ -3418,8 +3737,7 @@ describe('BaseClient', () => { }, }, }; - TestClient._mergedFileConfig = undefined; - TestClient._endpointFileConfig = undefined; + TestClient.options.agent = routedAgent(TestClient.options.agent); const message = {}; const file = { user: 'user1', diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index 4ae39e90a1..0176ad23ce 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -137,6 +137,7 @@ const { isAttachmentObjectNotFoundError, buildAgentScopedContext, buildAgentScopedAttachmentMap, + resolveScopedTurnAttachments, buildAgentContextAttachmentsByAgentId, buildSkillPrimeContentParts, buildInitialToolSessions, @@ -389,7 +390,7 @@ class AgentClient extends BaseClient { } } - async processAttachments(message, attachments) { + async processAttachments(message, attachments, fileConsumers) { const modelBoundAttachments = this.getModelBoundAttachmentsForEndpoint(attachments); const processableAttachments = this.getProcessableAttachmentsForEndpoint( attachments, @@ -409,7 +410,7 @@ class AgentClient extends BaseClient { }; logAgentMemorySnapshot('before_process_attachments', memoryContext); try { - return await super.processAttachments(message, processableAttachments); + return await super.processAttachments(message, processableAttachments, fileConsumers); } finally { logAgentMemorySnapshot('after_process_attachments', memoryContext); } @@ -2374,6 +2375,16 @@ class AgentClient extends BaseClient { ...modelBoundRequestAttachments, ]; const sharedRunAttachmentIds = collectFileIds(sharedAttachmentFiles); + this.options.agentContextAttachmentsByAgentId = resolveScopedTurnAttachments({ + agents: allAgents, + sharedConversationAgentIds: [this.options.agent.id, ...(this.agentConfigs?.keys() ?? [])], + resendFiles: this.options.resendFiles, + messages: orderedMessages, + historicalFiles: this.authorizedHistoricalFiles, + requestAttachments, + sharedRunAttachmentIds, + attachmentsByAgentId: this.options.agentContextAttachmentsByAgentId, + }); const scopedAttachmentMap = buildAgentScopedAttachmentMap({ agentIds: allAgents.map(({ agentId }) => agentId), attachmentsByAgentId: this.options.agentContextAttachmentsByAgentId, diff --git a/api/server/controllers/agents/client.test.js b/api/server/controllers/agents/client.test.js index 7a986a19cf..852487c2de 100644 --- a/api/server/controllers/agents/client.test.js +++ b/api/server/controllers/agents/client.test.js @@ -5471,6 +5471,9 @@ describe('AgentClient - titleConvo', () => { }, }; mockRes = {}; + mockAgent.deliveryRouting = jest + .requireActual('@librechat/api') + .resolveTurnDeliveryRouting({ agent: mockAgent, config: mockReq.config }); client = new AgentClient({ req: mockReq, @@ -6708,6 +6711,71 @@ describe('AgentClient - titleConvo', () => { }, ); + it.each(['current', 'history', 'history-disabled'])( + 'resolves %s tool-routed text only for an authorized handoff without a reader', + async (location) => { + const file = { + ...makeUploadedFile('fallback-file', 'sales.csv', 'text/csv'), + text: 'handoff fallback content', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + }; + const { resolveTurnDeliveryRouting } = jest.requireActual('@librechat/api'); + client.options.req.config.fileConfig = { + endpoints: { + default: { + defaultLLMDeliveryPath: { overrides: { 'text/csv': 'none' } }, + textFallbackWithoutTools: true, + }, + }, + }; + mockAgent.deliveryRouting = resolveTurnDeliveryRouting({ + agent: mockAgent, + config: client.options.req.config, + }); + mockAgent.fileConsumers = { executeCode: true, fileSearch: false }; + const handoffAgent = { + id: 'handoff-agent', + endpoint: EModelEndpoint.openAI, + provider: EModelEndpoint.openAI, + instructions: 'Handoff instructions', + model_parameters: { model: 'gpt-4' }, + tools: [], + deliveryRouting: mockAgent.deliveryRouting, + fileConsumers: { executeCode: false, fileSearch: false }, + }; + const isolatedAgent = { ...handoffAgent, id: 'isolated-agent' }; + mockAgent.subagentAgentConfigs = new Map([['isolated-agent', isolatedAgent]]); + client.agentConfigs = new Map([['handoff-agent', handoffAgent]]); + client.options.resendFiles = location !== 'history-disabled'; + client.options.attachments = location === 'current' ? [file] : []; + client.authorizedHistoricalFiles = new Map([[file.file_id, file]]); + client.message_file_map = {}; + const messages = [ + { + messageId: 'msg-1', + sender: 'User', + text: 'Read it', + isCreatedByUser: true, + ...(location !== 'current' ? { files: [{ file_id: file.file_id }] } : {}), + }, + ]; + const result = await client.buildMessages(messages, 'msg-1', {}); + expect(JSON.stringify(result.prompt)).not.toContain(file.text); + expect(mockAgent.additional_instructions ?? '').not.toContain(file.text); + expect(isolatedAgent.additional_instructions ?? '').not.toContain(file.text); + if (location === 'history-disabled') { + expect(handoffAgent.additional_instructions ?? '').not.toContain(file.text); + } else { + expect(handoffAgent.additional_instructions).toContain(file.text); + expect(client.turnScopedAttachmentsByAgentId.get('handoff-agent')).toEqual([ + { ...file, llmDeliveryPath: 'text' }, + ]); + } + expect(file.llmDeliveryPath).toBe('none'); + }, + ); + it('places request context inline and applies each agent context doc only once', async () => { const requestFile = makeTextFile('request-file', 'request.txt', 'Shared request context'); const primaryContext = makeTextFile( diff --git a/api/server/services/Endpoints/agents/skillDeps.js b/api/server/services/Endpoints/agents/skillDeps.js index c8aa0f2b30..7f695bbfea 100644 --- a/api/server/services/Endpoints/agents/skillDeps.js +++ b/api/server/services/Endpoints/agents/skillDeps.js @@ -291,10 +291,11 @@ function buildAgentToolContext({ agent, config }) { agent, fileEncodingAgent: { provider: config.provider, - endpoint: config.endpoint, model_parameters: config.model_parameters, imageDetail: config.imageDetail, agentContextAttachments: config.agentContextAttachments, + fileConsumers: config.fileConsumers, + deliveryRouting: config.deliveryRouting, }, /** Per-agent resolved endpoint token/pricing config. Retained here because * `agentToolContexts` is the one map that holds every agent — including diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js index 7ddf9ab729..05b02ae6df 100644 --- a/api/server/services/Files/process.js +++ b/api/server/services/Files/process.js @@ -36,7 +36,9 @@ const { assertExtractedTextInspectable, getFileExtractionLogDetails, getUploadExtractedTextPlan, + resolveUploadFallbackText, UPLOAD_EXTRACTED_TEXT_PLANS, + MAX_STORED_EXTRACTED_TEXT_BYTES, inspectContent, extractFileContent, hasActiveFileFieldPolicy, @@ -942,9 +944,9 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { }); } const textBytes = Buffer.byteLength(text, 'utf8'); - if (textBytes > 15 * megabyte) { + if (textBytes > MAX_STORED_EXTRACTED_TEXT_BYTES) { throw new Error( - `Extracted text from "${file.originalname}" exceeds the 15MB storage limit (${Math.round(textBytes / megabyte)}MB). Try a shorter document.`, + `Extracted text from "${file.originalname}" exceeds the ${MAX_STORED_EXTRACTED_TEXT_BYTES / megabyte}MB storage limit (${Math.round(textBytes / megabyte)}MB). Try a shorter document.`, ); } if ( @@ -1152,6 +1154,17 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { return await createTextFile({ text }); } + /* Extracted before storage, which may move the temporary upload the extractors read. */ + const fallbackText = await resolveUploadFallbackText({ + file, + fileId: file_id, + deliveryPath: llmDeliveryPath, + destinationChosen: uploadChoiceMetadata.destinationChosen, + isMessageAttachment: messageAttachment, + endpointConfig, + filters: appConfig?.filters, + }); + // Dual storage pattern for RAG files: Storage + Vector DB let storageResult, embeddingResult; let storedType = file.mimetype; @@ -1348,6 +1361,7 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { width, tenantId: req.user.tenantId, llmDeliveryPath, + text: fallbackText, }), ...retentionExpiry, }; diff --git a/api/server/services/Files/process.spec.js b/api/server/services/Files/process.spec.js index fbf1aca9b4..efbb8a1b00 100644 --- a/api/server/services/Files/process.spec.js +++ b/api/server/services/Files/process.spec.js @@ -125,6 +125,9 @@ jest.mock('@librechat/api', () => { /** Grants both; these specs vary the capability set, not the role. */ resolveToolRoleGrants: jest.fn(async () => ({ runCode: true, fileSearch: true })), parseText: jest.fn().mockResolvedValue({ text: '', bytes: 0 }), + /** Stores no fallback text unless a test opts in; its own rules are covered in packages/api. */ + resolveUploadFallbackText: jest.fn(async () => undefined), + MAX_STORED_EXTRACTED_TEXT_BYTES: 15 * 1024 * 1024, processAudioFile: jest.fn(), extractInspectableFileText: jest.fn(async ({ extract }) => extract()), assertExtractedTextInspectable: jest.fn(), @@ -3190,3 +3193,104 @@ describe('filterFile endpoint resolution', () => { expect(() => filterFile({ req, image: true, isAvatar: true })).not.toThrow(); }); }); + +describe('fallback text for uploads left to tools', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockRes.status.mockReturnThis(); + mockRes.json.mockReturnValue({}); + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + endpoints: { + 'Custom Provider': { + defaultLLMDeliveryPath: { fallback: 'none' }, + textFallbackWithoutTools: true, + }, + }, + }); + }); + + const uploadCsv = (metadata) => { + const req = makeReq({ mimetype: 'text/csv', ocrConfig: null }); + req.body.endpoint = EModelEndpoint.agents; + return { + req, + upload: processAgentFileUpload({ + req, + res: mockRes, + metadata: { + agent_id: 'agent-abc', + message_file: 'true', + file_id: 'file-uuid-csv', + effectiveEndpoint: 'Custom Provider', + ...metadata, + }, + }), + }; + }; + + test('stores the text a turn without a reading tool can fall back to', async () => { + const { createFile } = require('~/models'); + const { resolveUploadFallbackText } = require('@librechat/api'); + setupStoredFileUpload(); + resolveUploadFallbackText.mockResolvedValueOnce('region,total'); + + const { req, upload } = uploadCsv(); + await upload; + + expect(resolveUploadFallbackText).toHaveBeenCalledWith( + expect.objectContaining({ + file: req.file, + fileId: 'file-uuid-csv', + deliveryPath: 'none', + destinationChosen: false, + isMessageAttachment: true, + endpointConfig: expect.objectContaining({ textFallbackWithoutTools: true }), + }), + ); + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ llmDeliveryPath: 'none', text: 'region,total' }), + true, + ); + }); + + test('marks a legacy chooser upload as chosen, so no fallback text is extracted for it', async () => { + const { resolveUploadFallbackText } = require('@librechat/api'); + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + endpoints: { + 'Custom Provider': { + defaultLLMDeliveryPath: { fallback: 'none' }, + textFallbackWithoutTools: true, + legacyFileUploadUX: true, + }, + }, + }); + setupStoredFileUpload(); + + const { upload } = uploadCsv(); + await upload; + + expect(resolveUploadFallbackText).toHaveBeenCalledWith( + expect.objectContaining({ destinationChosen: true, isMessageAttachment: true }), + ); + }); + + test('stores fallback text for an attachment filed under a tool a later turn may not run', async () => { + const { createFile } = require('~/models'); + const { resolveUploadFallbackText } = require('@librechat/api'); + setupStoredFileUpload(); + resolveUploadFallbackText.mockResolvedValueOnce('region,total'); + + const { upload } = uploadCsv({ agentTools: [EToolResources.execute_code] }); + await upload; + + expect(resolveUploadFallbackText).toHaveBeenCalledWith( + expect.objectContaining({ destinationChosen: false, isMessageAttachment: true }), + ); + expect(createFile).toHaveBeenCalledWith( + expect.objectContaining({ llmDeliveryPath: 'none', text: 'region,total' }), + true, + ); + }); +}); diff --git a/librechat.example.yaml b/librechat.example.yaml index a114cd3057..dc77303721 100644 --- a/librechat.example.yaml +++ b/librechat.example.yaml @@ -1084,6 +1084,12 @@ endpoints: # tokens: 2000 # Also preserve up to this many recent tokens # fileConfig: +# # Store best-effort text for inferred tool-routed message attachments. On each turn, +# # deliver it only if no available tool can read the type. Off by default; also configurable +# # under endpoints.default or a provider/custom endpoint name. Explicit destinations stand. +# # Existing files without extracted text are not backfilled. Paused runs retain encoded text +# # and conservatively count referenced fallback text against current limits on resume. +# textFallbackWithoutTools: false # fileContextSizeLimit: 128 # Maximum aggregate model-bound attachment size in one agent turn (MB) # fileContextCharLimit: 1000000 # Maximum aggregate extracted-text characters in one agent turn # endpoints: diff --git a/packages/api/src/agents/__tests__/initialize.test.ts b/packages/api/src/agents/__tests__/initialize.test.ts index c67e45f03d..a6792435b8 100644 --- a/packages/api/src/agents/__tests__/initialize.test.ts +++ b/packages/api/src/agents/__tests__/initialize.test.ts @@ -133,6 +133,7 @@ jest.mock('../../middleware/modelBoundContent', () => { }); import { initializeAgent } from '../initialize'; +import { primeResources } from '../resources'; import { isFatalAgentInitializationError } from '../errors'; const realUtils = jest.requireActual('~/utils'); @@ -1655,7 +1656,7 @@ describe('initializeAgent — attachment scoping', () => { expect(db.updateFilesUsage).not.toHaveBeenCalled(); expect(primeResources).not.toHaveBeenCalled(); expect(loadTools).not.toHaveBeenCalled(); - expect(mockGetProviderConfig).not.toHaveBeenCalled(); + expect(mockGetProviderConfig).toHaveBeenCalledTimes(1); }); }); @@ -4499,3 +4500,352 @@ describe('initializeAgent — provider-native web search role gate', () => { expect(result.tools).toContainEqual(OPENAI_SEARCH); }); }); + +describe('initializeAgent tool-routed text fallback', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + const routedCsv = () => + ({ + file_id: 'csv-file', + filename: 'sales.csv', + type: 'text/csv', + text: 'region,total', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + }) as IMongoFile; + + async function initializeWith({ + tools, + csv, + textFallbackWithoutTools = true, + provider = Providers.OPENAI, + overrideProvider, + supportedMimeTypes, + softFailure = false, + fileSearchAvailable = true, + fileContextCharLimit, + useResponsesApi, + }: { + tools: string[]; + csv: IMongoFile; + textFallbackWithoutTools?: boolean; + provider?: string; + overrideProvider?: string; + supportedMimeTypes?: string[]; + softFailure?: boolean; + fileSearchAvailable?: boolean; + fileContextCharLimit?: number; + useResponsesApi?: boolean; + }) { + const { filterFilesByEndpointRuntimeConfig } = jest.requireMock('~/files') as { + filterFilesByEndpointRuntimeConfig: jest.Mock; + }; + const { agent, req, res, loadTools, db } = createMocks({ + provider, + overrideProvider, + loadedToolDefinitions: tools.includes(EToolResources.file_search) + ? [{ name: EToolResources.file_search }] + : [], + }); + if (useResponsesApi != null) { + mockGetProviderConfig.mockReturnValue({ + overrideProvider: overrideProvider ?? provider, + getOptions: jest + .fn() + .mockResolvedValue({ llmConfig: { model: agent.model, useResponsesApi } }), + }); + } + (primeResources as jest.Mock).mockImplementationOnce(async ({ attachments }) => { + const files = await attachments; + return { + attachments: files, + requestAttachments: files, + agentContextAttachments: [], + tool_resources: {}, + }; + }); + agent.tools = tools; + if (softFailure) loadTools.mockResolvedValue(undefined); + req.config = { + fileConfig: { + fileContextCharLimit, + endpoints: { + [provider]: { + defaultLLMDeliveryPath: { overrides: { 'text/csv': 'none' } }, + textFallbackWithoutTools, + ...(supportedMimeTypes != null && { supportedMimeTypes }), + }, + }, + }, + } as unknown as ServerRequest['config']; + (db.getFiles as jest.Mock).mockResolvedValueOnce([csv]); + filterFilesByEndpointRuntimeConfig.mockImplementationOnce( + (_config: ServerRequest['config'], { files }: { files: IMongoFile[] }) => files, + ); + + const result = await initializeAgent( + { + req, + res, + agent, + loadTools, + requestFiles: [csv], + codeEnvAvailable: true, + fileSearchAvailable, + endpointOption: { endpoint: EModelEndpoint.agents }, + allowedProviders: new Set([provider]), + isInitialAgent: true, + }, + db, + ); + return { result, filterFilesByEndpointRuntimeConfig }; + } + + it.each([true, false])( + 'admits Azure PDFs with the final Responses mode %s', + async (useResponsesApi) => { + const pdf = { ...routedCsv(), type: 'application/pdf', text: undefined }; + const { result, filterFilesByEndpointRuntimeConfig } = await initializeWith({ + tools: [], + csv: pdf, + provider: EModelEndpoint.azureOpenAI, + overrideProvider: Providers.OPENAI, + useResponsesApi, + }); + expect(result.deliveryRouting.useResponsesApi).toBe(useResponsesApi); + expect(filterFilesByEndpointRuntimeConfig).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + files: [{ ...pdf, llmDeliveryPath: useResponsesApi ? 'provider' : 'text' }], + }), + ); + }, + ); + + it('keeps fallback out of the prompt when file search loads successfully', async () => { + const csv = routedCsv(); + const { result } = await initializeWith({ tools: [EToolResources.file_search], csv }); + expect(result.fileConsumers).toEqual({ executeCode: false, fileSearch: true }); + expect(result.requestAttachments).toEqual([csv]); + }); + + it('falls back after file search soft-fails and admits the returned text copy', async () => { + const csv = routedCsv(); + const { result } = await initializeWith({ + tools: [EToolResources.file_search], + csv, + softFailure: true, + }); + expect(result.fileConsumers).toEqual({ executeCode: false, fileSearch: false }); + expect(result.requestAttachments).toEqual([{ ...csv, llmDeliveryPath: 'text' }]); + expect(csv.llmDeliveryPath).toBe('none'); + }); + + it('rejects fallback text that exceeds admission after a reader soft-fails', async () => { + await expect( + initializeWith({ + tools: [EToolResources.file_search], + csv: routedCsv(), + softFailure: true, + fileContextCharLimit: 3, + }), + ).rejects.toMatchObject({ name: 'AgentAttachmentLimitError' }); + }); + + it('hands endpoint filtering a text copy when the agent runs no tool that can read the file', async () => { + const csv = routedCsv(); + + const { result, filterFilesByEndpointRuntimeConfig } = await initializeWith({ tools: [], csv }); + + expect(filterFilesByEndpointRuntimeConfig).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ files: [{ ...csv, llmDeliveryPath: 'text' }] }), + ); + expect(result.fileConsumers).toEqual({ executeCode: false, fileSearch: false }); + expect(csv.llmDeliveryPath).toBe('none'); + }); + + it('hands endpoint filtering the provider route a tool-routed file takes on this turn', async () => { + /* Admission reads the route, so a record left for tools that this endpoint sends to the + * provider must reach filtering, limits and inspection as a provider file. */ + const image = { + file_id: 'image-file', + filename: 'chart.png', + type: 'image/png', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + } as IMongoFile; + + const { filterFilesByEndpointRuntimeConfig } = await initializeWith({ + tools: [], + csv: image, + }); + + expect(filterFilesByEndpointRuntimeConfig).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ files: [{ ...image, llmDeliveryPath: 'provider' }] }), + ); + expect(image.llmDeliveryPath).toBe('none'); + }); + + it('keeps media a custom endpoint opted into on its provider route before the provider swap', async () => { + /* The encoders send OpenAI-format media to a custom endpoint that lists the type, so the + * route has to read that endpoint's dialect from config, not the name still in `provider`. */ + const video = { + file_id: 'video-file', + filename: 'clip.mp4', + type: 'video/mp4', + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + } as IMongoFile; + + const { filterFilesByEndpointRuntimeConfig } = await initializeWith({ + tools: [], + csv: video, + provider: 'MyGateway', + overrideProvider: Providers.OPENAI, + supportedMimeTypes: ['video/mp4'], + }); + + expect(filterFilesByEndpointRuntimeConfig).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ files: [video] }), + ); + }); + + it('resolves a custom endpoint agent under the endpoint its upload was routed by', async () => { + /* Uploads resolve the agent's saved provider, which for a custom endpoint is its name. + * Initialization later swaps the provider for the backing client, so an opt-in set only + * on the custom endpoint must still be read under that name, here and after the swap. */ + const csv = routedCsv(); + + const { result, filterFilesByEndpointRuntimeConfig } = await initializeWith({ + tools: [], + csv, + provider: 'MyGateway', + overrideProvider: Providers.OPENAI, + }); + + expect(filterFilesByEndpointRuntimeConfig).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ files: [{ ...csv, llmDeliveryPath: 'text' }] }), + ); + expect(result.provider).toBe(Providers.OPENAI); + expect(result.endpoint).toBe('MyGateway'); + }); + + it('leaves the file on its tool route where the endpoint has not enabled the fallback', async () => { + const csv = routedCsv(); + + const { filterFilesByEndpointRuntimeConfig } = await initializeWith({ + tools: [], + csv, + textFallbackWithoutTools: false, + }); + + expect(filterFilesByEndpointRuntimeConfig).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ files: [csv] }), + ); + }); + + it('leaves the file to Run Code when the agent can run code', async () => { + const csv = routedCsv(); + + const { result, filterFilesByEndpointRuntimeConfig } = await initializeWith({ + tools: [EToolResources.execute_code], + csv, + }); + + expect(filterFilesByEndpointRuntimeConfig).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ files: [csv] }), + ); + expect(result.fileConsumers).toEqual({ executeCode: true, fileSearch: false }); + }); +}); + +describe('initializeAgent turn delivery routing', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('settles the routing once, after the provider swap and the Responses API decision', async () => { + const { agent, req, res, loadTools, db } = createMocks({ provider: 'MyClaude' }); + req.config = { + fileConfig: { endpoints: { MyClaude: { supportedMimeTypes: ['video/mp4'] } } }, + endpoints: { custom: [{ name: 'MyClaude', provider: 'anthropic' }] }, + } as unknown as ServerRequest['config']; + mockGetProviderConfig.mockReturnValue({ + overrideProvider: Providers.ANTHROPIC, + getOptions: jest + .fn() + .mockResolvedValue({ llmConfig: { model: 'test-model', useResponsesApi: true } }), + }); + + const result = await initializeAgent( + { + req, + res, + agent, + loadTools, + endpointOption: { endpoint: EModelEndpoint.agents }, + allowedProviders: new Set(['MyClaude']), + isInitialAgent: true, + }, + db, + ); + + expect(result.provider).toBe(Providers.ANTHROPIC); + expect(result.deliveryRouting).toMatchObject({ + endpoint: 'MyClaude', + endpointProvider: 'anthropic', + useResponsesApi: true, + sttConfigured: false, + }); + expect(result.deliveryRouting.endpointConfig.supportedMimeTypes).toEqual([/video\/mp4/]); + }); + + it('resolves the provider and its options before any attachment is loaded', async () => { + const { filterFilesByEndpointRuntimeConfig } = jest.requireMock('~/files') as { + filterFilesByEndpointRuntimeConfig: jest.Mock; + }; + const { agent, req, res, loadTools, db } = createMocks(); + const getOptions = jest.fn().mockResolvedValue({ llmConfig: { model: 'test-model' } }); + mockGetProviderConfig.mockReturnValue({ overrideProvider: Providers.OPENAI, getOptions }); + const file = { + file_id: 'file-1', + filename: 'notes.txt', + type: 'text/plain', + bytes: 10, + text: 'notes', + llmDeliveryPath: 'text', + } as IMongoFile; + (db.getFiles as jest.Mock).mockResolvedValueOnce([file]); + filterFilesByEndpointRuntimeConfig.mockImplementationOnce( + (_config: ServerRequest['config'], { files }: { files: IMongoFile[] }) => files, + ); + + await initializeAgent( + { + req, + res, + agent, + loadTools, + requestFiles: [file], + endpointOption: { endpoint: EModelEndpoint.agents }, + allowedProviders: new Set([Providers.OPENAI]), + isInitialAgent: true, + }, + db, + ); + + const [optionsOrder] = getOptions.mock.invocationCallOrder; + const [filesOrder] = (db.getFiles as jest.Mock).mock.invocationCallOrder; + const [toolsOrder] = loadTools.mock.invocationCallOrder; + expect(optionsOrder).toBeLessThan(filesOrder); + expect(filesOrder).toBeLessThan(toolsOrder); + }); +}); diff --git a/packages/api/src/agents/files/delivery.spec.ts b/packages/api/src/agents/files/delivery.spec.ts new file mode 100644 index 0000000000..fb5c490daf --- /dev/null +++ b/packages/api/src/agents/files/delivery.spec.ts @@ -0,0 +1,291 @@ +import type { TurnFileConsumers, TurnDeliveryFile } from 'librechat-data-provider'; +import { + applyTurnDelivery as materializeTurnDelivery, + resolveTurnDeliveryRouting, + resolveScopedTurnAttachments, +} from './delivery'; + +function applyTurnDelivery( + files: T[], + { + agent, + config, + consumers, + }: { + agent?: Parameters[0]['agent']; + config?: Parameters[0]['config']; + consumers?: TurnFileConsumers; + }, +) { + return materializeTurnDelivery(files, { + routing: agent ? resolveTurnDeliveryRouting({ agent, config }) : undefined, + consumers, + }); +} + +const config = { + fileConfig: { + endpoints: { + openAI: { + defaultLLMDeliveryPath: { overrides: { 'text/csv': 'none' as const } }, + textFallbackWithoutTools: true, + }, + }, + }, +}; +const noReader: TurnFileConsumers = { executeCode: false, fileSearch: false }; +const runsCode: TurnFileConsumers = { executeCode: true, fileSearch: false }; + +describe('resolveTurnDeliveryRouting', () => { + it('routes under the endpoint an agent names before its provider', () => { + expect(resolveTurnDeliveryRouting({ agent: { provider: 'openAI' }, config }).endpoint).toBe( + 'openAI', + ); + expect( + resolveTurnDeliveryRouting({ + agent: { provider: 'openAI', endpoint: 'Azure Foundry' }, + config, + }).endpoint, + ).toBe('Azure Foundry'); + }); + + it('reads a custom endpoint dialect as upload does, before and after the provider swap', () => { + /* Initialization first stores the endpoint name in both fields and only later replaces the + * provider with the backing client, so the dialect has to come from config either way. */ + const declared = { + ...config, + endpoints: { + custom: [{ name: 'MyClaude', provider: 'anthropic' }], + }, + } as Parameters[0]['config']; + const dialect = (agent: { provider: string; endpoint?: string }, routingConfig = declared) => + resolveTurnDeliveryRouting({ agent, config: routingConfig }).endpointProvider; + + expect(dialect({ provider: 'MyClaude', endpoint: 'MyClaude' })).toBe('anthropic'); + expect(dialect({ provider: 'anthropic', endpoint: 'MyClaude' })).toBe('anthropic'); + expect(dialect({ provider: 'MyGateway', endpoint: 'MyGateway' }, config)).toBeUndefined(); + expect(dialect({ provider: 'openAI', endpoint: 'MyGateway' }, config)).toBeUndefined(); + }); + + it('carries the agent Responses API choice into routing', () => { + expect( + resolveTurnDeliveryRouting({ + agent: { provider: 'openAI', model_parameters: { useResponsesApi: true } }, + config, + }).useResponsesApi, + ).toBe(true); + }); +}); + +describe('applyTurnDelivery', () => { + const agent = { provider: 'openAI' }; + const csv = { + file_id: 'csv', + type: 'text/csv', + text: 'region,total\nwest,4', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + }; + const pdf = { file_id: 'pdf', type: 'application/pdf', llmDeliveryPath: 'provider' }; + + it('marks a copy of each file this turn delivers as text, leaving the rest untouched', () => { + const result = applyTurnDelivery([csv, pdf], { agent, config, consumers: noReader }); + + expect(result[0]).toEqual({ ...csv, llmDeliveryPath: 'text' }); + expect(result[0]).not.toBe(csv); + expect(result[1]).toBe(pdf); + expect(csv.llmDeliveryPath).toBe('none'); + }); + + it('returns the same array when a tool this turn runs can read every file', () => { + const files = [csv, pdf]; + + expect(applyTurnDelivery(files, { agent, config, consumers: runsCode })).toBe(files); + }); + + it('marks nothing where the endpoint has not enabled the fallback', () => { + const files = [csv]; + const disabled = { + fileConfig: { + endpoints: { + openAI: { defaultLLMDeliveryPath: { overrides: { 'text/csv': 'none' as const } } }, + }, + }, + }; + + expect(applyTurnDelivery(files, { agent, config: disabled, consumers: noReader })).toBe(files); + }); + + it('reads the opt-in under the custom endpoint an initialized agent names', () => { + /* After initialization the provider is the backing client and the endpoint keeps the name + * the upload resolved, so a setting made only on that endpoint still applies. */ + const files = [csv]; + const customOnly = { + fileConfig: { + endpoints: { + MyGateway: { + defaultLLMDeliveryPath: { overrides: { 'text/csv': 'none' as const } }, + textFallbackWithoutTools: true, + }, + }, + }, + }; + + expect( + applyTurnDelivery(files, { + agent: { provider: 'openAI', endpoint: 'MyGateway' }, + config: customOnly, + consumers: noReader, + }), + ).toEqual([{ ...csv, llmDeliveryPath: 'text' }]); + }); + + it('marks a stored tool-routed file the endpoint now routes to text, with the fallback off', () => { + /* Only this mark lets the admission checks and `extractFileContext`, which read the stored + * route, see the text the resolver now delivers. */ + const rerouted = { + fileConfig: { + endpoints: { + openAI: { defaultLLMDeliveryPath: { overrides: { 'text/csv': 'text' as const } } }, + }, + }, + }; + + expect(applyTurnDelivery([csv], { agent, config: rerouted, consumers: runsCode })).toEqual([ + { ...csv, llmDeliveryPath: 'text' }, + ]); + expect(applyTurnDelivery([csv], { agent, config: rerouted })).toEqual([ + { ...csv, llmDeliveryPath: 'text' }, + ]); + }); + + it('falls back only for a turn whose tools are known', () => { + const files = [csv]; + + expect(applyTurnDelivery(files, { agent, config })).toBe(files); + }); + + it('marks nothing without an agent to route by', () => { + const files = [csv]; + + expect(applyTurnDelivery(files, { config, consumers: noReader })).toBe(files); + }); + + it('leaves a record predating routing to its legacy handling', () => { + const files = [{ file_id: 'legacy', type: 'text/csv', text: 'region,total' }]; + + expect(applyTurnDelivery(files, { agent, config, consumers: noReader })).toBe(files); + }); + + it('does not mark a tool-routed file that stored no text', () => { + const files = [{ ...csv, text: undefined }]; + + expect(applyTurnDelivery(files, { agent, config, consumers: noReader })).toBe(files); + }); + + it('gives a stored tool-routed file the provider route this turn sends it by', () => { + /* Admission would otherwise skip a record the client then encodes for the provider. */ + const image = { + file_id: 'image', + type: 'image/png', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + }; + + expect(applyTurnDelivery([image], { agent, config, consumers: noReader })).toEqual([ + { ...image, llmDeliveryPath: 'provider' }, + ]); + }); + + it('removes a record this turn leaves to tools from model admission', () => { + /* Over-admitting cannot pass a limit; dropping a record the client still sends would. */ + const files = [{ ...csv, llmDeliveryPath: 'text' }]; + + expect(applyTurnDelivery(files, { agent, config, consumers: runsCode })).toEqual([ + { ...csv, llmDeliveryPath: 'none' }, + ]); + }); + + it('materializes a final text route even without stored text', () => { + const files = [{ ...pdf, metadata: { destinationChosen: false } }]; + const pdfToText = { + fileConfig: { + endpoints: { + openAI: { defaultLLMDeliveryPath: { overrides: { 'application/pdf': 'text' as const } } }, + }, + }, + }; + + expect(applyTurnDelivery(files, { agent, config: pdfToText, consumers: noReader })).toEqual( + files.map((file) => ({ ...file, llmDeliveryPath: 'text' })), + ); + }); + + it('does not mark a destination the user chose', () => { + const files = [{ ...csv, metadata: { destinationChosen: true } }]; + + expect(applyTurnDelivery(files, { agent, config, consumers: noReader })).toBe(files); + }); +}); + +describe('resolveScopedTurnAttachments', () => { + const file = { + file_id: 'csv', + type: 'text/csv', + text: 'sales,total', + llmDeliveryPath: 'none' as const, + metadata: { destinationChosen: false }, + }; + const routing = resolveTurnDeliveryRouting({ agent: { provider: 'openAI' }, config }); + const receiver = { + agentId: 'handoff', + agent: { deliveryRouting: routing, fileConsumers: noReader }, + }; + + it.each([ + ['explicit destination', { ...file, metadata: { destinationChosen: true } }, routing], + ['legacy destination', { ...file, llmDeliveryPath: undefined, metadata: undefined }, routing], + [ + 'disabled fallback', + file, + resolveTurnDeliveryRouting({ + agent: { provider: 'openAI' }, + config: { + fileConfig: { + endpoints: { + openAI: { ...config.fileConfig.endpoints.openAI, textFallbackWithoutTools: false }, + }, + }, + }, + }), + ], + ])('preserves %s when a receiver has no reader', (_name, candidate, deliveryRouting) => { + expect( + resolveScopedTurnAttachments({ + agents: [{ ...receiver, agent: { ...receiver.agent, deliveryRouting } }], + sharedConversationAgentIds: ['handoff'], + messages: [], + requestAttachments: [candidate], + sharedRunAttachmentIds: new Set(), + }).get('handoff'), + ).toEqual([]); + }); + + it('ignores unhydrated and no-longer-retained history and deduplicates shared prompt files', () => { + const stale = { ...file, file_id: 'stale' }; + const unhydrated = { file_id: 'unhydrated', text: 'untrusted' }; + expect( + resolveScopedTurnAttachments({ + agents: [receiver], + sharedConversationAgentIds: ['handoff'], + messages: [{ files: [unhydrated, { file_id: 'csv' }] }], + historicalFiles: new Map([ + [file.file_id, file], + [stale.file_id, stale], + ]), + requestAttachments: [file], + sharedRunAttachmentIds: new Set(['csv']), + }).get('handoff'), + ).toEqual([]); + }); +}); diff --git a/packages/api/src/agents/files/delivery.ts b/packages/api/src/agents/files/delivery.ts new file mode 100644 index 0000000000..97dc85e80a --- /dev/null +++ b/packages/api/src/agents/files/delivery.ts @@ -0,0 +1,153 @@ +import { + mergeFileConfig, + getEndpointFileConfig, + resolveUseResponsesApi, + getCustomEndpointProvider, + isSpeechProviderConfigured, + resolveTurnLLMDeliveryPath, + hasInferredLLMDeliveryPath, +} from 'librechat-data-provider'; +import type { + TurnDeliveryRouting, + TurnDeliveryFile, + TurnFileConsumers, +} from 'librechat-data-provider'; +import type { AppConfig } from '@librechat/data-schemas'; +import { collectModelBoundHistoricalFileIdState } from '~/middleware/modelBoundContent'; + +/** The app config a turn's attachment routing reads. */ +export type TurnDeliveryConfig = Pick; + +/** The fields of an agent that route its attachments. */ +export interface TurnDeliveryAgent { + provider: string; + /** The endpoint name initialization records before it swaps `provider` for the backing + * client; an agent loaded without one is routed by its provider. */ + endpoint?: string | null; + /** The Responses API setting the turn runs on, once initialization has decided it. */ + model_parameters?: { useResponsesApi?: boolean } | null; +} + +/** + * Settles how the agent running a turn receives its attachments. + * + * Read once, after initialization has resolved the backing provider and the Responses API + * decision: the file policy is the one configured under the endpoint's own name, the media + * dialect is the one its config declares rather than the client family it runs as, and the + * Responses setting is the one the model call uses. Every reader of a turn route consumes the + * returned value, so delivery, steering and child run-file encoding cannot answer differently. + */ +export function resolveTurnDeliveryRouting({ + agent, + config, +}: { + agent: TurnDeliveryAgent; + config?: TurnDeliveryConfig; +}): TurnDeliveryRouting { + const endpoint = agent.endpoint ?? agent.provider; + const fileConfig = mergeFileConfig(config?.fileConfig); + return { + fileConfig, + endpointConfig: getEndpointFileConfig({ fileConfig, endpoint }), + endpoint, + endpointProvider: getCustomEndpointProvider(config?.endpoints?.custom, endpoint), + useResponsesApi: resolveUseResponsesApi(agent.model_parameters?.useResponsesApi), + sttConfigured: isSpeechProviderConfigured(config?.speech?.stt), + }; +} + +/** Materialize the exact turn route before admission without mutating stored records. */ +export function applyTurnDelivery( + files: T[], + { routing, consumers }: { routing?: TurnDeliveryRouting; consumers?: TurnFileConsumers }, +): T[] { + if (routing == null) { + return files; + } + let changed = false; + const result = files.map((file) => { + if (!hasInferredLLMDeliveryPath(file)) { + return file; + } + const llmDeliveryPath = resolveTurnLLMDeliveryPath(routing, file, consumers); + if (llmDeliveryPath == null || llmDeliveryPath === file.llmDeliveryPath) { + return file; + } + changed = true; + return { ...file, llmDeliveryPath }; + }); + return changed ? result : files; +} + +/** + * Checkpoints replay already encoded content; current tools and policy cannot remove it. + * A referenced inferred tool file carrying extracted text may have been delivered as fallback. + * Charge that text conservatively even if fallback was since disabled. Explicit destinations + * and records predating routing keep their existing accounting; no stored record is changed. + */ +export function applyCheckpointDelivery(files: T[]): T[] { + return files.map((file) => + hasInferredLLMDeliveryPath(file) && file.llmDeliveryPath === 'none' && file.text + ? { ...file, llmDeliveryPath: 'text' } + : file, + ); +} + +/** + * A primary agent's tool route cannot decide whether a handoff receives text. Keep + * owner-hydrated candidates until each receiver resolves them, then add only text absent + * from the shared prompt to its scoped context. The existing scoped-context pipeline owns + * endpoint filtering, aggregate admission, inspection and extraction for these copies. + */ +export function resolveScopedTurnAttachments({ + agents, + sharedConversationAgentIds, + resendFiles, + messages, + historicalFiles, + requestAttachments, + sharedRunAttachmentIds, + attachmentsByAgentId, +}: { + agents: readonly { + agentId: string; + agent: { + deliveryRouting?: TurnDeliveryRouting; + fileConsumers?: TurnFileConsumers; + }; + }[]; + /** Only the primary/handoff graph shares root conversation files. */ + sharedConversationAgentIds: readonly string[]; + resendFiles?: boolean; + messages: Parameters[0]; + historicalFiles?: ReadonlyMap; + requestAttachments: readonly T[]; + sharedRunAttachmentIds: ReadonlySet; + attachmentsByAgentId?: Map | Record; +}): Map { + const candidates = new Map(); + for (const fileId of collectModelBoundHistoricalFileIdState(resendFiles === false ? [] : messages) + .fileIds) { + const file = historicalFiles?.get(fileId); + if (file && !sharedRunAttachmentIds.has(fileId)) candidates.set(fileId, file); + } + for (const file of requestAttachments) { + if (!sharedRunAttachmentIds.has(file.file_id)) candidates.set(file.file_id, file); + } + const sharedAgents = new Set(sharedConversationAgentIds); + const result = new Map(); + for (const { agentId, agent } of agents) { + const scoped = + attachmentsByAgentId instanceof Map + ? (attachmentsByAgentId.get(agentId) ?? []) + : (attachmentsByAgentId?.[agentId] ?? []); + const files = new Map(scoped.map((file) => [file.file_id, file])); + for (const [fileId, file] of sharedAgents.has(agentId) ? candidates : []) { + if (files.has(fileId)) continue; + const path = resolveTurnLLMDeliveryPath(agent.deliveryRouting, file, agent.fileConsumers); + if (path === 'text' && file.text) files.set(fileId, { ...file, llmDeliveryPath: path }); + } + result.set(agentId, [...files.values()]); + } + return result; +} diff --git a/packages/api/src/agents/files/encode.spec.ts b/packages/api/src/agents/files/encode.spec.ts index 753db6d098..f0982f7ac7 100644 --- a/packages/api/src/agents/files/encode.spec.ts +++ b/packages/api/src/agents/files/encode.spec.ts @@ -3,6 +3,7 @@ import type { TFile } from 'librechat-data-provider'; import type { RunFileEncodingAgent, RunFileMessageEncoderDeps } from './encode'; import type { ServerRequest } from '~/types'; import { AgentAttachmentLimitError, AgentAttachmentPolicyError } from '../attachments'; +import { resolveTurnDeliveryRouting } from './delivery'; import { createRunFileMessageEncoder } from './encode'; jest.mock('~/utils/tokenizer', () => ({ countTokens: (text: string) => text.length })); @@ -28,14 +29,32 @@ const nativeDocument = { }; const nativeImage = { type: 'image_url', image_url: { url: 'data:image/png;base64,aW1n' } }; +/** A child as the host loads it, before initialization settles its delivery routing. */ +type LoadedAgent = Omit & { + endpoint?: string; + useResponsesApi?: boolean; +}; + function setup({ fileConfig = {}, agents = { child: { provider: 'openAI' } }, }: { fileConfig?: NonNullable['fileConfig']; - agents?: Record; + agents?: Record; } = {}) { const req = { body: {}, config: { fileConfig } } as ServerRequest; + const initialized = Object.fromEntries( + Object.entries(agents).map(([id, { endpoint, useResponsesApi, ...agent }]) => [ + id, + { + ...agent, + deliveryRouting: resolveTurnDeliveryRouting({ + agent: { provider: agent.provider, endpoint, model_parameters: { useResponsesApi } }, + config: req.config, + }), + }, + ]), + ); const encodeImages = jest.fn(async () => ({ image_urls: [nativeImage] })); const encodeDocuments = jest.fn(async () => ({ documents: [nativeDocument] })); const encodeAudios = jest.fn(async () => ({ audios: [{ type: 'media', data: 'audio' }] })); @@ -46,7 +65,7 @@ function setup({ >(extractFileContext); const deps: RunFileMessageEncoderDeps = { req, - getAgent: (id) => agents[id], + getAgent: (id) => initialized[id], encodeImages, encodeDocuments, encodeAudios, @@ -65,7 +84,7 @@ describe('createRunFileMessageEncoder', () => { provider: 'openAI', endpoint: 'child-provider', model: 'child-model', - model_parameters: { useResponsesApi: true }, + useResponsesApi: true, imageDetail: ImageDetail.high, }, }, @@ -138,6 +157,49 @@ describe('createRunFileMessageEncoder', () => { expect(pdf.llmDeliveryPath).toBe('provider'); }); + it('delivers stored text only to a child that runs no tool able to read a tool-routed file', async () => { + const csv: TFile = { + ...pdf, + file_id: 'input-csv', + filename: 'sales.csv', + filepath: '/files/sales.csv', + type: 'text/csv', + text: 'region,total\nwest,4', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + }; + const harness = setup({ + agents: { + noReader: { provider: 'openAI', fileConsumers: { executeCode: false, fileSearch: false } }, + runsCode: { provider: 'openAI', fileConsumers: { executeCode: true, fileSearch: false } }, + unknown: { provider: 'openAI' }, + }, + fileConfig: { + endpoints: { + openAI: { + defaultLLMDeliveryPath: { overrides: { 'text/csv': 'none' } }, + textFallbackWithoutTools: true, + }, + }, + }, + }); + + const [noReader, runsCode, unknown] = await Promise.all([ + harness.encode([csv], 'noReader'), + harness.encode([csv], 'runsCode'), + harness.encode([csv], 'unknown'), + ]); + + expect(JSON.stringify(noReader[0].content)).toContain('region,total'); + expect(runsCode).toEqual([]); + expect(unknown).toEqual([]); + expect(harness.extractText).toHaveBeenCalledWith( + expect.objectContaining({ attachments: [{ ...csv, llmDeliveryPath: 'text' }] }), + ); + expect(harness.encodeDocuments).not.toHaveBeenCalled(); + expect(csv.llmDeliveryPath).toBe('none'); + }); + it('honors an explicit tool destination even when the receiving endpoint supports native files', async () => { const harness = setup(); const file: TFile = { diff --git a/packages/api/src/agents/files/encode.ts b/packages/api/src/agents/files/encode.ts index d4de921577..a5312c5299 100644 --- a/packages/api/src/agents/files/encode.ts +++ b/packages/api/src/agents/files/encode.ts @@ -3,14 +3,15 @@ import { HumanMessage } from '@librechat/agents/langchain'; import { FileSources, EModelEndpoint, - mergeFileConfig, - getEndpointFileConfig, isBedrockDocumentType, - resolveUseResponsesApi, - isSpeechProviderConfigured, - resolveUploadLLMDeliveryPath, + resolveTurnLLMDeliveryPath, +} from 'librechat-data-provider'; +import type { + TFile, + ImageDetail, + TurnDeliveryRouting, + TurnFileConsumers, } from 'librechat-data-provider'; -import type { TFile, ImageDetail } from 'librechat-data-provider'; import type { BaseMessage } from '@librechat/agents/langchain'; import type { ServerRequest, StrategyFunctions } from '~/types'; import type { TokenCountFn } from '~/utils/text'; @@ -28,11 +29,13 @@ type ContentBlock = Exclude[number]; /** The already loaded child configuration; storage documents never cross this boundary. */ export interface RunFileEncodingAgent { provider: string; - endpoint?: string | null; model?: string | null; - model_parameters?: { model?: string; useResponsesApi?: boolean }; + model_parameters?: { model?: string }; imageDetail?: ImageDetail; agentContextAttachments?: readonly TFile[]; + /** How the child receives attachments, settled when its configuration was initialized. */ + deliveryRouting: TurnDeliveryRouting; + fileConsumers?: TurnFileConsumers; } export interface RunFileEncodingParams { @@ -80,34 +83,26 @@ export function createRunFileMessageEncoder( if (!agent) { throw new Error('The target agent is not available for shared file delivery.'); } - const endpoint = agent.endpoint ?? agent.provider; - const fileConfig = mergeFileConfig(deps.req.config?.fileConfig); - const endpointConfig = getEndpointFileConfig({ fileConfig, endpoint }); - const useResponsesApi = resolveUseResponsesApi(agent.model_parameters?.useResponsesApi); + const { deliveryRouting } = agent; + const { endpoint, fileConfig, endpointConfig } = deliveryRouting; const params: RunFileEncodingParams = { provider: agent.provider, endpoint, model: agent.model_parameters?.model ?? agent.model ?? undefined, - useResponsesApi, + useResponsesApi: deliveryRouting.useResponsesApi, imageDetail: agent.imageDetail, }; const resolveDelivery = (file: TFile): TFile => { - if (file.llmDeliveryPath == null || file.metadata?.destinationChosen === true) { + const llmDeliveryPath = resolveTurnLLMDeliveryPath( + deliveryRouting, + file, + agent.fileConsumers, + ); + if (llmDeliveryPath == null || llmDeliveryPath === file.llmDeliveryPath) { return file; } - return { - ...file, - llmDeliveryPath: resolveUploadLLMDeliveryPath({ - mimeType: file.metadata?.routingMimeType ?? file.type, - endpointConfig, - fileConfig, - endpoint, - endpointProvider: agent.provider, - useResponsesApi, - sttConfigured: isSpeechProviderConfigured(deps.req.config?.speech?.stt), - }), - }; + return { ...file, llmDeliveryPath }; }; const sharedFiles = files.map(resolveDelivery); const compatibleFiles = filterFilesByEndpointRuntimeConfig(deps.req.config, { diff --git a/packages/api/src/agents/files/index.ts b/packages/api/src/agents/files/index.ts index 9293cd7f4e..0bdab2ea81 100644 --- a/packages/api/src/agents/files/index.ts +++ b/packages/api/src/agents/files/index.ts @@ -3,3 +3,4 @@ export * from './session'; export * from './host'; export * from './binding'; export * from './encode'; +export * from './delivery'; diff --git a/packages/api/src/agents/files/session.spec.ts b/packages/api/src/agents/files/session.spec.ts index ad786df7cc..9a01735a09 100644 --- a/packages/api/src/agents/files/session.spec.ts +++ b/packages/api/src/agents/files/session.spec.ts @@ -7,6 +7,7 @@ import type { RunFileSessionDeps } from './session'; import type { ServerRequest } from '~/types'; import { createRunFileSession, getAuthorizedRunFileSnapshot } from './session'; import { AgentAttachmentLimitError } from '../attachments'; +import { resolveTurnDeliveryRouting } from './delivery'; import { createRunFileMessageEncoder } from './encode'; function setup( @@ -219,6 +220,10 @@ it.each([{ totalSizeLimit: 1 }, { fileLimit: 1 }])( getAgent: (id) => ({ provider: 'openAI', agentContextAttachments: id === 'writer' ? writerAttachments : [], + deliveryRouting: resolveTurnDeliveryRouting({ + agent: { provider: 'openAI' }, + config: { fileConfig }, + }), }), encodeDocuments, encodeImages: async () => ({ image_urls: [] }), diff --git a/packages/api/src/agents/hitl/protection.spec.ts b/packages/api/src/agents/hitl/protection.spec.ts index 6c7c194915..26c877cdc9 100644 --- a/packages/api/src/agents/hitl/protection.spec.ts +++ b/packages/api/src/agents/hitl/protection.spec.ts @@ -3,6 +3,11 @@ import type { ResumeContentProtectionDependencies, ResumeRuntimeContentProtectionDependencies, } from './protection'; +import { + assertAgentAttachmentLimits, + AgentAttachmentLimitError, + isModelBoundAttachmentFile, +} from '../attachments'; import { assertResumeContentAllowed, assertResumeRuntimeContentAllowed } from './protection'; const user = { @@ -99,6 +104,48 @@ describe('assertResumeRuntimeContentAllowed', () => { }, ); + it('charges retained fallback text after settings change, using owner-hydrated records', async () => { + const dependencies = createDependencies(); + dependencies.getAgentCheckpointer.mockResolvedValue({ + getTuple: jest.fn().mockResolvedValue({ + checkpoint: { + channel_values: { + messages: [{ role: 'human', additional_kwargs: { sourceMessageId: 'source-message' } }], + }, + }, + }), + }); + dependencies.getMessages.mockResolvedValue([ + { messageId: 'source-message', files: [{ file_id: 'fallback-file' }] }, + ]); + const storedFile = { + file_id: 'fallback-file', + filename: 'sales.csv', + type: 'text/csv', + source: 'local', + bytes: 20, + text: 'region,total', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + }; + dependencies.getFiles.mockResolvedValue([storedFile]); + const input = { + ...createInput({ fileConfig: { textFallbackWithoutTools: false } }), + isTemporary: false, + }; + const projection = await assertResumeRuntimeContentAllowed(input, dependencies); + expect(projection.checkpointFiles).toEqual([{ ...storedFile, llmDeliveryPath: 'text' }]); + expect(storedFile.llmDeliveryPath).toBe('none'); + expect(() => + assertAgentAttachmentLimits({ + attachments: projection.checkpointFiles.filter(isModelBoundAttachmentFile), + fileConfig: { fileContextCharLimit: 3 }, + endpoint: 'openAI', + }), + ).toThrow(AgentAttachmentLimitError); + expect(dependencies.getFiles).toHaveBeenCalledTimes(1); + }); + it('hydrates checkpoint-bound files when content filters are inactive', async () => { const dependencies = createDependencies(); dependencies.getAgentCheckpointer.mockResolvedValue({ diff --git a/packages/api/src/agents/hitl/protection.ts b/packages/api/src/agents/hitl/protection.ts index 500e7b3126..7ba81fed60 100644 --- a/packages/api/src/agents/hitl/protection.ts +++ b/packages/api/src/agents/hitl/protection.ts @@ -58,6 +58,7 @@ import { LIBRECHAT_CHECKPOINT_NAMESPACE_KEY } from '../checkpointer'; import { AttachmentObjectNotFoundError } from '~/files/encode/utils'; import { ASK_USER_QUESTION_TOOL_NAME } from './askUserQuestionTool'; import { ContentFilterError } from '~/middleware/contentFilter'; +import { applyCheckpointDelivery } from '../files/delivery'; import { hasActiveFilePolicy } from '~/protection/files'; import { parseSkillMarkdown } from '../../skills/parse'; import { inspectContent } from '~/protection/runtime'; @@ -1137,7 +1138,7 @@ export async function assertResumeRuntimeContentAllowed( ); throw new AttachmentObjectNotFoundError(unresolvedFileId ?? 'unknown'); } - return { resolvedFiles, checkpointFiles }; + return { resolvedFiles, checkpointFiles: applyCheckpointDelivery(checkpointFiles) }; } export function getUserFacingResumeError( diff --git a/packages/api/src/agents/initialize.ts b/packages/api/src/agents/initialize.ts index 52ab19d279..7f9cc7391a 100644 --- a/packages/api/src/agents/initialize.ts +++ b/packages/api/src/agents/initialize.ts @@ -21,10 +21,12 @@ import type { TEndpointOption, ReasoningResponseKey, StatefulCodeEnvironment, + TurnDeliveryRouting, ImageDetail, TFile, Agent, TUser, + TurnFileConsumers, } from 'librechat-data-provider'; import type { GenericTool, LCToolRegistry, ToolMap, LCTool } from '@librechat/agents'; import type { IMongoFile, FileOwnerScope } from '@librechat/data-schemas'; @@ -113,10 +115,12 @@ import { applyIntentLabels, sanitizeIntentLabels } from './intent'; import { ContentFilterError } from '../middleware/contentFilter'; import { resolveToolRoleGrants } from '~/tools/rolePermissions'; import { createRequestAgentExecutionContext } from './runtime'; +import { resolveTurnDeliveryRouting } from './files/delivery'; import { filterFilesByEndpointRuntimeConfig } from '~/files'; import { hasActiveFileFieldPolicy } from '~/protection'; import { PARTIAL_RESOLVED_CONVERSATION } from './guard'; import { applyBackgroundToolCalls } from './background'; +import { applyTurnDelivery } from './files/delivery'; import { generateArtifactsPrompt } from '~/prompts'; import { getProviderConfig } from '~/endpoints'; import { primeResources } from './resources'; @@ -616,6 +620,8 @@ export type InitializedAgent = Agent & { currentRequestAttachments: TFile[]; /** Files attached to this agent's permanent context via tool_resources. */ agentContextAttachments: IMongoFile[]; + /** File-reading tools this turn runs, which decide when a tool-routed file falls back to text. */ + fileConsumers?: TurnFileConsumers; toolContextMap: Record; dynamicToolContextMap?: Record; maxContextTokens: number; @@ -626,6 +632,8 @@ export type InitializedAgent = Agent & { /** Detail level LibreChat encodes image content blocks with, from the agent's * model parameters. Absent when the agent does not configure one. */ imageDetail?: ImageDetail; + /** How this agent receives its attachments this turn, settled once every routing input is final. */ + deliveryRouting: TurnDeliveryRouting; tool_resources?: AgentToolResources; userMCPAuthMap?: Record>; /** Tool map for ToolNode to use when executing tools (required for PTC) */ @@ -1282,6 +1290,88 @@ export async function initializeAgent( const provider = agent.provider; agent.endpoint = provider; + /** Settle the provider and its client options before any attachment is judged. The file + * policy reads the endpoint's own name, but the route each attachment takes also depends on + * the backing client and on the Responses API decision `getOptions` makes, and nothing + * between here and tool loading feeds either. */ + const { getOptions, overrideProvider, customEndpointConfig } = getProviderConfig({ + provider, + appConfig, + }); + if (overrideProvider !== agent.provider) { + agent.provider = overrideProvider; + } + + const finalModelOptions = { + ...modelOptions, + model: agent.model, + }; + + const options: InitializeResultBase = await getOptions({ + runtime: { + appConfig, + user, + requestBody: runtime.requestBody, + }, + endpoint: provider, + model_parameters: finalModelOptions, + db, + }); + + const llmConfig = options.llmConfig as Record; + const webSearchDenied = + hasProviderWebSearch(options.tools, llmConfig) && + !(await resolveWebSearchGrant({ + req: params.req as Request | undefined, + user, + resolve: params.resolveWebSearchGrant, + getRoleByName: db.getRoleByName, + })); + if (webSearchDenied && stripWebSearchPlugin(llmConfig) > 0) { + logger.debug( + `[initializeAgent] Removed the OpenRouter web search plugin; role denies WEB_SEARCH.`, + ); + } + const tokensModel = + agent.provider === EModelEndpoint.azureOpenAI ? agent.model : (llmConfig?.model as string); + const maxOutputTokens = optionalChainWithEmptyCheck( + llmConfig?.maxOutputTokens as number | undefined, + llmConfig?.maxTokens as number | undefined, + 0, + ); + const agentMaxContextTokens = optionalChainWithEmptyCheck( + maxContextTokens, + getModelMaxTokens( + tokensModel ?? '', + providerEndpointMap[overrideProvider as keyof typeof providerEndpointMap], + options.endpointTokenConfig, + ), + DEFAULT_MAX_CONTEXT_TOKENS, + ); + + if ( + agent.endpoint === EModelEndpoint.azureOpenAI && + (llmConfig?.azureOpenAIApiInstanceName as string | undefined) == null + ) { + agent.provider = Providers.OPENAI; + } + + if (options.provider != null) { + agent.provider = options.provider; + } + + const deliveryRouting = resolveTurnDeliveryRouting({ + agent: { + provider: agent.provider, + endpoint: agent.endpoint, + model_parameters: { + useResponsesApi: + typeof llmConfig.useResponsesApi === 'boolean' ? llmConfig.useResponsesApi : undefined, + }, + }, + config: appConfig, + }); + /** Resolve the per-agent Code API route before resource/tool priming. A * stateful agent must perform freshness checks and recovery uploads against * the same isolated deployment its eventual `/exec` request will use. */ @@ -1371,6 +1461,10 @@ export async function initializeAgent( * that lookup runs whichever way the setting is configured. */ const wantsCodeFiles = toolResourceSet.has(EToolResources.execute_code); const wantsSearchFiles = toolResourceSet.has(EToolResources.file_search); + const fileConsumers: TurnFileConsumers = { + executeCode: wantsCodeFiles, + fileSearch: wantsSearchFiles, + }; const wantsProvisioning = wantsCodeFiles || wantsSearchFiles; if ( @@ -1553,6 +1647,14 @@ export async function initializeAgent( } else if (requestFiles.length > 0 || toolFileIds.length > 0) { currentFiles = requestUsageFiles.concat(toolUsageFiles); } + if (currentFiles?.length) { + /* Before any check reads the route: endpoint filtering, model-bound limits and content + * inspection below all have to judge each file by the route this turn delivers it by. */ + currentFiles = applyTurnDelivery(currentFiles, { + routing: deliveryRouting, + consumers: fileConsumers, + }); + } let endpointFileType: EModelEndpoint | undefined; if (!paramEndpoints.has(agent.endpoint ?? '')) { @@ -1917,72 +2019,6 @@ export async function initializeAgent( } } - const { getOptions, overrideProvider, customEndpointConfig } = getProviderConfig({ - provider, - appConfig, - }); - if (overrideProvider !== agent.provider) { - agent.provider = overrideProvider; - } - - const finalModelOptions = { - ...modelOptions, - model: agent.model, - }; - - const options: InitializeResultBase = await getOptions({ - runtime: { - appConfig, - user, - requestBody: runtime.requestBody, - }, - endpoint: provider, - model_parameters: finalModelOptions, - db, - }); - - const llmConfig = options.llmConfig as Record; - const webSearchDenied = - hasProviderWebSearch(options.tools, llmConfig) && - !(await resolveWebSearchGrant({ - req: params.req as Request | undefined, - user, - resolve: params.resolveWebSearchGrant, - getRoleByName: db.getRoleByName, - })); - if (webSearchDenied && stripWebSearchPlugin(llmConfig) > 0) { - logger.debug( - `[initializeAgent] Removed the OpenRouter web search plugin; role denies WEB_SEARCH.`, - ); - } - const tokensModel = - agent.provider === EModelEndpoint.azureOpenAI ? agent.model : (llmConfig?.model as string); - const maxOutputTokens = optionalChainWithEmptyCheck( - llmConfig?.maxOutputTokens as number | undefined, - llmConfig?.maxTokens as number | undefined, - 0, - ); - const agentMaxContextTokens = optionalChainWithEmptyCheck( - maxContextTokens, - getModelMaxTokens( - tokensModel ?? '', - providerEndpointMap[overrideProvider as keyof typeof providerEndpointMap], - options.endpointTokenConfig, - ), - DEFAULT_MAX_CONTEXT_TOKENS, - ); - - if ( - agent.endpoint === EModelEndpoint.azureOpenAI && - (llmConfig?.azureOpenAIApiInstanceName as string | undefined) == null - ) { - agent.provider = Providers.OPENAI; - } - - if (options.provider != null) { - agent.provider = options.provider; - } - /** * Unify code-execution tools around `bash_tool` + `read_file` when the * agent explicitly lists `execute_code` in its tools and the admin @@ -2331,11 +2367,60 @@ export async function initializeAgent( const toMongoFiles = (files: Array | undefined): IMongoFile[] => (files ?? []).filter((a): a is TFile => a != null).map((a) => a as unknown as IMongoFile); - const finalAttachments: IMongoFile[] = toMongoFiles(primedAttachments); - const requestAttachments: IMongoFile[] = toMongoFiles(primedRequestAttachments); - const agentContextAttachments: IMongoFile[] = toMongoFiles(primedAgentContextAttachments); + const loadedFileToolNames = new Set([ + ...(structuredTools ?? []).map((tool) => tool.name), + ...(toolDefinitions ?? []).map((tool) => tool.name), + ]); + const finalFileConsumers: TurnFileConsumers = { + executeCode: + wantsCodeFiles && + (loadedFileToolNames.has(Tools.execute_code) || + (loadedFileToolNames.has('bash_tool') && loadedFileToolNames.has('read_file'))), + fileSearch: wantsSearchFiles && loadedFileToolNames.has(Tools.file_search), + }; + const consumersChanged = + fileConsumers.executeCode !== finalFileConsumers.executeCode || + fileConsumers.fileSearch !== finalFileConsumers.fileSearch; + Object.assign(fileConsumers, finalFileConsumers); + const finalizeAttachments = (files: Array | undefined): IMongoFile[] => { + const hydrated = toMongoFiles(files); + return consumersChanged + ? applyTurnDelivery(hydrated, { routing: deliveryRouting, consumers: fileConsumers }) + : hydrated; + }; + const finalAttachments = finalizeAttachments(primedAttachments); + const finalRequestAttachments = consumersChanged + ? applyTurnDelivery( + (primedRequestAttachments ?? []).filter((file): file is TFile => file != null), + { + routing: deliveryRouting, + consumers: fileConsumers, + }, + ) + : primedRequestAttachments; + const requestAttachments = toMongoFiles(finalRequestAttachments); + const agentContextAttachments = finalizeAttachments(primedAgentContextAttachments); + if (consumersChanged) { + /* A loader may drop a reader, including a skill extra on retry. Newly model-bound text + * must pass admission before it can escape initialization, just like initial fallback. */ + const admissionFileIds = new Set( + (authorizedRunFiles ?? requestUsageFiles).map((file) => file.file_id), + ); + assertAgentAttachmentLimits({ + attachments: requestAttachments.filter( + (file) => admissionFileIds.has(file.file_id) && isModelBoundAttachmentFile(file), + ), + fileConfig: appConfig?.fileConfig, + endpoint: agent.endpoint ?? '', + endpointType: endpointFileType, + }); + assertModelBoundContent({ + filters: appConfig?.filters, + files: [...finalAttachments, ...requestAttachments, ...agentContextAttachments], + }); + } const currentRequestFileIds = new Set(requestFileIds); - const currentRequestAttachments: TFile[] = (primedRequestAttachments ?? []) + const currentRequestAttachments: TFile[] = (finalRequestAttachments ?? []) .filter((file): file is TFile => file != null && currentRequestFileIds.has(file.file_id)) .map((file) => ({ ...file, @@ -2371,6 +2456,7 @@ export async function initializeAgent( azureOptions: options.azureOptions, resendFiles, imageDetail, + deliveryRouting, toolRegistry, mcpAvailableTools, requestScopedConnections, @@ -2405,6 +2491,7 @@ export async function initializeAgent( requestAttachments, currentRequestAttachments, agentContextAttachments, + fileConsumers, toolContextMap: toolContextMap ?? {}, dynamicToolContextMap: dynamicToolContextMap ?? {}, useLegacyContent: !!options.useLegacyContent, diff --git a/packages/api/src/agents/steering/__tests__/media.spec.ts b/packages/api/src/agents/steering/__tests__/media.spec.ts index acc1ff7e57..460f162aa1 100644 --- a/packages/api/src/agents/steering/__tests__/media.spec.ts +++ b/packages/api/src/agents/steering/__tests__/media.spec.ts @@ -16,8 +16,9 @@ function createClient({ image_urls?: Array>; documents?: Array>; fileContext?: string; -} = {}): SteerMediaClient & { processAttachments: jest.Mock } { +} = {}): SteerMediaClient & { processAttachments: jest.Mock; resolveTurnAttachments: jest.Mock } { return { + resolveTurnAttachments: jest.fn((files: IMongoFile[]) => files), addFileContextToMessage: jest.fn(async (pseudo: Record) => { if (fileContext) { pseudo.fileContext = fileContext; @@ -104,10 +105,11 @@ describe('buildSteerMedia', () => { getFiles, }); - expect(client.processAttachments).toHaveBeenCalledWith(expect.anything(), [ - secondDoc, - imageDoc, - ]); + expect(client.processAttachments).toHaveBeenCalledWith( + expect.anything(), + [secondDoc, imageDoc], + { executeCode: false, fileSearch: false }, + ); expect(result?.files?.map((file) => file.file_id)).toEqual(['f2', 'f1']); }); @@ -134,6 +136,39 @@ describe('buildSteerMedia', () => { expect(client.processAttachments).not.toHaveBeenCalled(); }); + it('checks and encodes the turn view of the records it loads', async () => { + /* A tool-routed file this turn delivers as text is stored as `none`: the preflight and the + * encoders must both see the turn's copy, or the text would skip the model-bound checks. */ + const storedCsv = { file_id: 'csv', type: 'text/csv', llmDeliveryPath: 'none' }; + const turnCsv = { ...storedCsv, llmDeliveryPath: 'text' }; + const getFiles: SteerFileFetcher = jest.fn(async () => [storedCsv as unknown as IMongoFile]); + const client = createClient(); + client.resolveTurnAttachments.mockReturnValueOnce([turnCsv]); + const assertFilesAllowed = jest.fn(); + + await buildSteerMedia({ + client, + user, + item: steerItem([{ file_id: 'csv' }]), + getFiles, + assertFilesAllowed, + }); + + expect(client.resolveTurnAttachments).toHaveBeenCalledWith([storedCsv], { + executeCode: false, + fileSearch: false, + }); + expect(assertFilesAllowed).toHaveBeenCalledWith([turnCsv]); + expect(client.addFileContextToMessage).toHaveBeenCalledWith(expect.anything(), [turnCsv], { + executeCode: false, + fileSearch: false, + }); + expect(client.processAttachments).toHaveBeenCalledWith(expect.anything(), [turnCsv], { + executeCode: false, + fileSearch: false, + }); + }); + it('prepends extracted file context to the steer text', async () => { const getFiles: SteerFileFetcher = jest.fn(async () => [ { file_id: 'f2', type: 'text/plain' } as unknown as IMongoFile, @@ -239,6 +274,25 @@ describe('stampSteerPartMedia', () => { ]); }); + it('encodes the turn view of the records it fetches itself', async () => { + const storedCsv = { file_id: 'csv', type: 'text/csv', llmDeliveryPath: 'none' }; + const turnCsv = { ...storedCsv, llmDeliveryPath: 'text' }; + const getFiles: SteerFileFetcher = jest.fn(async () => [storedCsv as unknown as IMongoFile]); + const client = createClient(); + client.resolveTurnAttachments.mockReturnValueOnce([turnCsv]); + const message = { + role: 'assistant', + content: [ + { type: 'steer', steer: 'use the sheet', steerId: 's3', files: [{ file_id: 'csv' }] }, + ], + }; + + await stampSteerPartMedia({ client, user, payload: [message], getFiles }); + + expect(client.resolveTurnAttachments).toHaveBeenCalledWith([storedCsv]); + expect(client.processAttachments).toHaveBeenCalledWith(expect.anything(), [turnCsv], undefined); + }); + it('consumes prefetched docs without issuing a second query', async () => { const getFiles: SteerFileFetcher = jest.fn(async () => []); const client = createClient({ image_urls: [imagePart] }); @@ -261,7 +315,11 @@ describe('stampSteerPartMedia', () => { expect(getFiles).not.toHaveBeenCalled(); expect(stamped).toHaveLength(1); expect(stamped[0].index).toBe(0); - expect(client.processAttachments).toHaveBeenCalledWith(expect.anything(), [imageDoc]); + expect(client.processAttachments).toHaveBeenCalledWith( + expect.anything(), + [imageDoc], + undefined, + ); }); it('does nothing when no steer part carries files', async () => { diff --git a/packages/api/src/agents/steering/media.ts b/packages/api/src/agents/steering/media.ts index 4536f5678f..977549a59f 100644 --- a/packages/api/src/agents/steering/media.ts +++ b/packages/api/src/agents/steering/media.ts @@ -1,8 +1,8 @@ import { logger } from '@librechat/data-schemas'; import { formatMessage } from '@librechat/agents'; import { ContentTypes } from 'librechat-data-provider'; +import type { TFile, TurnFileConsumers } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; -import type { TFile } from 'librechat-data-provider'; import type { SteerQueueItem } from '~/stream/interfaces/IJobStore'; import type { SteerFileFetcher } from './request'; import type { SteerMediaResult } from './runtime'; @@ -14,10 +14,17 @@ import { prependFileContext } from '../client'; /** The BaseClient encode surface the steer media pipeline reuses. */ export interface SteerMediaClient { - addFileContextToMessage(message: Record, files: IMongoFile[]): Promise; + /** The turn's view of stored records, which every check and encode below must share. */ + resolveTurnAttachments(files: IMongoFile[], consumers?: TurnFileConsumers): IMongoFile[]; + addFileContextToMessage( + message: Record, + files: IMongoFile[], + consumers?: TurnFileConsumers, + ): Promise; processAttachments( message: Record, files: IMongoFile[], + consumers?: TurnFileConsumers, ): Promise; } @@ -75,17 +82,19 @@ async function encodeSteerContent({ quotes, steerId, fileDocs, + fileConsumers, }: { client: SteerMediaClient; text: string; quotes?: string[] | null; steerId: string; fileDocs: IMongoFile[]; + fileConsumers?: TurnFileConsumers; }): Promise { const modelText = mergeSteerModelText(text, quotes); const pseudo: PseudoMessage = { messageId: `steer:${steerId}` }; - await client.addFileContextToMessage(pseudo, fileDocs); - const validated = await client.processAttachments(pseudo, fileDocs); + await client.addFileContextToMessage(pseudo, fileDocs, fileConsumers); + const validated = await client.processAttachments(pseudo, fileDocs, fileConsumers); const formatted = formatMessage({ message: { role: 'user', @@ -139,9 +148,13 @@ export async function buildSteerMedia({ return undefined; } const docsById = new Map(rawDocs.map((file) => [file.file_id, file])); - const fileDocs = ids - .map((id) => docsById.get(id)) - .filter((doc): doc is IMongoFile => doc != null); + /* These files arrived after resource priming. A loaded tool is not evidence that + * it can access them; use the opted-in text fallback until a new turn provisions them. */ + const fileConsumers: TurnFileConsumers = { executeCode: false, fileSearch: false }; + const fileDocs = client.resolveTurnAttachments( + ids.map((id) => docsById.get(id)).filter((doc): doc is IMongoFile => doc != null), + fileConsumers, + ); assertFilesAllowed?.(fileDocs); return encodeSteerContent({ client, @@ -149,6 +162,7 @@ export async function buildSteerMedia({ quotes: item.quotes, steerId: item.steerId, fileDocs, + fileConsumers, }); } @@ -247,7 +261,9 @@ export async function stampSteerPartMedia({ if (filter != null) { const fileDocs = await getFiles(filter, {}, {}); if (Array.isArray(fileDocs) && fileDocs.length > 0) { - resolvedDocsById = new Map(fileDocs.map((file) => [file.file_id, file])); + resolvedDocsById = new Map( + client.resolveTurnAttachments(fileDocs).map((file) => [file.file_id, file]), + ); } } } diff --git a/packages/api/src/files/extract.ts b/packages/api/src/files/extract.ts index 190e243db8..48606c6eba 100644 --- a/packages/api/src/files/extract.ts +++ b/packages/api/src/files/extract.ts @@ -1,3 +1,4 @@ +import { megabyte } from 'librechat-data-provider'; import type { FiltersConfig } from 'librechat-data-provider'; import { getBlockedUninspectableFileField, @@ -6,6 +7,13 @@ import { } from '~/protection/files'; import { getSafeErrorMetadata } from '~/utils/errors'; +/** + * The most extracted text a file document can hold. MongoDB refuses a document past 16 MiB and + * the text shares it with the file's other fields, so this bounds storage rather than policy: + * what reaches a model is governed by `fileContextCharLimit` and `fileTokenLimit`. + */ +export const MAX_STORED_EXTRACTED_TEXT_BYTES: number = 15 * megabyte; + interface ExtractedFileText { readonly text?: string | null; } diff --git a/packages/api/src/files/index.ts b/packages/api/src/files/index.ts index 87df6f4105..5bda69aea7 100644 --- a/packages/api/src/files/index.ts +++ b/packages/api/src/files/index.ts @@ -16,6 +16,7 @@ export * from './preflight'; export * from './provision'; export * from './upload/routing'; export * from './upload/diagnostics'; +export * from './upload/fallback'; export * from './rag'; export * from './regexEngine'; export * from './retention'; diff --git a/packages/api/src/files/upload/fallback.spec.ts b/packages/api/src/files/upload/fallback.spec.ts new file mode 100644 index 0000000000..1b65139d1f --- /dev/null +++ b/packages/api/src/files/upload/fallback.spec.ts @@ -0,0 +1,210 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import type { FiltersConfig } from 'librechat-data-provider'; +import type { UploadFallbackTextExtractors } from './fallback'; +import { + UPLOAD_FALLBACK_TEXT_PLANS, + getUploadFallbackTextPlan, + resolveUploadFallbackText, +} from './fallback'; +import { MAX_STORED_EXTRACTED_TEXT_BYTES } from '~/files/extract'; +import { parseDocument } from '~/files/documents/crud'; +import { parseTextNative } from '~/files/text'; + +const XLSX_MIME = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; +const csvRoute = { + deliveryPath: 'none' as const, + destinationChosen: false, + isMessageAttachment: true, + mimeType: 'text/csv', + endpointConfig: { textFallbackWithoutTools: true }, +}; + +describe('getUploadFallbackTextPlan', () => { + it('reads a natively textual file left to tools', () => { + expect(getUploadFallbackTextPlan(csvRoute)).toBe(UPLOAD_FALLBACK_TEXT_PLANS.nativeText); + }); + + it('parses a spreadsheet with the built-in document parser', () => { + expect(getUploadFallbackTextPlan({ ...csvRoute, mimeType: XLSX_MIME })).toBe( + UPLOAD_FALLBACK_TEXT_PLANS.documentParser, + ); + }); + + it('runs nothing for a type no built-in extractor can read', () => { + expect(getUploadFallbackTextPlan({ ...csvRoute, mimeType: 'image/png' })).toBeNull(); + expect(getUploadFallbackTextPlan({ ...csvRoute, mimeType: 'application/zip' })).toBeNull(); + }); + + it('runs nothing where the upload endpoint has not enabled the fallback', () => { + expect(getUploadFallbackTextPlan({ ...csvRoute, endpointConfig: undefined })).toBeNull(); + expect( + getUploadFallbackTextPlan({ + ...csvRoute, + endpointConfig: { textFallbackWithoutTools: false }, + }), + ).toBeNull(); + }); + + it('runs nothing for an upload that already reaches the model', () => { + expect(getUploadFallbackTextPlan({ ...csvRoute, deliveryPath: 'text' })).toBeNull(); + expect(getUploadFallbackTextPlan({ ...csvRoute, deliveryPath: 'provider' })).toBeNull(); + }); + + it('runs nothing for a destination the user chose, which no turn re-resolves', () => { + expect(getUploadFallbackTextPlan({ ...csvRoute, destinationChosen: true })).toBeNull(); + }); + + it('runs nothing for a file kept on an agent, which no turn delivers as text', () => { + expect(getUploadFallbackTextPlan({ ...csvRoute, isMessageAttachment: false })).toBeNull(); + }); +}); + +describe('resolveUploadFallbackText', () => { + const { mimeType: _mimeType, ...route } = csvRoute; + const privateTokenFilters: FiltersConfig = { + files: { + pii: { + fields: ['extracted_text'], + starterPatterns: [], + customPatterns: [{ id: 'private', label: 'private token', regex: 'PRIVATE-[A-Z]+' }], + }, + }, + }; + let uploadDir: string; + + beforeAll(() => { + uploadDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fallback-text-')); + }); + + afterAll(() => { + fs.rmSync(uploadDir, { recursive: true, force: true }); + }); + + /** A CSV upload as multer leaves it; without content, its temporary file is already gone. */ + function csvUpload(name: string, content?: string): Express.Multer.File { + const filePath = path.join(uploadDir, name); + if (content != null) { + fs.writeFileSync(filePath, content); + } + return { + originalname: name, + path: filePath, + mimetype: 'text/csv', + size: content != null ? Buffer.byteLength(content) : 0, + } as Express.Multer.File; + } + + /** The real built-in extractors, observed. */ + function spiedExtractors() { + return { + parseDocument: jest.fn(parseDocument), + parseTextNative: jest.fn(parseTextNative), + } satisfies UploadFallbackTextExtractors; + } + + it('stores the native text of a CSV without running the document parser', async () => { + const extractors = spiedExtractors(); + const file = csvUpload('sales.csv', 'region,total\nwest,4'); + + await expect( + resolveUploadFallbackText({ ...route, file, fileId: 'file-1', extractors }), + ).resolves.toBe('region,total\nwest,4'); + expect(extractors.parseTextNative).toHaveBeenCalledWith(file); + expect(extractors.parseDocument).not.toHaveBeenCalled(); + }); + + it('stores the parsed text of a spreadsheet without decoding its bytes', async () => { + const extractors = spiedExtractors(); + const file = { + originalname: 'sample.xlsx', + path: path.join(__dirname, '../documents/sample.xlsx'), + mimetype: XLSX_MIME, + } as Express.Multer.File; + + await expect( + resolveUploadFallbackText({ ...route, file, fileId: 'file-1', extractors }), + ).resolves.toBe('Sheet One:\nData,on,first,sheet\nSecond Sheet:\nData,On\nSecond,Sheet\n'); + expect(extractors.parseDocument).toHaveBeenCalledWith({ file }); + expect(extractors.parseTextNative).not.toHaveBeenCalled(); + }); + + it('extracts nothing when no plan applies', async () => { + const extractors = spiedExtractors(); + + await expect( + resolveUploadFallbackText({ + ...route, + isMessageAttachment: false, + file: csvUpload('kept.csv', 'region,total'), + fileId: 'file-1', + extractors, + }), + ).resolves.toBeUndefined(); + expect(extractors.parseDocument).not.toHaveBeenCalled(); + expect(extractors.parseTextNative).not.toHaveBeenCalled(); + }); + + it('keeps the upload when extraction fails', async () => { + await expect( + resolveUploadFallbackText({ ...route, file: csvUpload('missing.csv'), fileId: 'file-1' }), + ).resolves.toBeUndefined(); + }); + + it('stores nothing for text with no content', async () => { + await expect( + resolveUploadFallbackText({ ...route, file: csvUpload('blank.csv', ' \n '), fileId: 'f' }), + ).resolves.toBeUndefined(); + }); + + it('stores nothing past the extracted-text storage cap', async () => { + const extractors: UploadFallbackTextExtractors = { + parseDocument, + parseTextNative: async () => ({ text: 'a'.repeat(MAX_STORED_EXTRACTED_TEXT_BYTES + 1) }), + }; + + await expect( + resolveUploadFallbackText({ + ...route, + file: csvUpload('huge.csv', 'a'), + fileId: 'file-1', + extractors, + }), + ).resolves.toBeUndefined(); + }); + + it('stores nothing a configured content policy flags, and keeps text it does not', async () => { + await expect( + resolveUploadFallbackText({ + ...route, + file: csvUpload('flagged.csv', 'token,PRIVATE-SECRET'), + fileId: 'file-1', + filters: privateTokenFilters, + }), + ).resolves.toBeUndefined(); + await expect( + resolveUploadFallbackText({ + ...route, + file: csvUpload('clean.csv', 'region,total'), + fileId: 'file-2', + filters: privateTokenFilters, + }), + ).resolves.toBe('region,total'); + }); + + it('stores nothing a blocking policy cannot inspect', async () => { + const filters: FiltersConfig = { + files: { pii: { fields: ['extracted_text'], starterPatterns: [], uninspectable: 'block' } }, + }; + + await expect( + resolveUploadFallbackText({ + ...route, + file: csvUpload('empty.csv', ''), + fileId: 'file-1', + filters, + }), + ).resolves.toBeUndefined(); + }); +}); diff --git a/packages/api/src/files/upload/fallback.ts b/packages/api/src/files/upload/fallback.ts new file mode 100644 index 0000000000..96b8bce20a --- /dev/null +++ b/packages/api/src/files/upload/fallback.ts @@ -0,0 +1,143 @@ +import { logger } from '@librechat/data-schemas'; +import { isNativelyReadableText, documentParserMimeTypes } from 'librechat-data-provider'; +import type { + FiltersConfig, + EndpointFileConfig, + TDefaultLLMDeliveryPath, +} from 'librechat-data-provider'; +import { + extractInspectableFileText, + getFileExtractionLogDetails, + MAX_STORED_EXTRACTED_TEXT_BYTES, +} from '~/files/extract'; +import { extractFileContent } from '~/protection/adapters/submissions'; +import { hasActiveFileFieldPolicy } from '~/protection/files'; +import { parseDocument } from '~/files/documents/crud'; +import { inspectContent } from '~/protection/runtime'; +import { parseTextNative } from '~/files/text'; + +export const UPLOAD_FALLBACK_TEXT_PLANS = { + documentParser: 'document_parser', + nativeText: 'native_text', +} as const; + +export type UploadFallbackTextPlan = + (typeof UPLOAD_FALLBACK_TEXT_PLANS)[keyof typeof UPLOAD_FALLBACK_TEXT_PLANS]; + +export interface UploadFallbackTextRoute { + /** The route the upload resolved to. */ + deliveryPath: TDefaultLLMDeliveryPath; + /** Whether the user chose the destination, by tool resource or the legacy chooser. */ + destinationChosen: boolean; + /** Whether the upload attaches to a message; an agent's own tool files never reach a prompt. */ + isMessageAttachment: boolean; + mimeType: string; + /** The upload endpoint's file config, which opts in through `textFallbackWithoutTools`. */ + endpointConfig?: Pick; +} + +interface ExtractedText { + readonly text?: string | null; +} + +/** The built-in readers fallback text comes from. */ +export interface UploadFallbackTextExtractors { + parseDocument: (params: { file: Express.Multer.File }) => Promise; + parseTextNative: (file: Express.Multer.File) => Promise; +} + +const builtInExtractors: UploadFallbackTextExtractors = { parseDocument, parseTextNative }; + +/** + * Which built-in extractor stores text for an upload left to tools, or `null` when none runs. + * + * Where the endpoint enables `textFallbackWithoutTools`, a turn that runs no tool able to read a + * `none`-routed file delivers this text instead (`resolveTurnLLMDeliveryPath`). Only an inferred + * route on a message attachment qualifies: a turn never re-resolves a destination the user chose, + * and files kept on an agent's tool resources never reach a prompt. A file filed + * under a tool that reads it still gets text, because a later turn may run without that tool: a + * handoff agent, or the same agent after its tools or grants change. Only built-in extractors + * run, so a file meant for a tool never costs a RAG or OCR call. + */ +export function getUploadFallbackTextPlan( + route: UploadFallbackTextRoute, +): UploadFallbackTextPlan | null { + if ( + route.endpointConfig?.textFallbackWithoutTools !== true || + route.deliveryPath !== 'none' || + route.destinationChosen || + !route.isMessageAttachment + ) { + return null; + } + if (documentParserMimeTypes.some((pattern) => pattern.test(route.mimeType))) { + return UPLOAD_FALLBACK_TEXT_PLANS.documentParser; + } + return isNativelyReadableText(route.mimeType) ? UPLOAD_FALLBACK_TEXT_PLANS.nativeText : null; +} + +/** + * Text a turn without a reading tool can fall back to for this upload, when one applies. + * + * Reads the upload before storage can move it, with the extractor its plan names. Best effort by + * design: an extractor failure, text a policy cannot inspect, an overrun of the storage cap, or a + * content finding leaves the upload without fallback text rather than refusing a file whose + * primary reader is a tool. Text that is delivered later is inspected again as model-bound + * content. + */ +export async function resolveUploadFallbackText({ + file, + fileId, + filters, + extractors = builtInExtractors, + ...route +}: Omit & { + file: Express.Multer.File; + fileId: string; + filters?: FiltersConfig; + extractors?: UploadFallbackTextExtractors; +}): Promise { + const plan = getUploadFallbackTextPlan({ ...route, mimeType: file.mimetype }); + if (plan == null) { + return undefined; + } + const skip = (reason: string, error?: unknown): undefined => { + const { fileLabel, errorMetadata } = getFileExtractionLogDetails({ + filters, + filename: file.originalname, + fileId, + error, + }); + logger.warn( + `[resolveUploadFallbackText] No fallback text for ${fileLabel}: ${reason}`, + errorMetadata, + ); + return undefined; + }; + try { + const result = await extractInspectableFileText({ + filters, + extract: () => + plan === UPLOAD_FALLBACK_TEXT_PLANS.documentParser + ? extractors.parseDocument({ file }) + : extractors.parseTextNative(file), + }); + const text = result?.text; + if (typeof text !== 'string' || text.trim().length === 0) { + return undefined; + } + if (Buffer.byteLength(text, 'utf8') > MAX_STORED_EXTRACTED_TEXT_BYTES) { + return skip('extracted text exceeds the storage limit'); + } + if ( + filters != null && + hasActiveFileFieldPolicy(filters, ['extracted_text']) && + inspectContent(extractFileContent({ extractedText: text }), { filters }) != null + ) { + return skip('extracted text matched a content policy'); + } + return text; + } catch (error) { + return skip('extraction failed', error); + } +} diff --git a/packages/api/src/files/upload/routing.spec.ts b/packages/api/src/files/upload/routing.spec.ts index 71ae6289ff..3f1cf8b563 100644 --- a/packages/api/src/files/upload/routing.spec.ts +++ b/packages/api/src/files/upload/routing.spec.ts @@ -1,5 +1,5 @@ import { Readable } from 'node:stream'; -import { EModelEndpoint, EToolResources, Providers } from 'librechat-data-provider'; +import { Constants, EModelEndpoint, EToolResources, Providers } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { ServerRequest } from '~/types'; import { resolveUploadEndpoint, resolveEffectiveToolResource } from './routing'; @@ -42,6 +42,20 @@ describe('resolveUploadEndpoint', () => { expect(getAgent).not.toHaveBeenCalled(); }); + it.each(['openAI', 'azureOpenAI', 'MyGateway'])( + 'keeps the provider endpoint carried by an ephemeral conversation (%s)', + async (endpoint) => { + const getAgent = jest.fn().mockResolvedValue(null); + await expect( + resolveUploadEndpoint({ + req, + metadata: { endpoint, agent_id: String(Constants.EPHEMERAL_AGENT_ID) }, + getAgent, + }), + ).resolves.toBe(endpoint); + }, + ); + it('leaves an upload naming no agent alone', async () => { const getAgent = jest.fn(); diff --git a/packages/data-provider/src/file-config.spec.ts b/packages/data-provider/src/file-config.spec.ts index 164e75fb75..908117c913 100644 --- a/packages/data-provider/src/file-config.spec.ts +++ b/packages/data-provider/src/file-config.spec.ts @@ -2008,6 +2008,58 @@ describe('defaultLLMDeliveryPath config merging', () => { }); }); +describe('textFallbackWithoutTools config merging', () => { + const resolveFor = (dynamic: Parameters[0], endpoint: string) => + getEndpointFileConfig({ fileConfig: mergeFileConfig(dynamic), endpoint }) + .textFallbackWithoutTools; + + it('is off unless configured', () => { + expect(mergeFileConfig(undefined).textFallbackWithoutTools).toBeUndefined(); + expect(resolveFor(undefined, EModelEndpoint.openAI)).toBeUndefined(); + expect(resolveFor({ endpoints: { default: {} } }, EModelEndpoint.openAI)).toBeUndefined(); + }); + + it('accepts the setting at the top level and on an endpoint', () => { + expect( + fileConfigSchema.safeParse({ + textFallbackWithoutTools: true, + endpoints: { openAI: { textFallbackWithoutTools: false } }, + }).success, + ).toBe(true); + expect(fileConfigSchema.safeParse({ textFallbackWithoutTools: 'yes' }).success).toBe(false); + }); + + it('reaches an endpoint configured for it', () => { + expect( + resolveFor( + { endpoints: { [EModelEndpoint.openAI]: { textFallbackWithoutTools: true } } }, + EModelEndpoint.openAI, + ), + ).toBe(true); + }); + + it('is inherited from the top level and from the default endpoint', () => { + expect(resolveFor({ textFallbackWithoutTools: true }, EModelEndpoint.anthropic)).toBe(true); + expect( + resolveFor({ endpoints: { default: { textFallbackWithoutTools: true } } }, 'MyGateway'), + ).toBe(true); + }); + + it('lets an endpoint turn off what it would inherit', () => { + expect( + resolveFor( + { + endpoints: { + default: { textFallbackWithoutTools: true }, + [EModelEndpoint.openAI]: { textFallbackWithoutTools: false }, + }, + }, + EModelEndpoint.openAI, + ), + ).toBe(false); + }); +}); + describe('agent attachment context limits', () => { it('keeps the turn-memory ceiling separate from agent upload storage', () => { expect(baseFileConfig.fileContextSizeLimit).toBe(128 * 1024 * 1024); diff --git a/packages/data-provider/src/file-config.ts b/packages/data-provider/src/file-config.ts index 0890dd24ae..0123c2b6a6 100644 --- a/packages/data-provider/src/file-config.ts +++ b/packages/data-provider/src/file-config.ts @@ -583,6 +583,7 @@ export const endpointFileConfigSchema = z.object({ supportedMimeTypes: supportedMimeTypesSchema.optional(), defaultLLMDeliveryPath: defaultLLMDeliveryPathSchema.optional(), legacyFileUploadUX: z.boolean().optional(), + textFallbackWithoutTools: z.boolean().optional(), }); const skillFileConfigSchema = z.object({ @@ -624,6 +625,7 @@ export const fileConfigSchema = z.object({ .optional(), defaultLLMDeliveryPath: defaultLLMDeliveryPathSchema.optional(), legacyFileUploadUX: z.boolean().optional(), + textFallbackWithoutTools: z.boolean().optional(), }); export type TFileConfig = z.infer; @@ -970,6 +972,8 @@ function mergeWithDefault( defaultConfig.defaultLLMDeliveryPath, ), legacyFileUploadUX: endpointConfig.legacyFileUploadUX ?? defaultConfig.legacyFileUploadUX, + textFallbackWithoutTools: + endpointConfig.textFallbackWithoutTools ?? defaultConfig.textFallbackWithoutTools, }; } @@ -1060,6 +1064,8 @@ export function getEndpointFileConfig(params: { baseDefaultConfig.defaultLLMDeliveryPath, ), legacyFileUploadUX: mergedFileConfig.legacyFileUploadUX ?? baseDefaultConfig.legacyFileUploadUX, + textFallbackWithoutTools: + mergedFileConfig.textFallbackWithoutTools ?? baseDefaultConfig.textFallbackWithoutTools, }; const userDefaultConfig = mergedFileConfig.endpoints.default; const defaultConfig = userDefaultConfig @@ -1184,6 +1190,10 @@ export function mergeFileConfig(dynamic: z.infer | unde mergedConfig.legacyFileUploadUX = dynamic.legacyFileUploadUX; } + if (dynamic.textFallbackWithoutTools !== undefined) { + mergedConfig.textFallbackWithoutTools = dynamic.textFallbackWithoutTools; + } + if (dynamic.serverFileSizeLimit !== undefined) { mergedConfig.serverFileSizeLimit = mbToBytes(dynamic.serverFileSizeLimit); } @@ -1299,6 +1309,10 @@ export function mergeFileConfig(dynamic: z.infer | unde if (dynamicEndpoint.legacyFileUploadUX !== undefined) { mergedEndpoint.legacyFileUploadUX = dynamicEndpoint.legacyFileUploadUX; } + + if (dynamicEndpoint.textFallbackWithoutTools !== undefined) { + mergedEndpoint.textFallbackWithoutTools = dynamicEndpoint.textFallbackWithoutTools; + } } return mergedConfig; diff --git a/packages/data-provider/src/resolve-llm-delivery-path.spec.ts b/packages/data-provider/src/resolve-llm-delivery-path.spec.ts index b62c3a26f9..9fb477bfa0 100644 --- a/packages/data-provider/src/resolve-llm-delivery-path.spec.ts +++ b/packages/data-provider/src/resolve-llm-delivery-path.spec.ts @@ -1,16 +1,36 @@ +import type { + TurnFileConsumers, + TurnDeliveryFile, + TurnDeliveryRouting, +} from './resolve-llm-delivery-path'; import type { TDefaultLLMDeliveryPathConfig } from './file-config'; +import type { EndpointFileConfig } from './types/files'; import type { TEndpoint } from './config'; import { + hasTurnFileConsumer, isNativelyReadableText, canToolResourceConsume, resolveUploadDestination, getCustomEndpointProvider, + resolveTurnLLMDeliveryPath as resolveStoredTurnPath, + hasInferredLLMDeliveryPath, resolveDefaultLLMDeliveryPath, resolveUploadLLMDeliveryPath, SYSTEM_LLM_DELIVERY_DEFAULTS, } from './resolve-llm-delivery-path'; import { mergeFileConfig, supportedMimeTypes, getEndpointFileConfig } from './file-config'; +function resolveTurnLLMDeliveryPath({ + file, + consumers, + ...routing +}: { + file: TurnDeliveryFile; + consumers?: TurnFileConsumers; +} & Partial) { + return resolveStoredTurnPath(routing, file, consumers); +} + describe('resolveDefaultLLMDeliveryPath', () => { it('should return system default for images when no config provided', () => { expect(resolveDefaultLLMDeliveryPath('image/png')).toBe('provider'); @@ -750,3 +770,179 @@ describe('provider document capability', () => { ).toBe('provider'); }); }); + +describe('hasTurnFileConsumer', () => { + it('finds a reader only among the tools this turn runs', () => { + expect(hasTurnFileConsumer('text/csv', { executeCode: false, fileSearch: false })).toBe(false); + expect(hasTurnFileConsumer('text/csv', { executeCode: true, fileSearch: false })).toBe(true); + expect(hasTurnFileConsumer('text/csv', { executeCode: false, fileSearch: true })).toBe(true); + }); + + it('does not count a tool that cannot read the type', () => { + expect(hasTurnFileConsumer('video/mp4', { executeCode: false, fileSearch: true })).toBe(false); + }); +}); + +describe('hasInferredLLMDeliveryPath', () => { + it('re-resolves only a route upload inferred', () => { + expect(hasInferredLLMDeliveryPath({ llmDeliveryPath: 'none' })).toBe(true); + expect( + hasInferredLLMDeliveryPath({ + llmDeliveryPath: 'text', + metadata: { destinationChosen: false }, + }), + ).toBe(true); + expect( + hasInferredLLMDeliveryPath({ + llmDeliveryPath: 'none', + metadata: { destinationChosen: true }, + }), + ).toBe(false); + expect(hasInferredLLMDeliveryPath({ type: 'text/csv' })).toBe(false); + }); +}); + +describe('resolveTurnLLMDeliveryPath', () => { + const endpointConfig: EndpointFileConfig = { + defaultLLMDeliveryPath: { overrides: { 'text/csv': 'none' } }, + textFallbackWithoutTools: true, + }; + const noReader: TurnFileConsumers = { executeCode: false, fileSearch: false }; + const routedCsv = { + type: 'text/csv', + text: 'region,total\nwest,4', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + }; + + it('delivers stored text when the turn runs no tool that can read the file', () => { + expect( + resolveTurnLLMDeliveryPath({ file: routedCsv, consumers: noReader, endpointConfig }), + ).toBe('text'); + }); + + it('keeps the tool route on an endpoint that has not enabled the fallback', () => { + expect( + resolveTurnLLMDeliveryPath({ + file: routedCsv, + consumers: noReader, + endpointConfig: { ...endpointConfig, textFallbackWithoutTools: undefined }, + }), + ).toBe('none'); + expect( + resolveTurnLLMDeliveryPath({ + file: routedCsv, + consumers: noReader, + endpointConfig: { ...endpointConfig, textFallbackWithoutTools: false }, + }), + ).toBe('none'); + }); + + it('leaves the file to Run Code when the turn can read it with code', () => { + expect( + resolveTurnLLMDeliveryPath({ + file: routedCsv, + consumers: { executeCode: true, fileSearch: false }, + endpointConfig, + }), + ).toBe('none'); + }); + + it('leaves the file to File Search when the turn can read it by retrieval', () => { + expect( + resolveTurnLLMDeliveryPath({ + file: routedCsv, + consumers: { executeCode: false, fileSearch: true }, + endpointConfig, + }), + ).toBe('none'); + }); + + it('does not judge a turn whose tools are unknown', () => { + expect(resolveTurnLLMDeliveryPath({ file: routedCsv, endpointConfig })).toBe('none'); + }); + + it('keeps the tool route when upload stored no text to fall back to', () => { + expect( + resolveTurnLLMDeliveryPath({ + file: { ...routedCsv, text: undefined }, + consumers: noReader, + endpointConfig, + }), + ).toBe('none'); + expect( + resolveTurnLLMDeliveryPath({ + file: { ...routedCsv, text: '' }, + consumers: noReader, + endpointConfig, + }), + ).toBe('none'); + }); + + it('keeps a destination the user chose even when nothing this turn can read it', () => { + expect( + resolveTurnLLMDeliveryPath({ + file: { ...routedCsv, metadata: { destinationChosen: true } }, + consumers: noReader, + endpointConfig, + }), + ).toBe('none'); + }); + + it('leaves a record predating routing to its legacy handling', () => { + expect( + resolveTurnLLMDeliveryPath({ + file: { type: 'text/csv', text: 'region,total' }, + consumers: noReader, + endpointConfig, + }), + ).toBeUndefined(); + }); + + it('does not fall back from a route that already reaches the model', () => { + expect( + resolveTurnLLMDeliveryPath({ + file: routedCsv, + consumers: noReader, + endpointConfig: { defaultLLMDeliveryPath: { overrides: { 'text/csv': 'provider' } } }, + }), + ).toBe('provider'); + }); + + it('re-resolves media against the provider the endpoint runs as', () => { + /* A custom endpoint whose admin listed video receives it only while it speaks OpenAI's + * format, so the turn route has to see the declared provider the upload route saw. */ + const video = { + type: 'video/mp4', + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + }; + const gateway = { + file: video, + consumers: noReader, + endpoint: 'MyGateway', + endpointConfig: { supportedMimeTypes: [/^video\/mp4$/] }, + }; + + expect(resolveTurnLLMDeliveryPath({ ...gateway, endpointProvider: 'openAI' })).toBe('provider'); + expect(resolveTurnLLMDeliveryPath({ ...gateway, endpointProvider: 'anthropic' })).toBe('none'); + }); + + it('judges readers against the type routing saw before conversion', () => { + /* File Search reads the original CSV but not the converted image type, so checking the + * stored type here would wrongly find no reader and paste the file into the prompt. */ + const converted = { + ...routedCsv, + type: 'image/png', + metadata: { destinationChosen: false, routingMimeType: 'text/csv' }, + }; + + expect( + resolveTurnLLMDeliveryPath({ + file: converted, + consumers: { executeCode: false, fileSearch: true }, + endpointConfig, + }), + ).toBe('none'); + }); +}); diff --git a/packages/data-provider/src/resolve-llm-delivery-path.ts b/packages/data-provider/src/resolve-llm-delivery-path.ts index bccf7f566c..09726f74c2 100644 --- a/packages/data-provider/src/resolve-llm-delivery-path.ts +++ b/packages/data-provider/src/resolve-llm-delivery-path.ts @@ -353,6 +353,93 @@ export function canToolResourceConsume(toolResource: string, mimeType: string): const matchesMimeList = (mimeType: string, patterns: RegExp[]): boolean => patterns.some((pattern) => pattern.test(mimeType)); +/** + * The file-reading tools one agent's turn runs. Each flag requires deployment capability, + * the caller's role grant, and a reader in the final loaded tool set. + */ +export interface TurnFileConsumers { + executeCode: boolean; + fileSearch: boolean; +} + +/** Whether a tool this turn runs can read a file of this type. */ +export function hasTurnFileConsumer(mimeType: string, consumers: TurnFileConsumers): boolean { + return ( + (consumers.executeCode && canToolResourceConsume(EToolResources.execute_code, mimeType)) || + (consumers.fileSearch && canToolResourceConsume(EToolResources.file_search, mimeType)) + ); +} + +/** + * The inputs that route every attachment for the agent running a turn. Initialization + * settles them once, after the provider swap and the Responses API decision, and every + * reader of a turn route consumes this value rather than deriving one from the agent. + */ +export interface TurnDeliveryRouting { + fileConfig: FileConfig; + endpointConfig: EndpointFileConfig; + /** The endpoint the file policy is configured under: a custom endpoint's own name, not + * the client family initialization runs it as. */ + endpoint: string; + /** The dialect a custom endpoint declares, which decides whether it receives OpenAI-format + * media; undefined for a built-in or OpenAI-compatible endpoint. */ + endpointProvider?: string; + useResponsesApi?: boolean; + sttConfigured: boolean; +} + +/** The fields of an attachment record that decide its delivery on a turn. */ +export interface TurnDeliveryFile { + type?: string; + text?: string | null; + /** Stored as an upload-time inference, so any string may be read back. */ + llmDeliveryPath?: string | null; + metadata?: { routingMimeType?: string; destinationChosen?: boolean } | null; +} + +const isLLMDeliveryPath = (value: unknown): value is TDefaultLLMDeliveryPath => + value === 'provider' || value === 'text' || value === 'none'; + +/** Whether a record's stored route was inferred at upload, so each turn resolves it again. */ +export function hasInferredLLMDeliveryPath(file: TurnDeliveryFile): boolean { + return file.llmDeliveryPath != null && file.metadata?.destinationChosen !== true; +} + +/** + * Delivery path for one attachment on one agent's turn. + * + * A record predating routing and a destination the user chose keep what they stored. An + * inferred route re-resolves against the endpoint handling the turn. A `none` route leaves + * the file for a tool; where the endpoint enables `textFallbackWithoutTools` and this turn + * runs no tool that can read the file, the text extracted at upload is delivered rather than + * the file reaching nothing. Consumers left undefined are unknown and not judged, as in + * {@link resolveUploadDestination}. + */ +export function resolveTurnLLMDeliveryPath( + routing: Partial | undefined, + file: TurnDeliveryFile, + consumers?: TurnFileConsumers, +): TDefaultLLMDeliveryPath | undefined { + if (routing == null || !hasInferredLLMDeliveryPath(file)) { + return isLLMDeliveryPath(file.llmDeliveryPath) ? file.llmDeliveryPath : undefined; + } + const { endpointConfig } = routing; + /* Conversion changes the stored type, so use the type routing originally saw. */ + const mimeType = file.metadata?.routingMimeType ?? file.type ?? ''; + const path = resolveUploadLLMDeliveryPath({ mimeType, ...routing }); + const hasFallbackText = typeof file.text === 'string' && file.text.length > 0; + if ( + path === 'none' && + endpointConfig?.textFallbackWithoutTools === true && + consumers != null && + hasFallbackText && + !hasTurnFileConsumer(mimeType, consumers) + ) { + return 'text'; + } + return path; +} + /** Why an upload cannot be accepted, when nothing would be able to read it. */ export type UploadRejection = 'no-agent-resource' | 'context-disabled' | 'no-consumer'; diff --git a/packages/data-provider/src/types/files.ts b/packages/data-provider/src/types/files.ts index 3102c8ee72..e7a80d81d6 100644 --- a/packages/data-provider/src/types/files.ts +++ b/packages/data-provider/src/types/files.ts @@ -52,6 +52,9 @@ export type EndpointFileConfig = { supportedMimeTypes?: RegexLike[]; defaultLLMDeliveryPath?: TDefaultLLMDeliveryPathConfig; legacyFileUploadUX?: boolean; + /** Delivers the text extracted at upload for a file routed to tools (`none`) on a turn that + * runs no tool able to read it. Off by default, which leaves such a file out of the prompt. */ + textFallbackWithoutTools?: boolean; }; export type FileConfig = { @@ -88,6 +91,9 @@ export type FileConfig = { checkType?: (fileType: string, supportedTypes: RegexLike[]) => boolean; defaultLLMDeliveryPath?: TDefaultLLMDeliveryPathConfig; legacyFileUploadUX?: boolean; + /** Delivers the text extracted at upload for a file routed to tools (`none`) on a turn that + * runs no tool able to read it. Off by default, which leaves such a file out of the prompt. */ + textFallbackWithoutTools?: boolean; }; export type FileConfigInput = { @@ -119,6 +125,9 @@ export type FileConfigInput = { checkType?: (fileType: string, supportedTypes: RegexLike[]) => boolean; defaultLLMDeliveryPath?: TDefaultLLMDeliveryPathConfig; legacyFileUploadUX?: boolean; + /** Delivers the text extracted at upload for a file routed to tools (`none`) on a turn that + * runs no tool able to read it. Off by default, which leaves such a file out of the prompt. */ + textFallbackWithoutTools?: boolean; }; /** The immutable origin of a file explicitly published from an agent execution. */