From 2f8bc3dfcd5f0e5320df3b8381caf677d1c7c066 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 16:38:51 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=9E=EF=B8=8F=20fix:=20Deliver=20Opted-?= =?UTF-8?q?In=20Custom=20Endpoint=20Media=20Safely=20(#15937)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: send video/audio to custom endpoints that opt in via supportedMimeTypes The video and audio encoders only emitted OpenAI-format parts (video_url, input_audio) for OpenRouter, and Google's media part for Google/Vertex. For every other provider the file was validated and then silently dropped, so a custom OpenAI-compatible endpoint (vLLM, LiteLLM, ...) never received the attachment even though the wire format is identical to OpenRouter's. Emit the OpenAI-format part for OpenAI-like providers when the endpoint's fileConfig supportedMimeTypes explicitly allows the file's type. The inherited default list does not count as opting in (isExplicitMimeConfig, keyed on referential identity like the client's picker), so endpoints whose gateway cannot handle media are unaffected. * feat: let a configured custom endpoint pick and attach video/audio The attach menu's Upload to Provider filter and the drag-drop viability check only opened video/audio for Google and OpenRouter, or for a custom endpoint with a fully permissive (.*) supportedMimeTypes. Honor any admin-configured allowlist on a custom endpoint instead, so a finite list that includes video/.* or audio/.* is enough, matching the server-side encoders. * fix: canonicalize input_audio format from MIME type The `input_audio.format` value was derived from the filename extension via `filename.split('.').pop()`, which produces values providers reject: - `clip.wave` (audio/wave) emitted `wave`, but the accepted format is `wav` - a file with no extension (`recording`) emitted the whole filename, since splitting a dotless string yields a single-element array, so the existing non-empty guard never fired - `clip.mpeg` (audio/mpeg) emitted `mpeg` rather than `mp3` All of these pass MIME validation and then fail at the provider. Derive the format from the MIME type instead, mirroring the canonicalization already in STTService, and fall back to the filename extension only when it is itself a supported format. Throws when neither source yields one rather than sending an unsupported value. Co-Authored-By: Claude Opus 5 (1M context) * fix: route opted-in custom endpoint media to the provider path In unified upload mode an upload carries no tool resource, so its delivery path is inferred by resolveDefaultLLMDeliveryPath. That resolver judged audio and video for any named endpoint with isMediaSupportedProvider, which lists only Google, Vertex and OpenRouter, so a custom endpoint that opted into media through supportedMimeTypes still recorded video as none and audio as text, and the encoder branch was never reached. The upload resolver already holds the endpoint's file config, so it now passes the endpoint's supportedMimeTypes through, and an explicit match on an OpenAI-compatible or custom endpoint counts as provider-capable, mirroring isConfiguredProviderMediaType on the encoder side. The inherited default list is still not an opt-in, and known providers whose encoders emit nothing for media are unchanged. BaseClient re-resolves per turn with the same function, so the turn path follows. * fix: scope the media opt-in to custom endpoints that run as OpenAI The opt-in applied to the built-in openAI and azureOpenAI endpoints too, but the picker and drag-drop only open media for custom endpoints, so that was a route the client could not send to. It is now limited to endpoints that are not a known provider. A custom endpoint may also declare provider: anthropic, in which case initialization runs it as Anthropic and the encoders emit no media part for it. getCustomEndpointProvider reads that declaration off the app config, the upload callers pass it as endpointProvider, the agent encoder and BaseClient pass the agent's resolved provider, and an endpoint known to run as something other than OpenAI keeps its previous route: video stays off the model path and audio still reaches transcription. * fix: reject unsupported provider audio during upload --------- Co-authored-by: Renato Garita Figueiredo Co-authored-by: Claude Opus 5 (1M context) --- api/app/clients/BaseClient.js | 7 + api/server/routes/files/files.agents.test.js | 27 +++ api/server/routes/files/images.js | 2 + api/server/services/Files/process.js | 3 + .../Chat/Input/Files/AttachFileMenu.tsx | 18 +- .../Files/__tests__/AttachFileMenu.spec.tsx | 40 ++++ .../Files/__tests__/useFileHandling.test.ts | 36 ++- client/src/locales/en/translation.json | 1 + .../__tests__/getViableUploadOptions.spec.ts | 27 +++ client/src/utils/files.ts | 10 +- packages/api/src/agents/files/encode.ts | 1 + packages/api/src/files/encode/audio.spec.ts | 211 ++++++++++++++++++ packages/api/src/files/encode/audio.ts | 27 ++- packages/api/src/files/encode/utils.spec.ts | 116 +++++++++- packages/api/src/files/encode/utils.ts | 108 ++++++++- packages/api/src/files/encode/video.spec.ts | 130 +++++++++++ packages/api/src/files/encode/video.ts | 12 +- packages/api/src/files/upload/errors.ts | 9 + packages/api/src/files/upload/routing.spec.ts | 144 +++++++++++- packages/api/src/files/upload/routing.ts | 32 ++- packages/api/src/types/files.ts | 4 +- packages/api/src/utils/files.spec.ts | 9 + packages/api/src/utils/files.ts | 4 + .../data-provider/src/file-config.spec.ts | 30 +++ packages/data-provider/src/file-config.ts | 12 + .../src/resolve-llm-delivery-path.spec.ts | 116 ++++++++++ .../src/resolve-llm-delivery-path.ts | 80 ++++++- 27 files changed, 1184 insertions(+), 32 deletions(-) create mode 100644 packages/api/src/files/encode/audio.spec.ts create mode 100644 packages/api/src/files/encode/video.spec.ts create mode 100644 packages/api/src/files/upload/errors.ts diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index 5ec6e7db96..81f8998760 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -45,6 +45,7 @@ const { resolveUploadLLMDeliveryPath, isSpeechProviderConfigured, resolveUseResponsesApi, + getCustomEndpointProvider, } = require('librechat-data-provider'); const { getStrategyFunctions } = require('~/server/services/Files/strategies'); const { logViolation } = require('~/cache'); @@ -1826,6 +1827,12 @@ class BaseClient { endpointConfig: this._endpointFileConfig, fileConfig: this._mergedFileConfig, endpoint: this._deliveryEndpoint, + endpointProvider: + this.options.agent?.provider ?? + getCustomEndpointProvider( + this.options.req?.config?.endpoints?.custom, + this._deliveryEndpoint, + ), useResponsesApi: this.usesResponsesApi(), sttConfigured: isSpeechProviderConfigured(this.options.req?.config?.speech?.stt), }); diff --git a/api/server/routes/files/files.agents.test.js b/api/server/routes/files/files.agents.test.js index 4323a28e81..45b082601d 100644 --- a/api/server/routes/files/files.agents.test.js +++ b/api/server/routes/files/files.agents.test.js @@ -427,6 +427,33 @@ describe('File Routes - Agent Files Endpoint', () => { return testApp; }; + it.each([ + [false, 'application/json'], + [true, 'text/event-stream'], + ])( + 'rejects unsupported provider audio before persistence (legacy=%s, accept=%s)', + async (legacyFileUploadUX, accept) => { + const testApp = createAppWithUser( + otherUserId, + SystemRoles.USER, + { + fileConfig: { + endpoints: { MyGateway: { supportedMimeTypes: ['audio/.*'], legacyFileUploadUX } }, + }, + }, + { originalname: 'clip.wma', mimetype: 'audio/wma' }, + ); + const response = await request(testApp).post('/files').set('Accept', accept).send({ + endpoint: 'MyGateway', + file_id: uuidv4(), + }); + expect(response.status).toBe(415); + expect(response.body.message).toBe('com_error_files_provider_audio_format'); + expect(processAgentFileUpload).not.toHaveBeenCalled(); + expect(fs.unlink).toHaveBeenCalledWith('/tmp/test.txt'); + }, + ); + it('inspects the canonical sanitized filename used by upload processing', async () => { await createAgent({ id: agentCustomId, diff --git a/api/server/routes/files/images.js b/api/server/routes/files/images.js index 7e5512f0cd..435027a3b0 100644 --- a/api/server/routes/files/images.js +++ b/api/server/routes/files/images.js @@ -23,6 +23,7 @@ const { resolveUploadLLMDeliveryPath, isResponsesApiUpload, isSpeechProviderConfigured, + getCustomEndpointProvider, } = require('librechat-data-provider'); const { processAgentFileUpload, @@ -107,6 +108,7 @@ router.post('/', async (req, res) => { endpointConfig: getEndpointFileConfig({ fileConfig, endpoint: effectiveEndpoint }), fileConfig, endpoint: effectiveEndpoint, + endpointProvider: getCustomEndpointProvider(req.config?.endpoints?.custom, effectiveEndpoint), useResponsesApi: isResponsesApiUpload(metadata.useResponsesApi), sttConfigured: isSpeechProviderConfigured(req.config?.speech?.stt), }); diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js index 0a5705e60c..7ddf9ab729 100644 --- a/api/server/services/Files/process.js +++ b/api/server/services/Files/process.js @@ -25,6 +25,7 @@ const { isMessageFileUpload, isResponsesApiUpload, isSpeechProviderConfigured, + getCustomEndpointProvider, } = require('librechat-data-provider'); const { logger, runAsSystem } = require('@librechat/data-schemas'); const { @@ -494,6 +495,7 @@ const processImageFile = async ({ req, res, metadata, returnFile = false, sseStr endpointConfig, fileConfig, endpoint: configEndpoint, + endpointProvider: getCustomEndpointProvider(appConfig?.endpoints?.custom, configEndpoint), useResponsesApi: isResponsesApiUpload(metadata.useResponsesApi ?? req.body?.useResponsesApi), sttConfigured: isSpeechProviderConfigured(appConfig?.speech?.stt), }); @@ -810,6 +812,7 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { endpointConfig, fileConfig, endpoint, + endpointProvider: getCustomEndpointProvider(appConfig?.endpoints?.custom, endpoint), useResponsesApi: isResponsesApiUpload(metadata.useResponsesApi ?? req.body?.useResponsesApi), sttConfigured: isSpeechProviderConfigured(appConfig?.speech?.stt), }); diff --git a/client/src/components/Chat/Input/Files/AttachFileMenu.tsx b/client/src/components/Chat/Input/Files/AttachFileMenu.tsx index 71f89b03ee..ead2d57bc1 100644 --- a/client/src/components/Chat/Input/Files/AttachFileMenu.tsx +++ b/client/src/components/Chat/Input/Files/AttachFileMenu.tsx @@ -20,6 +20,7 @@ import { Providers, EToolResources, EModelEndpoint, + isExplicitMimeConfig, getConfiguredMimeAccept, bedrockDocumentMimeTypes, defaultAgentCapabilities, @@ -52,7 +53,8 @@ type FileUploadType = | 'document' | 'image_document' | 'image_document_extended' - | 'image_document_video_audio'; + | 'image_document_video_audio' + | 'image_document_video_audio_configured'; /** What each provider upload path can actually send, used to scope the picker filter to selectable files. */ const fileTypeCapabilities: Record = { @@ -68,6 +70,10 @@ const fileTypeCapabilities: Record = { categories: ['image', 'document', 'audio', 'video'], documentMimeTypes: ['application/pdf'], }, + /** Custom endpoint with an admin-configured allowlist: the config decides, including video/audio. */ + image_document_video_audio_configured: { + categories: ['image', 'document', 'audio', 'video'], + }, }; interface AttachFileMenuProps { @@ -164,6 +170,10 @@ const AttachFileMenu = ({ inputRef.current.accept = `image/*,.heif,.heic,${bedrockDocumentExtensions}`; } else if (fileType === 'image_document_video_audio') { inputRef.current.accept = 'image/*,.heif,.heic,.pdf,application/pdf,video/*,audio/*'; + } else if (fileType === 'image_document_video_audio_configured') { + /** Only reached with an explicit allowlist the accept string cannot represent; leave the + * picker open so it never hides a file the admin allowed (backend still enforces). */ + inputRef.current.accept = ''; } else { inputRef.current.accept = ''; } @@ -237,6 +247,11 @@ const AttachFileMenu = ({ endpointType === EModelEndpoint.bedrock ) { fileType = 'image_document_extended'; + } else if ( + endpointType === EModelEndpoint.custom && + isExplicitMimeConfig(endpointFileConfig?.supportedMimeTypes) + ) { + fileType = 'image_document_video_audio_configured'; } onAction(fileType); }, @@ -324,6 +339,7 @@ const AttachFileMenu = ({ handleUploadClick, setEphemeralAgent, sharePointEnabled, + endpointFileConfig?.supportedMimeTypes, codeAllowedByAgent, fileSearchAllowedByAgent, setIsSharePointDialogOpen, diff --git a/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx b/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx index 13d60564e4..7fcd7ba32b 100644 --- a/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx +++ b/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx @@ -208,6 +208,46 @@ describe('AttachFileMenu', () => { }); }); + describe('Upload to Provider picker filter', () => { + /** `accept` is reset right after `click()`, so capture it at click time. */ + const captureAcceptOnClick = () => { + const seen: string[] = []; + const spy = jest.spyOn(HTMLInputElement.prototype, 'click').mockImplementation(function ( + this: HTMLInputElement, + ) { + seen.push(this.accept); + }); + return { seen, spy }; + }; + + it('keeps images and PDFs only for a custom endpoint with the inherited default config', () => { + const { seen, spy } = captureAcceptOnClick(); + setupMocks({ provider: 'MyGateway' }); + renderMenu({ endpointType: EModelEndpoint.custom, endpointFileConfig: {} }); + openMenu(); + fireEvent.click(screen.getByText('Upload to Provider')); + expect(seen).toEqual(['image/*,.heif,.heic,.pdf,application/pdf']); + spy.mockRestore(); + }); + + it('adds video when a custom endpoint config explicitly allows it', () => { + const { seen, spy } = captureAcceptOnClick(); + setupMocks({ provider: 'MyGateway' }); + renderMenu({ + endpointType: EModelEndpoint.custom, + endpointFileConfig: { + supportedMimeTypes: [/^image\/.*$/, /^application\/pdf$/, /^video\/.*$/], + }, + }); + openMenu(); + fireEvent.click(screen.getByText('Upload to Provider')); + expect(seen).toHaveLength(1); + expect(seen[0]).toContain('video/*'); + expect(seen[0]).not.toContain('audio/*'); + spy.mockRestore(); + }); + }); + describe('Upload to Provider vs Upload Image', () => { it('shows "Upload to Provider" when endpointType is custom (resolved from agent provider)', () => { setupMocks({ provider: 'Moonshot' }); diff --git a/client/src/hooks/Files/__tests__/useFileHandling.test.ts b/client/src/hooks/Files/__tests__/useFileHandling.test.ts index bcbc2e1b56..e741324f7d 100644 --- a/client/src/hooks/Files/__tests__/useFileHandling.test.ts +++ b/client/src/hooks/Files/__tests__/useFileHandling.test.ts @@ -1,4 +1,4 @@ -import { renderHook, act } from '@testing-library/react'; +import { renderHook, act, waitFor } from '@testing-library/react'; import { megabyte, Constants, @@ -210,6 +210,40 @@ describe('useFileHandling', () => { const loadHook = async () => (await import('../useFileHandling')).default; + it('removes rejected provider audio and localizes the upload error before retry', async () => { + const consoleLog = jest.spyOn(console, 'log').mockImplementation(() => undefined); + const useFileHandling = await loadHook(); + const { result } = renderHook(() => useFileHandling()); + const recovery = jest.fn(); + await act(async () => { + await result.current.handleFiles( + [new File(['audio'], 'clip.wma', { type: 'audio/wma' })], + undefined, + { onError: recovery }, + ); + }); + const body = mockMutate.mock.calls[0][0] as FormData; + const fileId = body.get('file_id'); + act(() => + mockUploadOptions.onError?.( + { + response: { status: 415, data: { message: 'com_error_files_provider_audio_format' } }, + }, + body, + ), + ); + expect(mockDeleteFileById).toHaveBeenCalledWith(fileId); + await waitFor(() => + expect(mockLocalize).toHaveBeenCalledWith('com_error_files_provider_audio_format'), + ); + expect(recovery).toHaveBeenCalledWith(fileId); + await act(async () => { + await result.current.handleFiles([new File(['audio'], 'clip.wav', { type: 'audio/wav' })]); + }); + expect(mockMutate).toHaveBeenCalledTimes(2); + consoleLog.mockRestore(); + }); + describe('endpointOverride', () => { it('clears the loading state when file validation throws', async () => { const consoleError = jest.spyOn(console, 'error').mockImplementation(() => undefined); diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.json index 64de2fe31b..6ca1ba26a7 100644 --- a/client/src/locales/en/translation.json +++ b/client/src/locales/en/translation.json @@ -1089,6 +1089,7 @@ "com_ui_attach_error_size": "File size limit exceeded for endpoint:", "com_ui_attach_error_total_size": "Total file size limit exceeded for endpoint:", "com_ui_attach_error_type": "Unsupported file type for endpoint:", + "com_error_files_provider_audio_format": "This audio format cannot be attached directly to this model. Convert the file to WAV or MP3 and upload it again, or choose Upload as Text if transcription is configured.", "com_ui_attach_remove": "Remove file", "com_ui_attach_warn_endpoint": "Non-Assistant files may be ignored without a compatible tool", "com_ui_attached_image": "Attached image", diff --git a/client/src/utils/__tests__/getViableUploadOptions.spec.ts b/client/src/utils/__tests__/getViableUploadOptions.spec.ts index 261ea70e40..6d2ea63580 100644 --- a/client/src/utils/__tests__/getViableUploadOptions.spec.ts +++ b/client/src/utils/__tests__/getViableUploadOptions.spec.ts @@ -132,6 +132,33 @@ describe('getViableUploadOptions', () => { expect(getViableUploadOptions([file(XLSX, 'report.xlsx')], ctx)).toEqual([undefined]); }); + it('offers direct attach for a video when the custom config explicitly allows video', () => { + const ctx = baseCtx({ + provider: 'MyGateway', + endpoint: 'MyGateway', + endpointType: 'custom', + fileSearchEnabled: false, + codeEnabled: false, + contextEnabled: false, + endpointSupportedMimeTypes: [/^image\/.*$/, /^application\/pdf$/, /^video\/.*$/], + }); + expect(getViableUploadOptions([file('video/mp4', 'clip.mp4')], ctx)).toEqual([undefined]); + expect(getViableUploadOptions([file('audio/wav', 'tone.wav')], ctx)).toEqual([]); + }); + + it('does not offer video for a custom endpoint that inherits the default config', () => { + const ctx = baseCtx({ + provider: 'MyGateway', + endpoint: 'MyGateway', + endpointType: 'custom', + fileSearchEnabled: false, + codeEnabled: false, + contextEnabled: false, + endpointSupportedMimeTypes: undefined, + }); + expect(getViableUploadOptions([file('video/mp4', 'clip.mp4')], ctx)).toEqual([]); + }); + it('does not treat a non-permissive custom config as broad provider support', () => { const ctx = baseCtx({ provider: 'MyGateway', diff --git a/client/src/utils/files.ts b/client/src/utils/files.ts index 62e77dacaf..857742f98a 100644 --- a/client/src/utils/files.ts +++ b/client/src/utils/files.ts @@ -16,7 +16,7 @@ import { EModelEndpoint, retrievalMimeTypes, isBedrockDocumentType, - isPermissiveMimeConfig, + isExplicitMimeConfig, codeInterpreterMimeTypes, isDocumentSupportedProvider, fileConfig as defaultFileConfig, @@ -522,12 +522,12 @@ const isProviderAttachType = (type: string, ctx: UploadOptionContext): boolean = isDocumentSupportedProvider(currentProvider) || isAzureWithResponsesApi ) { - /** Custom endpoints that the admin opened up (permissive config) honor that allowlist, - * matching the file picker; an inherited default config is not treated as opened up. */ + /** Custom endpoints with an admin-configured allowlist honor it for direct attach (this is + * how video/audio get opted in for an OpenAI-compatible gateway), matching the file picker + * and the server-side encoders; an inherited default config is not treated as opened up. */ if ( ctx.endpointType === EModelEndpoint.custom && - ctx.endpointSupportedMimeTypes != null && - isPermissiveMimeConfig(ctx.endpointSupportedMimeTypes) + isExplicitMimeConfig(ctx.endpointSupportedMimeTypes) ) { return checkType(type, ctx.endpointSupportedMimeTypes); } diff --git a/packages/api/src/agents/files/encode.ts b/packages/api/src/agents/files/encode.ts index 63b273bd76..d4de921577 100644 --- a/packages/api/src/agents/files/encode.ts +++ b/packages/api/src/agents/files/encode.ts @@ -103,6 +103,7 @@ export function createRunFileMessageEncoder( endpointConfig, fileConfig, endpoint, + endpointProvider: agent.provider, useResponsesApi, sttConfigured: isSpeechProviderConfigured(deps.req.config?.speech?.stt), }), diff --git a/packages/api/src/files/encode/audio.spec.ts b/packages/api/src/files/encode/audio.spec.ts new file mode 100644 index 0000000000..e795042fd6 --- /dev/null +++ b/packages/api/src/files/encode/audio.spec.ts @@ -0,0 +1,211 @@ +import { Providers } from '@librechat/agents'; +import type { IMongoFile } from '@librechat/data-schemas'; +import type { ServerRequest } from '~/types'; +import { encodeAndFormatAudios } from './audio'; + +jest.mock('~/files/validation', () => ({ + validateAudio: jest.fn(), +})); + +jest.mock('./utils', () => ({ + ...jest.requireActual('./utils'), + getFileStream: jest.fn(), + getConfiguredFileSizeLimit: jest.fn(), + isConfiguredProviderMediaType: jest.fn(), +})); + +jest.mock('./memoryGuard', () => ({ + runGuardedEncode: jest.fn((_bytes: number, fn: () => unknown) => fn()), +})); + +import { validateAudio } from '~/files/validation'; +import { getFileStream, isConfiguredProviderMediaType } from './utils'; +import { Types } from 'mongoose'; + +const mockedValidateAudio = validateAudio as jest.MockedFunction; +const mockedGetFileStream = getFileStream as jest.MockedFunction; +const mockedIsConfigured = isConfiguredProviderMediaType as jest.MockedFunction< + typeof isConfiguredProviderMediaType +>; + +const createMockFile = (type = 'audio/wav', filename = 'tone.wav'): IMongoFile => + ({ + _id: new Types.ObjectId(), + file_id: 'audio-1', + filename, + filepath: `/uploads/${filename}`, + type, + bytes: 1024, + source: 'local', + user: 'user-1', + object: 'file', + usage: 0, + createdAt: new Date(), + updatedAt: new Date(), + }) as unknown as IMongoFile; + +const req = { config: {} } as unknown as ServerRequest; +const getStrategyFunctions = jest.fn(); + +describe('encodeAndFormatAudios - provider formatting', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockedValidateAudio.mockResolvedValue({ isValid: true }); + mockedIsConfigured.mockReturnValue(false); + const file = createMockFile(); + mockedGetFileStream.mockResolvedValue({ + file, + content: 'AAAA', + metadata: { + file_id: file.file_id, + filepath: file.filepath, + source: file.source, + filename: file.filename, + type: file.type, + }, + }); + }); + + it('emits a Google media block for google', async () => { + const result = await encodeAndFormatAudios( + req, + [createMockFile()], + { provider: Providers.GOOGLE }, + getStrategyFunctions, + ); + expect(result.audios).toEqual([{ type: 'media', mimeType: 'audio/wav', data: 'AAAA' }]); + }); + + it('emits an OpenAI-compatible input_audio block for openrouter without configuration', async () => { + const result = await encodeAndFormatAudios( + req, + [createMockFile()], + { provider: Providers.OPENROUTER }, + getStrategyFunctions, + ); + expect(mockedIsConfigured).not.toHaveBeenCalled(); + expect(result.audios).toEqual([ + { type: 'input_audio', input_audio: { data: 'AAAA', format: 'wav' } }, + ]); + }); + + it('drops the audio for a custom endpoint that is not configured for audio', async () => { + const result = await encodeAndFormatAudios( + req, + [createMockFile()], + { provider: Providers.OPENAI, endpoint: 'MyGateway' }, + getStrategyFunctions, + ); + expect(mockedIsConfigured).toHaveBeenCalledWith( + req, + { provider: Providers.OPENAI, endpoint: 'MyGateway' }, + 'audio/wav', + ); + expect(result.audios).toEqual([]); + expect(result.files).toHaveLength(1); + }); + + it('emits an input_audio block for a custom endpoint whose config allows audio', async () => { + mockedIsConfigured.mockReturnValue(true); + const result = await encodeAndFormatAudios( + req, + [createMockFile()], + { provider: Providers.OPENAI, endpoint: 'MyGateway' }, + getStrategyFunctions, + ); + expect(result.audios).toEqual([ + { type: 'input_audio', input_audio: { data: 'AAAA', format: 'wav' } }, + ]); + }); + + it('never emits a block for providers without an audio path, even when configured', async () => { + mockedIsConfigured.mockReturnValue(true); + const result = await encodeAndFormatAudios( + req, + [createMockFile()], + { provider: Providers.ANTHROPIC }, + getStrategyFunctions, + ); + expect(result.audios).toEqual([]); + }); + + describe('input_audio format canonicalization', () => { + /** Re-points the `getFileStream` mock at a specific file for one test. */ + const stageFile = (file: IMongoFile) => { + mockedGetFileStream.mockResolvedValue({ + file, + content: 'AAAA', + metadata: { + file_id: file.file_id, + filepath: file.filepath, + source: file.source, + filename: file.filename, + type: file.type, + }, + }); + }; + + const formatFor = async (file: IMongoFile) => { + stageFile(file); + mockedIsConfigured.mockReturnValue(true); + const result = await encodeAndFormatAudios( + req, + [file], + { provider: Providers.OPENAI, endpoint: 'MyGateway' }, + getStrategyFunctions, + ); + return (result.audios[0] as { input_audio?: { format?: string } })?.input_audio?.format; + }; + + it.each([ + ['audio/wave', 'clip.wave', 'wav'], + ['audio/x-wav', 'clip.x-wav', 'wav'], + ['audio/mpeg', 'clip.mpeg', 'mp3'], + ['audio/mpeg3', 'clip.mpeg3', 'mp3'], + ['audio/x-m4a', 'clip.x-m4a', 'm4a'], + ['audio/vorbis', 'clip.vorbis', 'ogg'], + ['audio/x-flac', 'clip.x-flac', 'flac'], + ])('canonicalizes %s to a provider-accepted format', async (type, filename, expected) => { + expect(await formatFor(createMockFile(type, filename))).toBe(expected); + }); + + it('derives the format from the MIME type when the filename has no extension', async () => { + expect(await formatFor(createMockFile('audio/mpeg', 'recording'))).toBe('mp3'); + }); + + it('prefers the canonical MIME format over a mismatched extension', async () => { + expect(await formatFor(createMockFile('audio/mpeg', 'clip.wav'))).toBe('mp3'); + }); + + it('falls back to a supported filename extension for MIME types with no mapping', async () => { + expect(await formatFor(createMockFile('audio/aac', 'clip.aac'))).toBe('aac'); + }); + + it('throws rather than emitting an unsupported format', async () => { + /** `audio/wma` passes MIME validation but has no provider-accepted format. */ + stageFile(createMockFile('audio/wma', 'recording')); + mockedIsConfigured.mockReturnValue(true); + await expect( + encodeAndFormatAudios( + req, + [createMockFile('audio/wma', 'recording')], + { provider: Providers.OPENAI, endpoint: 'MyGateway' }, + getStrategyFunctions, + ), + ).rejects.toThrow(/supported audio format/i); + }); + + it('canonicalizes for openrouter as well', async () => { + stageFile(createMockFile('audio/wave', 'clip.wave')); + const result = await encodeAndFormatAudios( + req, + [createMockFile('audio/wave', 'clip.wave')], + { provider: Providers.OPENROUTER }, + getStrategyFunctions, + ); + expect(result.audios).toEqual([ + { type: 'input_audio', input_audio: { data: 'AAAA', format: 'wav' } }, + ]); + }); + }); +}); diff --git a/packages/api/src/files/encode/audio.ts b/packages/api/src/files/encode/audio.ts index fa155cd3b3..ac51c8f772 100644 --- a/packages/api/src/files/encode/audio.ts +++ b/packages/api/src/files/encode/audio.ts @@ -1,11 +1,13 @@ import { Providers } from '@librechat/agents'; -import { isDocumentSupportedProvider } from 'librechat-data-provider'; +import { isDocumentSupportedProvider, isOpenAILikeProvider } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { ServerRequest, StrategyFunctions, AudioResult } from '~/types'; import { getFileStream, + getAudioFormat, getConfiguredFileSizeLimit, isAttachmentObjectNotFoundError, + isConfiguredProviderMediaType, } from './utils'; import { validateAudio } from '~/files/validation'; import { runGuardedEncode } from './memoryGuard'; @@ -15,7 +17,7 @@ import { runGuardedEncode } from './memoryGuard'; * @param req - The request object * @param files - Array of audio files * @param params - Object containing provider and optional endpoint - * @param params.provider - The provider to format for (currently only google is supported) + * @param params.provider - The provider to format for * @param params.endpoint - Optional endpoint name for file config lookup * @param getStrategyFunctions - Function to get strategy functions * @returns Promise that resolves to audio and file metadata @@ -91,13 +93,22 @@ export async function encodeAndFormatAudios( mimeType: file.type, data: content, }); - } else if (provider === Providers.OPENROUTER) { - // Extract format from filename extension (e.g., 'audio.mp3' -> 'mp3') - // OpenRouter expects format values like: wav, mp3, aiff, aac, ogg, flac, m4a, pcm16, pcm24 - // Note: MIME types don't always match (e.g., 'audio/mpeg' is mp3, not mpeg), so that is why we are using the file extension instead - const format = file.filename.split('.').pop()?.toLowerCase(); + } else if ( + provider === Providers.OPENROUTER || + (isOpenAILikeProvider(provider) && + isConfiguredProviderMediaType(req, { provider, endpoint }, file.type)) + ) { + /** OpenAI-compatible `input_audio` part (OpenRouter, vLLM, and other gateways). + * Custom endpoints only get here when the admin listed audio types in the + * endpoint's `supportedMimeTypes`, since not every gateway accepts audio. */ + /** Canonicalized from the MIME type: the accepted MIME list contains aliases + * (`audio/wave`, `audio/mpeg`) that are not valid format values, and the filename + * may carry no extension at all. */ + const format = getAudioFormat(file.type, file.filename); if (!format) { - throw new Error(`Could not extract audio format from filename: ${file.filename}`); + throw new Error( + `Could not determine a supported audio format for "${file.filename}" (${file.type})`, + ); } result.audios.push({ type: 'input_audio', diff --git a/packages/api/src/files/encode/utils.spec.ts b/packages/api/src/files/encode/utils.spec.ts index 24367409c7..d4882d952d 100644 --- a/packages/api/src/files/encode/utils.spec.ts +++ b/packages/api/src/files/encode/utils.spec.ts @@ -1,6 +1,13 @@ import { Readable } from 'node:stream'; +import { Providers } from '@librechat/agents'; +import { audioMimeTypes } from 'librechat-data-provider'; import type { ServerRequest } from '~/types'; -import { AttachmentObjectNotFoundError, getFileStream } from './utils'; +import { + AttachmentObjectNotFoundError, + getAudioFormat, + getFileStream, + isConfiguredProviderMediaType, +} from './utils'; const file = { file_id: 'file-1', @@ -84,3 +91,110 @@ describe('getFileStream', () => { expect(getDownloadStream).toHaveBeenCalledWith(req, legacyUrl); }); }); + +/** Uses the real data-provider merge logic so the "inherited default" identity check is exercised. */ +const reqWith = (fileConfig: unknown): ServerRequest => + ({ config: fileConfig === undefined ? undefined : { fileConfig } }) as unknown as ServerRequest; + +describe('isConfiguredProviderMediaType', () => { + const params = { provider: Providers.OPENAI, endpoint: 'MyGateway' }; + + it('is false without any fileConfig', () => { + expect(isConfiguredProviderMediaType(reqWith(undefined), params, 'video/mp4')).toBe(false); + }); + + it('is false when the endpoint only inherits the built-in default list', () => { + const req = reqWith({ endpoints: { OtherEndpoint: { fileLimit: 3 } } }); + expect(isConfiguredProviderMediaType(req, params, 'video/mp4')).toBe(false); + expect(isConfiguredProviderMediaType(req, params, 'audio/wav')).toBe(false); + }); + + it('is true when the endpoint config explicitly lists the media type', () => { + const req = reqWith({ + endpoints: { MyGateway: { supportedMimeTypes: ['image/.*', 'application/pdf', 'video/.*'] } }, + }); + expect(isConfiguredProviderMediaType(req, params, 'video/mp4')).toBe(true); + expect(isConfiguredProviderMediaType(req, params, 'audio/wav')).toBe(false); + }); + + it('is true for a permissive config', () => { + const req = reqWith({ endpoints: { MyGateway: { supportedMimeTypes: ['.*'] } } }); + expect(isConfiguredProviderMediaType(req, params, 'audio/wav')).toBe(true); + }); + + it('falls back to the provider key when no endpoint is given', () => { + const req = reqWith({ endpoints: { openAI: { supportedMimeTypes: ['audio/.*'] } } }); + expect(isConfiguredProviderMediaType(req, { provider: Providers.OPENAI }, 'audio/wav')).toBe( + true, + ); + }); +}); + +describe('getAudioFormat', () => { + it.each([ + ['audio/mp3', 'mp3'], + ['audio/mpeg', 'mp3'], + ['audio/mpeg3', 'mp3'], + ['audio/wav', 'wav'], + ['audio/wave', 'wav'], + ['audio/x-wav', 'wav'], + ['audio/ogg', 'ogg'], + ['audio/vorbis', 'ogg'], + ['audio/mp4', 'm4a'], + ['audio/x-m4a', 'm4a'], + ['audio/flac', 'flac'], + ['audio/x-flac', 'flac'], + ])('maps %s to %s regardless of the filename', (mimeType, expected) => { + expect(getAudioFormat(mimeType, 'clip.bogus')).toBe(expected); + expect(getAudioFormat(mimeType, 'recording')).toBe(expected); + }); + + it('is case-insensitive for the MIME type', () => { + expect(getAudioFormat('AUDIO/WAVE', 'clip.wave')).toBe('wav'); + }); + + it('falls back to a supported extension when the MIME type is unmapped', () => { + expect(getAudioFormat('audio/unknown', 'clip.mp3')).toBe('mp3'); + expect(getAudioFormat('audio/unknown', 'clip.PCM16')).toBe('pcm16'); + }); + + it('rejects an unsupported extension rather than passing it through', () => { + expect(getAudioFormat('audio/unknown', 'clip.wave')).toBeUndefined(); + expect(getAudioFormat('audio/unknown', 'clip.exe')).toBeUndefined(); + }); + + it('returns undefined when the filename has no extension and the MIME is unmapped', () => { + expect(getAudioFormat('audio/wma', 'recording')).toBeUndefined(); + expect(getAudioFormat('', '')).toBeUndefined(); + }); + + it('does not treat a leading-dot filename as an extension', () => { + expect(getAudioFormat('audio/unknown', '.mp3')).toBeUndefined(); + }); + + /** Guards against the accepted-MIME list drifting ahead of the format mapping. */ + it('resolves a format for every accepted audio MIME type that has one', () => { + const accepted = [ + 'audio/mp3', + 'audio/mpeg', + 'audio/mpeg3', + 'audio/wav', + 'audio/wave', + 'audio/x-wav', + 'audio/ogg', + 'audio/vorbis', + 'audio/mp4', + 'audio/m4a', + 'audio/x-m4a', + 'audio/flac', + 'audio/x-flac', + 'audio/webm', + 'audio/aac', + 'audio/opus', + ]; + for (const mimeType of accepted) { + expect(audioMimeTypes.test(mimeType)).toBe(true); + expect(getAudioFormat(mimeType, 'recording')).toBeDefined(); + } + }); +}); diff --git a/packages/api/src/files/encode/utils.ts b/packages/api/src/files/encode/utils.ts index 7d7ac21a3a..9c39ce2c13 100644 --- a/packages/api/src/files/encode/utils.ts +++ b/packages/api/src/files/encode/utils.ts @@ -1,6 +1,11 @@ import getStream from 'get-stream'; import { Providers } from '@librechat/agents'; -import { FileSources, mergeFileConfig, getEndpointFileConfig } from 'librechat-data-provider'; +import { + FileSources, + mergeFileConfig, + isExplicitMimeConfig, + getEndpointFileConfig, +} from 'librechat-data-provider'; import type { ServerRequest, StrategyFunctions, ProcessedFile } from '~/types'; import type { StoredFileRef } from '~/storage/path'; import { resolveDownloadPath } from '~/storage/path'; @@ -70,6 +75,107 @@ export const getConfiguredFileSizeLimit = ( return endpointConfig?.fileSizeLimit; }; +/** + * Whether the admin explicitly allowed `mimeType` for this endpoint via + * `fileConfig.endpoints..supportedMimeTypes`. The inherited default list does + * not count as opting in, mirroring the client's picker and drag-drop logic, so an + * OpenAI-compatible endpoint only receives video/audio parts when configured for them. + * @param req - The server request object containing config + * @param params - Object containing provider and optional endpoint + * @param params.provider - The provider to look up + * @param params.endpoint - Optional endpoint name for lookup + * @param mimeType - The MIME type of the file being attached + * @returns True when the endpoint config explicitly matches the MIME type + */ +export const isConfiguredProviderMediaType = ( + req: ServerRequest, + params: { + provider: Providers; + endpoint?: string; + }, + mimeType: string, +): boolean => { + if (!req.config?.fileConfig) { + return false; + } + const { provider, endpoint } = params; + const fileConfig = mergeFileConfig(req.config.fileConfig); + const endpointConfig = getEndpointFileConfig({ + fileConfig, + endpoint: endpoint ?? provider, + }); + const types = endpointConfig?.supportedMimeTypes; + if (!isExplicitMimeConfig(types)) { + return false; + } + return fileConfig.checkType?.(mimeType, types) ?? false; +}; + +/** + * Maps accepted audio MIME types to the format token OpenAI-compatible providers expect. + * Mirrors the canonicalization in `api/server/services/Files/Audio/STTService.js`: the + * accepted MIME list carries aliases (`audio/wave`, `audio/x-wav`, `audio/mpeg`) whose + * names are not themselves valid format values. + */ +const audioMimeToFormat: Record = { + 'audio/mp3': 'mp3', + 'audio/mpeg': 'mp3', + 'audio/mpeg3': 'mp3', + 'audio/wav': 'wav', + 'audio/wave': 'wav', + 'audio/x-wav': 'wav', + 'audio/ogg': 'ogg', + 'audio/vorbis': 'ogg', + 'audio/opus': 'ogg', + 'audio/mp4': 'm4a', + 'audio/m4a': 'm4a', + 'audio/x-m4a': 'm4a', + 'audio/flac': 'flac', + 'audio/x-flac': 'flac', + 'audio/webm': 'webm', + 'audio/aac': 'aac', +}; + +/** + * Format values OpenAI-compatible providers accept for an `input_audio` part. + * Used to validate an extension-derived fallback so an arbitrary filename suffix + * never reaches the provider as a format. + */ +const supportedAudioFormats = new Set([ + 'wav', + 'mp3', + 'aiff', + 'aac', + 'ogg', + 'flac', + 'm4a', + 'webm', + 'pcm16', + 'pcm24', +]); + +/** + * Resolves the `input_audio.format` value for a file. + * + * Prefers the MIME type, which is validated upstream against `audioMimeTypes`, because + * the filename extension is user-controlled: it may be absent (`recording`), an alias + * that is not a valid format (`clip.wave`), or disagree with the actual content. Falls + * back to the extension only when it is itself a supported format. + * @param mimeType - The file's MIME type + * @param filename - The original filename + * @returns The canonical format, or undefined when neither source yields a supported one + */ +export const getAudioFormat = (mimeType: string, filename: string): string | undefined => { + const fromMime = audioMimeToFormat[mimeType?.toLowerCase()]; + if (fromMime) { + return fromMime; + } + const parts = filename?.toLowerCase().split('.') ?? []; + /** `parts[0]` must be non-empty so a dotfile (`.mp3`) is not read as an extension. */ + const extension = parts.length > 1 && parts[0] ? parts[parts.length - 1] : undefined; + return extension && supportedAudioFormats.has(extension) ? extension : undefined; +}; + /** * Processes a file by downloading and encoding it to base64 * @param req - Express request object diff --git a/packages/api/src/files/encode/video.spec.ts b/packages/api/src/files/encode/video.spec.ts new file mode 100644 index 0000000000..06da3028d0 --- /dev/null +++ b/packages/api/src/files/encode/video.spec.ts @@ -0,0 +1,130 @@ +import { Providers } from '@librechat/agents'; +import type { IMongoFile } from '@librechat/data-schemas'; +import type { ServerRequest } from '~/types'; +import { encodeAndFormatVideos } from './video'; + +jest.mock('~/files/validation', () => ({ + validateVideo: jest.fn(), +})); + +jest.mock('./utils', () => ({ + getFileStream: jest.fn(), + getConfiguredFileSizeLimit: jest.fn(), + isConfiguredProviderMediaType: jest.fn(), +})); + +jest.mock('./memoryGuard', () => ({ + runGuardedEncode: jest.fn((_bytes: number, fn: () => unknown) => fn()), +})); + +import { validateVideo } from '~/files/validation'; +import { getFileStream, isConfiguredProviderMediaType } from './utils'; +import { Types } from 'mongoose'; + +const mockedValidateVideo = validateVideo as jest.MockedFunction; +const mockedGetFileStream = getFileStream as jest.MockedFunction; +const mockedIsConfigured = isConfiguredProviderMediaType as jest.MockedFunction< + typeof isConfiguredProviderMediaType +>; + +const createMockFile = (type = 'video/mp4'): IMongoFile => + ({ + _id: new Types.ObjectId(), + file_id: 'video-1', + filename: 'clip.mp4', + filepath: '/uploads/clip.mp4', + type, + bytes: 1024, + source: 'local', + user: 'user-1', + object: 'file', + usage: 0, + createdAt: new Date(), + updatedAt: new Date(), + }) as unknown as IMongoFile; + +const req = { config: {} } as unknown as ServerRequest; +const getStrategyFunctions = jest.fn(); + +describe('encodeAndFormatVideos - provider formatting', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockedValidateVideo.mockResolvedValue({ isValid: true }); + mockedIsConfigured.mockReturnValue(false); + const file = createMockFile(); + mockedGetFileStream.mockResolvedValue({ + file, + content: 'AAAA', + metadata: { + file_id: file.file_id, + filepath: file.filepath, + source: file.source, + filename: file.filename, + type: file.type, + }, + }); + }); + + it('emits a Google media block for google', async () => { + const result = await encodeAndFormatVideos( + req, + [createMockFile()], + { provider: Providers.GOOGLE }, + getStrategyFunctions, + ); + expect(result.videos).toEqual([{ type: 'media', mimeType: 'video/mp4', data: 'AAAA' }]); + }); + + it('emits an OpenAI-compatible video_url block for openrouter without configuration', async () => { + const result = await encodeAndFormatVideos( + req, + [createMockFile()], + { provider: Providers.OPENROUTER }, + getStrategyFunctions, + ); + expect(mockedIsConfigured).not.toHaveBeenCalled(); + expect(result.videos).toEqual([ + { type: 'video_url', video_url: { url: 'data:video/mp4;base64,AAAA' } }, + ]); + }); + + it('drops the video for a custom endpoint that is not configured for video', async () => { + const result = await encodeAndFormatVideos( + req, + [createMockFile()], + { provider: Providers.OPENAI, endpoint: 'MyGateway' }, + getStrategyFunctions, + ); + expect(mockedIsConfigured).toHaveBeenCalledWith( + req, + { provider: Providers.OPENAI, endpoint: 'MyGateway' }, + 'video/mp4', + ); + expect(result.videos).toEqual([]); + expect(result.files).toHaveLength(1); + }); + + it('emits a video_url block for a custom endpoint whose config allows video', async () => { + mockedIsConfigured.mockReturnValue(true); + const result = await encodeAndFormatVideos( + req, + [createMockFile()], + { provider: Providers.OPENAI, endpoint: 'MyGateway' }, + getStrategyFunctions, + ); + expect(result.videos).toEqual([ + { type: 'video_url', video_url: { url: 'data:video/mp4;base64,AAAA' } }, + ]); + }); + + it('never emits a block for providers without a video path, even when configured', async () => { + mockedIsConfigured.mockReturnValue(true); + const result = await encodeAndFormatVideos( + req, + [createMockFile()], + { provider: Providers.ANTHROPIC }, + getStrategyFunctions, + ); + expect(result.videos).toEqual([]); + }); +}); diff --git a/packages/api/src/files/encode/video.ts b/packages/api/src/files/encode/video.ts index 52a9331c8d..2715b36eed 100644 --- a/packages/api/src/files/encode/video.ts +++ b/packages/api/src/files/encode/video.ts @@ -1,11 +1,12 @@ import { Providers } from '@librechat/agents'; -import { isDocumentSupportedProvider } from 'librechat-data-provider'; +import { isDocumentSupportedProvider, isOpenAILikeProvider } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { ServerRequest, StrategyFunctions, VideoResult } from '~/types'; import { getFileStream, getConfiguredFileSizeLimit, isAttachmentObjectNotFoundError, + isConfiguredProviderMediaType, } from './utils'; import { validateVideo } from '~/files/validation'; import { runGuardedEncode } from './memoryGuard'; @@ -91,7 +92,14 @@ export async function encodeAndFormatVideos( mimeType: file.type, data: content, }); - } else if (provider === Providers.OPENROUTER) { + } else if ( + provider === Providers.OPENROUTER || + (isOpenAILikeProvider(provider) && + isConfiguredProviderMediaType(req, { provider, endpoint }, file.type)) + ) { + /** OpenAI-compatible `video_url` part (OpenRouter, vLLM, and other gateways). + * Custom endpoints only get here when the admin listed video types in the + * endpoint's `supportedMimeTypes`, since not every gateway accepts video. */ result.videos.push({ type: 'video_url', video_url: { diff --git a/packages/api/src/files/upload/errors.ts b/packages/api/src/files/upload/errors.ts new file mode 100644 index 0000000000..11e579abb8 --- /dev/null +++ b/packages/api/src/files/upload/errors.ts @@ -0,0 +1,9 @@ +/** A direct audio attachment must be representable before the upload is persisted. */ +export class UnsupportedProviderAudioError extends Error { + readonly userErrorStatusCode = 415; + + constructor() { + super('com_error_files_provider_audio_format'); + this.name = 'UnsupportedProviderAudioError'; + } +} diff --git a/packages/api/src/files/upload/routing.spec.ts b/packages/api/src/files/upload/routing.spec.ts index 2d8de0dc8a..71ae6289ff 100644 --- a/packages/api/src/files/upload/routing.spec.ts +++ b/packages/api/src/files/upload/routing.spec.ts @@ -1,6 +1,10 @@ -import { EModelEndpoint } from 'librechat-data-provider'; +import { Readable } from 'node:stream'; +import { EModelEndpoint, EToolResources, Providers } from 'librechat-data-provider'; +import type { IMongoFile } from '@librechat/data-schemas'; import type { ServerRequest } from '~/types'; import { resolveUploadEndpoint, resolveEffectiveToolResource } from './routing'; +import { encodeAndFormatAudios } from '~/files/encode/audio'; +import { UnsupportedProviderAudioError } from './errors'; describe('resolveUploadEndpoint', () => { const req = { user: { id: 'user-1' } } as unknown as ServerRequest; @@ -104,3 +108,141 @@ describe('resolveEffectiveToolResource Responses handling', () => { expect(usable).toBe('context'); }); }); + +describe('direct provider audio preflight', () => { + const makeReq = (legacyFileUploadUX = false, mimetype = 'audio/wma', originalname = 'clip.wma') => + Object.assign({} as ServerRequest, { + file: { + mimetype, + originalname, + fieldname: 'file', + encoding: '7bit', + size: 16, + destination: '/uploads', + filename: originalname, + path: '/uploads/audio-1', + stream: Readable.from(Buffer.alloc(16)), + buffer: Buffer.alloc(16), + }, + config: { + fileConfig: { + endpoints: { MyGateway: { supportedMimeTypes: ['audio/.*'], legacyFileUploadUX } }, + }, + }, + }); + + it.each([false, true])( + 'rejects unsupported audio before processing (legacy=%s)', + async (legacy) => { + await expect( + resolveEffectiveToolResource({ + req: makeReq(legacy), + metadata: { endpoint: 'MyGateway' }, + getAgent: jest.fn(), + }), + ).rejects.toMatchObject({ + message: 'com_error_files_provider_audio_format', + userErrorStatusCode: 415, + }); + }, + ); + + it('uses the resolved agent endpoint and reuses the agent read', async () => { + const req = makeReq(); + const getAgent = jest.fn().mockResolvedValue({ provider: 'MyGateway' }); + const metadata = { endpoint: EModelEndpoint.agents, agent_id: 'agent_saved01' }; + await resolveUploadEndpoint({ req, metadata, getAgent }); + await expect(resolveEffectiveToolResource({ req, metadata, getAgent })).rejects.toBeInstanceOf( + UnsupportedProviderAudioError, + ); + expect(getAgent).toHaveBeenCalledTimes(1); + }); + + it.each([EToolResources.context, EToolResources.ocr, EToolResources.execute_code])( + 'preserves explicit %s uploads', + async (tool_resource) => { + await expect( + resolveEffectiveToolResource({ + req: makeReq(true), + metadata: { endpoint: 'MyGateway', tool_resource }, + getAgent: jest.fn(), + }), + ).resolves.toBe( + tool_resource === EToolResources.ocr ? EToolResources.context : tool_resource, + ); + }, + ); + + it('preserves audio explicitly routed to transcription', async () => { + const req = makeReq(); + req.config!.fileConfig!.defaultLLMDeliveryPath = { overrides: { 'audio/*': 'text' } }; + await expect( + resolveEffectiveToolResource({ + req, + metadata: { endpoint: 'MyGateway' }, + getAgent: jest.fn(), + }), + ).resolves.toBe(EToolResources.context); + }); + + it.each([EModelEndpoint.google, Providers.VERTEXAI])( + 'does not apply input_audio rules to %s', + async (endpoint) => { + await expect( + resolveEffectiveToolResource({ + req: makeReq(), + metadata: { endpoint }, + getAgent: jest.fn(), + }), + ).resolves.toBeUndefined(); + }, + ); + + it('also rejects unsupported OpenRouter audio', async () => { + await expect( + resolveEffectiveToolResource({ + req: makeReq(), + metadata: { endpoint: Providers.OPENROUTER }, + getAgent: jest.fn(), + }), + ).rejects.toBeInstanceOf(UnsupportedProviderAudioError); + }); + + it.each([ + ['audio/wave', 'clip.wave', 'wav'], + ['audio/mpeg', 'recording', 'mp3'], + ['audio/unknown', 'clip.pcm16', 'pcm16'], + ])( + 'delivers an accepted %s upload through the real encoder', + async (mimetype, originalname, format) => { + const req = makeReq(false, mimetype, originalname); + await expect( + resolveEffectiveToolResource({ + req, + metadata: { endpoint: 'MyGateway' }, + getAgent: jest.fn(), + }), + ).resolves.toBeUndefined(); + const file = { + file_id: 'audio-1', + filename: originalname, + filepath: '/uploads/audio-1', + type: mimetype, + bytes: 16, + source: 'local', + } as IMongoFile; + const result = await encodeAndFormatAudios( + req, + [file], + { provider: Providers.OPENAI, endpoint: 'MyGateway' }, + () => ({ getDownloadStream: async () => Readable.from(Buffer.alloc(16)) }), + ); + expect(result.audios).toEqual([ + { + type: 'input_audio', + input_audio: { data: Buffer.alloc(16).toString('base64'), format }, + }, + ]); + }, + ); +}); diff --git a/packages/api/src/files/upload/routing.ts b/packages/api/src/files/upload/routing.ts index 31f571cbae..f6a5e1d7a7 100644 --- a/packages/api/src/files/upload/routing.ts +++ b/packages/api/src/files/upload/routing.ts @@ -1,13 +1,21 @@ import { + Providers, + EModelEndpoint, EToolResources, mergeFileConfig, + isExplicitMimeConfig, + isOpenAILikeProvider, + isKnownProviderIdentifier, isAssistantsEndpoint, isResponsesApiUpload, getEndpointFileConfig, resolveUploadLLMDeliveryPath, isSpeechProviderConfigured, + getCustomEndpointProvider, } from 'librechat-data-provider'; import type { ServerRequest } from '~/types'; +import { UnsupportedProviderAudioError } from './errors'; +import { getAudioFormat } from '~/files/encode/utils'; /** The subset of an agent record this module reads. */ export interface UploadAgent { @@ -27,6 +35,7 @@ export interface UploadMetadata { /** The uploaded file fields routing depends on. */ export interface UploadFile { mimetype: string; + originalname?: string; } export type GetUploadAgent = (params: { id: string }) => Promise; @@ -112,14 +121,35 @@ export async function resolveEffectiveToolResource({ const fileConfig = mergeFileConfig(req.config?.fileConfig); const endpoint = await resolveUploadEndpoint({ req, metadata, getAgent }); const endpointConfig = getEndpointFileConfig({ fileConfig, endpoint }); + const endpointProvider = getCustomEndpointProvider(req.config?.endpoints?.custom, endpoint); + const file = req.file as UploadFile; const path = resolveUploadLLMDeliveryPath({ toolResource: metadata.tool_resource, - mimeType: (req.file as UploadFile).mimetype, + mimeType: file.mimetype, endpointConfig, fileConfig, endpoint, + endpointProvider, useResponsesApi: isResponsesApiUpload(metadata.useResponsesApi), sttConfigured: isSpeechProviderConfigured(req.config?.speech?.stt), }); + const provider = + endpointProvider ?? (isKnownProviderIdentifier(endpoint) ? endpoint : EModelEndpoint.openAI); + const usesInputAudio = + provider?.toLowerCase() === Providers.OPENROUTER || + (isOpenAILikeProvider(provider) && + isExplicitMimeConfig(endpointConfig.supportedMimeTypes) && + fileConfig.checkType?.(file.mimetype, endpointConfig.supportedMimeTypes)); + if ( + path === 'provider' && + endpoint != null && + endpoint !== EModelEndpoint.agents && + !isAssistantsEndpoint(endpoint) && + file.mimetype.startsWith('audio/') && + usesInputAudio && + !getAudioFormat(file.mimetype, file.originalname ?? '') + ) { + throw new UnsupportedProviderAudioError(); + } return path === 'text' ? EToolResources.context : undefined; } diff --git a/packages/api/src/types/files.ts b/packages/api/src/types/files.ts index b49a1a1f90..4d606f2482 100644 --- a/packages/api/src/types/files.ts +++ b/packages/api/src/types/files.ts @@ -39,7 +39,7 @@ export interface GoogleVideoBlock { data: string; } -/** OpenRouter video block format */ +/** OpenAI-compatible video block format (OpenRouter, vLLM, configured custom endpoints) */ export interface OpenRouterVideoBlock { type: 'video_url'; video_url: { @@ -136,7 +136,7 @@ export interface GoogleAudioBlock { data: string; } -/** OpenRouter audio block format */ +/** OpenAI-compatible audio block format (OpenRouter, vLLM, configured custom endpoints) */ export interface OpenRouterAudioBlock { type: 'input_audio'; input_audio: { diff --git a/packages/api/src/utils/files.spec.ts b/packages/api/src/utils/files.spec.ts index f8910b37c4..59eb11decc 100644 --- a/packages/api/src/utils/files.spec.ts +++ b/packages/api/src/utils/files.spec.ts @@ -4,6 +4,7 @@ import { flattenArtifactPath, resolveUploadErrorMessage, } from './files'; +import { UnsupportedProviderAudioError } from '~/files/upload/errors'; jest.mock('node:crypto', () => { const actualModule = jest.requireActual('node:crypto'); @@ -503,6 +504,14 @@ describe('flattenArtifactPath', () => { }); describe('resolveUploadErrorMessage', () => { + it.each([false, true])( + 'preserves the localized audio preflight error (redaction=%s)', + (redact) => { + expect( + resolveUploadErrorMessage(new UnsupportedProviderAudioError(), undefined, redact), + ).toBe('com_error_files_provider_audio_format'); + }, + ); test('returns default message for null error', () => { expect(resolveUploadErrorMessage(null)).toBe('Error processing file'); }); diff --git a/packages/api/src/utils/files.ts b/packages/api/src/utils/files.ts index b313dc9b6f..5be59b27aa 100644 --- a/packages/api/src/utils/files.ts +++ b/packages/api/src/utils/files.ts @@ -2,6 +2,7 @@ import path from 'path'; import crypto from 'node:crypto'; import { createReadStream } from 'fs'; import { readFile, stat } from 'fs/promises'; +import { UnsupportedProviderAudioError } from '~/files/upload/errors'; const USER_FACING_UPLOAD_ERRORS = [ ['Invalid file format', 'Invalid file format'], @@ -98,6 +99,9 @@ export function resolveUploadErrorMessage( defaultMessage = 'Error processing file', redactDetails = false, ): string { + if (error instanceof UnsupportedProviderAudioError) { + return error.message; + } const errorMessage = error?.message; if (!errorMessage) { return defaultMessage; diff --git a/packages/data-provider/src/file-config.spec.ts b/packages/data-provider/src/file-config.spec.ts index 1ee9e035cc..164e75fb75 100644 --- a/packages/data-provider/src/file-config.spec.ts +++ b/packages/data-provider/src/file-config.spec.ts @@ -9,6 +9,7 @@ import { bedrockDocumentMimeTypes, isAnthropicDocumentType, isPermissiveMimeConfig, + isExplicitMimeConfig, convertStringsToRegex, setFileConfigRegexCompiler, documentParserMimeTypes, @@ -1399,6 +1400,35 @@ describe('getEndpointFileConfig', () => { }); }); +describe('isExplicitMimeConfig', () => { + it('is false for undefined or empty lists', () => { + expect(isExplicitMimeConfig(undefined)).toBe(false); + expect(isExplicitMimeConfig([])).toBe(false); + }); + + it('is false for the built-in default list (inherited, not configured)', () => { + expect(isExplicitMimeConfig(supportedMimeTypes)).toBe(false); + const endpointConfig = getEndpointFileConfig({ + fileConfig: mergeFileConfig({ endpoints: { Other: { fileLimit: 1 } } }), + endpoint: 'MyGateway', + endpointType: 'custom', + }); + expect(isExplicitMimeConfig(endpointConfig.supportedMimeTypes)).toBe(false); + }); + + it('is true for an admin-configured list, permissive or not', () => { + const endpointConfig = getEndpointFileConfig({ + fileConfig: mergeFileConfig({ + endpoints: { MyGateway: { supportedMimeTypes: ['image/.*', 'video/.*'] } }, + }), + endpoint: 'MyGateway', + endpointType: 'custom', + }); + expect(isExplicitMimeConfig(endpointConfig.supportedMimeTypes)).toBe(true); + expect(isExplicitMimeConfig([/.*/])).toBe(true); + }); +}); + describe('isPermissiveMimeConfig', () => { it('returns true for wildcard .* pattern', () => { expect(isPermissiveMimeConfig([/.*/])).toBe(true); diff --git a/packages/data-provider/src/file-config.ts b/packages/data-provider/src/file-config.ts index b3f55413b9..0890dd24ae 100644 --- a/packages/data-provider/src/file-config.ts +++ b/packages/data-provider/src/file-config.ts @@ -674,6 +674,18 @@ export const isPermissiveMimeConfig = (types?: RegexLike[]): boolean => { return types.some((regex) => regex.test('x-librechat/x-probe')); }; +/** + * Detects whether an endpoint's `supportedMimeTypes` were set by the admin rather than inherited + * from the built-in default list. Inheritance is signaled by referential identity with + * `supportedMimeTypes`, which `mergeWithDefault` preserves for unconfigured endpoints. + */ +export const isExplicitMimeConfig = (types?: RegexLike[]): types is RegexLike[] => { + if (!types || types.length === 0) { + return false; + } + return types !== supportedMimeTypes; +}; + /** The kind of content a provider upload path can actually send to the model. */ export type MimeUploadCategory = 'image' | 'document' | 'audio' | 'video'; diff --git a/packages/data-provider/src/resolve-llm-delivery-path.spec.ts b/packages/data-provider/src/resolve-llm-delivery-path.spec.ts index 520e60a63f..b62c3a26f9 100644 --- a/packages/data-provider/src/resolve-llm-delivery-path.spec.ts +++ b/packages/data-provider/src/resolve-llm-delivery-path.spec.ts @@ -1,11 +1,15 @@ import type { TDefaultLLMDeliveryPathConfig } from './file-config'; +import type { TEndpoint } from './config'; import { isNativelyReadableText, canToolResourceConsume, resolveUploadDestination, + getCustomEndpointProvider, resolveDefaultLLMDeliveryPath, + resolveUploadLLMDeliveryPath, SYSTEM_LLM_DELIVERY_DEFAULTS, } from './resolve-llm-delivery-path'; +import { mergeFileConfig, supportedMimeTypes, getEndpointFileConfig } from './file-config'; describe('resolveDefaultLLMDeliveryPath', () => { it('should return system default for images when no config provided', () => { @@ -300,6 +304,97 @@ describe('resolveDefaultLLMDeliveryPath', () => { ); }); + describe('media a custom endpoint opted into', () => { + /* The encoders emit OpenAI-format media parts for an OpenAI-compatible endpoint only + * when the admin listed the type in its `supportedMimeTypes`, so the route has to + * agree: an explicit match is provider-capable, the inherited default list is not. */ + const explicit = [/^image\/.*$/, /^application\/pdf$/, /^video\/.*$/, /^audio\/wav$/]; + const resolve = (mimeType: string, endpoint: string, types?: RegExp[]) => + resolveDefaultLLMDeliveryPath( + mimeType, + undefined, + undefined, + endpoint, + undefined, + true, + types, + ); + + it('keeps an explicitly allowed type on the provider path for a custom endpoint', () => { + expect(resolve('video/mp4', 'MyGateway', explicit)).toBe('provider'); + expect(resolve('audio/wav', 'MyGateway', explicit)).toBe('provider'); + }); + + it('still downgrades a media type the allowlist does not name', () => { + expect(resolve('audio/mpeg', 'MyGateway', explicit)).toBe('text'); + }); + + it('does not read the inherited default list as an opt-in', () => { + expect(resolve('video/mp4', 'MyGateway', supportedMimeTypes)).toBe('none'); + expect(resolve('video/mp4', 'MyGateway', [])).toBe('none'); + }); + + it('does not opt in a built-in endpoint, which the client offers no media for', () => { + /* Anthropic and Bedrock encoders have no media branch at all, and OpenAI/Azure are + * left out because the picker and drag-drop only open media for custom endpoints: + * a route the client cannot send to is a capability with no entry point. */ + expect(resolve('video/mp4', 'openAI', explicit)).toBe('none'); + expect(resolve('video/mp4', 'azureOpenAI', explicit)).toBe('none'); + expect(resolve('video/mp4', 'anthropic', explicit)).toBe('none'); + expect(resolve('video/mp4', 'bedrock', explicit)).toBe('none'); + }); + + it('keeps a custom endpoint that runs as Anthropic on its previous route', () => { + /* A custom endpoint may declare `provider: anthropic`, and the encoders emit + * OpenAI-format parts only, so the opt-in would deliver nothing there. Audio keeps + * its transcription route and video stays off the model path. */ + const endpointConfig = { supportedMimeTypes: explicit }; + const anthropic = { mimeType: 'video/mp4', endpointConfig, endpoint: 'MyClaude' }; + expect(resolveUploadLLMDeliveryPath({ ...anthropic, endpointProvider: 'anthropic' })).toBe( + 'none', + ); + expect( + resolveUploadLLMDeliveryPath({ + ...anthropic, + mimeType: 'audio/wav', + endpointProvider: 'anthropic', + sttConfigured: true, + }), + ).toBe('text'); + expect(resolveUploadLLMDeliveryPath({ ...anthropic, endpointProvider: 'openAI' })).toBe( + 'provider', + ); + expect(resolveUploadLLMDeliveryPath(anthropic)).toBe('provider'); + }); + + it('reaches the upload resolver through the merged endpoint config', () => { + /* The real merge, so the identity check that separates a configured list from the + * inherited default is exercised the way the upload route exercises it. */ + const fileConfig = mergeFileConfig({ + endpoints: { MyGateway: { supportedMimeTypes: ['image/.*', 'video/.*'] } }, + }); + const configured = getEndpointFileConfig({ fileConfig, endpoint: 'MyGateway' }); + const inherited = getEndpointFileConfig({ fileConfig, endpoint: 'OtherGateway' }); + + expect( + resolveUploadLLMDeliveryPath({ + mimeType: 'video/mp4', + endpointConfig: configured, + fileConfig, + endpoint: 'MyGateway', + }), + ).toBe('provider'); + expect( + resolveUploadLLMDeliveryPath({ + mimeType: 'video/mp4', + endpointConfig: inherited, + fileConfig, + endpoint: 'OtherGateway', + }), + ).toBe('none'); + }); + }); + it('leaves media alone when no endpoint is resolved at all', () => { /* An ephemeral agent reports no usable endpoint, which is not the same as naming one * we cannot identify. */ @@ -545,6 +640,27 @@ describe('resolveUploadDestination', () => { }); }); +describe('getCustomEndpointProvider', () => { + const custom = [ + { name: 'My Claude', provider: 'anthropic' }, + { name: 'Ollama', provider: 'anthropic' }, + { name: 'MyGateway' }, + ] as Array>>; + + it('returns the declared dialect for a custom endpoint, matching the normalized name', () => { + expect(getCustomEndpointProvider(custom, 'My Claude')).toBe('anthropic'); + /* The same normalization the file config lookup applies to endpoint names. */ + expect(getCustomEndpointProvider(custom, 'ollama')).toBe('anthropic'); + }); + + it('returns nothing for an endpoint without a dialect, an unknown one, or no config', () => { + expect(getCustomEndpointProvider(custom, 'MyGateway')).toBeUndefined(); + expect(getCustomEndpointProvider(custom, 'Other')).toBeUndefined(); + expect(getCustomEndpointProvider(undefined, 'My Claude')).toBeUndefined(); + expect(getCustomEndpointProvider(custom, undefined)).toBeUndefined(); + }); +}); + describe('isNativelyReadableText', () => { it('admits the application types whose payload is text', () => { /* Kept in step with the textual set in the content-protection code. Missing one sends diff --git a/packages/data-provider/src/resolve-llm-delivery-path.ts b/packages/data-provider/src/resolve-llm-delivery-path.ts index bb6148efaf..bccf7f566c 100644 --- a/packages/data-provider/src/resolve-llm-delivery-path.ts +++ b/packages/data-provider/src/resolve-llm-delivery-path.ts @@ -1,13 +1,57 @@ import type { TDefaultLLMDeliveryPath, TDefaultLLMDeliveryPathConfig } from './file-config'; -import type { EndpointFileConfig, FileConfig } from './types/files'; +import type { EndpointFileConfig, FileConfig, RegexLike } from './types/files'; +import type { TEndpoint } from './config'; +import { + retrievalMimeTypes, + isExplicitMimeConfig, + isBedrockDocumentType, + codeInterpreterMimeTypes, + fileConfig as baseFileConfig, +} from './file-config'; import { EModelEndpoint, - isDocumentSupportedProvider, + isOpenAILikeProvider, isKnownProviderIdentifier, isMediaSupportedProvider, + isDocumentSupportedProvider, } from './schemas'; -import { retrievalMimeTypes, isBedrockDocumentType, codeInterpreterMimeTypes } from './file-config'; import { EToolResources } from './types/assistants'; +import { normalizeEndpointName } from './utils'; + +/** + * The native provider a custom endpoint declares, when it declares one. A custom endpoint + * speaks OpenAI's API unless its config names another dialect, and the upload route needs + * that answer for the same reason request initialization does: the media encoders emit + * OpenAI-format parts, so a custom endpoint running as Anthropic receives none. + */ +export function getCustomEndpointProvider( + customEndpoints: Array>> | undefined, + endpoint?: string | null, +): string | undefined { + if (!customEndpoints || !endpoint) { + return undefined; + } + const normalized = normalizeEndpointName(endpoint); + return customEndpoints.find((config) => normalizeEndpointName(config.name ?? '') === normalized) + ?.provider; +} + +/** A custom endpoint emits OpenAI-format media parts only for the types the admin listed + * in its `supportedMimeTypes`; the inherited default list is not an opt-in. A name that + * is not a known provider is a custom endpoint. Mirrors `isConfiguredProviderMediaType` + * on the encoder side, so the route and the encoder agree on which uploads the provider + * actually receives; the built-in endpoints are left out because the client offers no + * media for them. */ +const isConfiguredMediaEndpoint = ( + mimeType: string, + endpoint: string, + supportedMimeTypes?: RegexLike[], +): boolean => { + if (!isExplicitMimeConfig(supportedMimeTypes) || isKnownProviderIdentifier(endpoint)) { + return false; + } + return baseFileConfig.checkType(mimeType, supportedMimeTypes); +}; /** Audio and video reach the model only through the media encoders, which support a * narrower provider set than documents. Images use the broadly supported vision @@ -16,9 +60,13 @@ const isProviderCapable = ( mimeType: string, endpoint: string, useResponsesApi?: boolean, + supportedMimeTypes?: RegexLike[], ): boolean => { if (mimeType.startsWith('audio/') || mimeType.startsWith('video/')) { - return isMediaSupportedProvider(endpoint); + return ( + isMediaSupportedProvider(endpoint) || + isConfiguredMediaEndpoint(mimeType, endpoint, supportedMimeTypes) + ); } if (mimeType === 'application/pdf') { /* Azure is out of the document set because it needs the Responses API for native @@ -104,6 +152,7 @@ export function resolveDefaultLLMDeliveryPath( endpoint?: string, useResponsesApi?: boolean, sttConfigured?: boolean, + supportedMimeTypes?: RegexLike[], ): TDefaultLLMDeliveryPath { const wildcard = mimeType.split('/')[0] + '/*'; @@ -149,10 +198,11 @@ export function resolveDefaultLLMDeliveryPath( const namedEndpoint = endpoint != null && endpoint !== EModelEndpoint.agents; const providerKnown = namedEndpoint && isKnownProviderIdentifier(endpoint); /* Media is judged for any named endpoint, identified or not. The media encoders emit a - * payload only for the providers they name, so a custom endpoint gets nothing whatever - * it proxies to, and leaving it on the provider path delivers neither media nor text. - * Documents keep the narrower rule: an unidentified endpoint is usually OpenAI- or - * Anthropic-compatible, both of which do carry them. */ + * payload only for the providers they name, or for an OpenAI-compatible endpoint whose + * admin listed the type in its `supportedMimeTypes`; any other custom endpoint gets + * nothing whatever it proxies to, and leaving it on the provider path delivers neither + * media nor text. Documents keep the narrower rule: an unidentified endpoint is usually + * OpenAI- or Anthropic-compatible, both of which do carry them. */ const isMedia = mimeType.startsWith('audio/') || mimeType.startsWith('video/'); /* Audio's text path is transcription, so on a deployment with no speech provider it is * not recoverable at all. Routing it to text there sends the upload to a service that @@ -164,7 +214,7 @@ export function resolveDefaultLLMDeliveryPath( if ( systemDefault === 'provider' && canJudgeCapability && - !isProviderCapable(mimeType, endpoint as string, useResponsesApi) + !isProviderCapable(mimeType, endpoint as string, useResponsesApi, supportedMimeTypes) ) { /* Downgrading is only useful where text can actually be recovered. Video has no * extraction step: speech-to-text covers audio, and the default text matcher accepts @@ -206,6 +256,7 @@ export function resolveDefaultUploadLLMDeliveryPath({ endpointConfig, fileConfig, endpoint, + endpointProvider, useResponsesApi, sttConfigured, }: { @@ -213,12 +264,19 @@ export function resolveDefaultUploadLLMDeliveryPath({ endpointConfig?: EndpointFileConfig; fileConfig?: FileConfig; endpoint?: string; + /** The provider the endpoint runs as, when the caller knows it: a custom endpoint's + * declared dialect at upload time, the agent's resolved provider at turn time. */ + endpointProvider?: string | null; useResponsesApi?: boolean; sttConfigured?: boolean; }): TDefaultLLMDeliveryPath { if (endpointConfig?.legacyFileUploadUX === true) { return 'provider'; } + /* The media opt-in exists for OpenAI-format parts, so an endpoint known to run as + * something else — a custom endpoint declaring `provider: anthropic` — keeps the + * capability gate it had, where audio still reaches transcription. */ + const runsAsOpenAI = endpointProvider == null || isOpenAILikeProvider(endpointProvider); return resolveDefaultLLMDeliveryPath( mimeType, endpointConfig?.defaultLLMDeliveryPath, @@ -226,6 +284,7 @@ export function resolveDefaultUploadLLMDeliveryPath({ endpoint, useResponsesApi, sttConfigured, + runsAsOpenAI ? endpointConfig?.supportedMimeTypes : undefined, ); } @@ -237,6 +296,7 @@ export function resolveUploadLLMDeliveryPath({ fileConfig, endpoint, useResponsesApi, + endpointProvider, sttConfigured, }: { toolResource?: string | null; @@ -244,6 +304,7 @@ export function resolveUploadLLMDeliveryPath({ endpointConfig?: EndpointFileConfig; fileConfig?: FileConfig; endpoint?: string; + endpointProvider?: string | null; useResponsesApi?: boolean; sttConfigured?: boolean; }): TDefaultLLMDeliveryPath { @@ -258,6 +319,7 @@ export function resolveUploadLLMDeliveryPath({ endpointConfig, fileConfig, endpoint, + endpointProvider, useResponsesApi, sttConfigured, });