From 413906abc37233c937d2bfa2ab74feff5f1d9ece Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 31 Aug 2026 08:34:05 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=96=8D=EF=B8=8F=20fix:=20Honor=20Agent=20?= =?UTF-8?q?Image=20Detail=20Settings=20(#15407)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🖼️ fix: Apply an Agent's Configured Image Detail The Agent Builder exposes an Image Detail slider and saves it to `model_parameters.imageDetail`, but nothing ever read it back: `encodeAndFormat` resolves the level from `req.body.imageDetail`, which the agents route never populates, so every image was encoded as `detail: "auto"`. Providers that reject `auto` — MiniMax's OpenAI-compatible API among them — fail every image upload through an agent, and the setting that would fix it has no effect. The root cause is a classification error rather than a missing argument. `extractLibreChatParams` splits LibreChat's own settings out of an agent's model parameters, and its key set is derived from the `librechat` group in `parameterSettings`. `imageDetail` sat in `baseDefinitions` instead, so it was neither extracted for LibreChat's use nor stripped from the options handed to the provider. It is now grouped with `resendFiles` and the other LibreChat-level parameters, which gives it the same path those already take: extracted during agent initialization, carried on the initialized agent, passed to the client, and read by `encodeAndFormat`. `AgentClient.getSaveOptions` already expected `options.imageDetail` — nothing had ever populated it. The request body still wins where it is set, so the classic endpoints keep resolving detail from conversation settings exactly as before. Fixes #15359 * 🔤 fix: Type Image Detail as Its Enum, Not a Bare String `TConversation.imageDetail` is the `ImageDetail` enum, so the extractor's return type and the initialized agent's field are now declared as that enum rather than widened to `string`, and the tests pass enum members instead of raw literals. Caught by CI's type check, which I had run before adding the tests and not after — jest compiles the specs through babel, so the suite stayed green over a type error. --- .../agents/client.imageDetail.spec.js | 55 +++++++++++++++++++ api/server/controllers/agents/client.js | 1 + .../services/Endpoints/agents/initialize.js | 1 + api/server/services/Files/images/encode.js | 4 +- .../services/Files/images/encode.spec.js | 53 ++++++++++++++++++ packages/api/src/agents/initialize.ts | 9 ++- packages/api/src/utils/llm.test.ts | 18 ++++++ packages/api/src/utils/llm.ts | 5 +- .../data-provider/src/parameterSettings.ts | 42 +++++++------- 9 files changed, 164 insertions(+), 24 deletions(-) create mode 100644 api/server/controllers/agents/client.imageDetail.spec.js diff --git a/api/server/controllers/agents/client.imageDetail.spec.js b/api/server/controllers/agents/client.imageDetail.spec.js new file mode 100644 index 0000000000..db49a84bc0 --- /dev/null +++ b/api/server/controllers/agents/client.imageDetail.spec.js @@ -0,0 +1,55 @@ +const mockEncodeAndFormat = jest.fn().mockResolvedValue({ files: [], image_urls: [] }); + +jest.mock('~/server/services/Files/images/encode', () => ({ + encodeAndFormat: (...args) => mockEncodeAndFormat(...args), +})); + +const AgentClient = require('./client'); + +describe('AgentClient.addImageURLs - image detail', () => { + const buildClient = (options) => { + const client = Object.create(AgentClient.prototype); + client.options = { + req: { body: {} }, + endpoint: 'agents', + agent: { id: 'agent_1', provider: 'openai' }, + ...options, + }; + return client; + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it("forwards the agent's configured detail to the encoder", async () => { + const client = buildClient({ imageDetail: 'high' }); + + await client.addImageURLs({}, [{ file_id: 'f_1' }]); + + expect(mockEncodeAndFormat).toHaveBeenCalledTimes(1); + expect(mockEncodeAndFormat.mock.calls[0][2]).toEqual( + expect.objectContaining({ imageDetail: 'high' }), + ); + }); + + it('leaves the detail undefined when the agent configures none, so the encoder falls back', async () => { + const client = buildClient({}); + + await client.addImageURLs({}, [{ file_id: 'f_1' }]); + + expect(mockEncodeAndFormat.mock.calls[0][2].imageDetail).toBeUndefined(); + }); + + it('still passes the provider and endpoint alongside the detail', async () => { + const client = buildClient({ imageDetail: 'low' }); + + await client.addImageURLs({}, [{ file_id: 'f_1' }]); + + expect(mockEncodeAndFormat.mock.calls[0][2]).toEqual({ + provider: 'openai', + endpoint: 'agents', + imageDetail: 'low', + }); + }); +}); diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index 3b12941947..9a90f46a1a 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -1675,6 +1675,7 @@ class AgentClient extends BaseClient { { provider: this.options.agent.provider, endpoint: this.options.endpoint, + imageDetail: this.options.imageDetail, }, VisionModes.agents, ); diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js index 42c0f82ccb..53732e2e5d 100644 --- a/api/server/services/Endpoints/agents/initialize.js +++ b/api/server/services/Endpoints/agents/initialize.js @@ -1567,6 +1567,7 @@ const initializeClient = async ({ agentContextAttachmentsByAgentId, endpointType: endpointOption.endpointType, resendFiles: primaryConfig.resendFiles ?? true, + imageDetail: primaryConfig.imageDetail, maxContextTokens: primaryConfig.maxContextTokens, endpoint: isEphemeralAgentId(primaryConfig.id) ? primaryConfig.endpoint : EModelEndpoint.agents, subagentAggregatorsByToolCallId, diff --git a/api/server/services/Files/images/encode.js b/api/server/services/Files/images/encode.js index 25110566ee..dc744542f1 100644 --- a/api/server/services/Files/images/encode.js +++ b/api/server/services/Files/images/encode.js @@ -94,6 +94,8 @@ const blobStorageSources = new Set([ * @param {object} params - Object containing provider/endpoint information * @param {Providers | EModelEndpoint | string} [params.provider] - The provider for the image * @param {string} [params.endpoint] - Optional: The endpoint for the image + * @param {string} [params.imageDetail] - Optional: Detail level resolved by the caller, used + * where the request body carries no conversation-level setting (the agents route). * @param {string} [mode] - Optional: The endpoint mode for the image. * @returns {Promise<{ files: MongoFile[]; image_urls: MessageContentImageUrl[] }>} - A promise that resolves to the result object containing the encoded images and file details. */ @@ -159,7 +161,7 @@ async function encodeAndFormat(req, files, params, mode) { promises.push(preparePayload(req, file)); } - const detail = req.body.imageDetail ?? ImageDetail.auto; + const detail = params.imageDetail ?? req.body.imageDetail ?? ImageDetail.auto; /** @type {Array<[MongoFile, string]>} */ const formattedImages = await Promise.all(promises); diff --git a/api/server/services/Files/images/encode.spec.js b/api/server/services/Files/images/encode.spec.js index 700f930d32..262a55be45 100644 --- a/api/server/services/Files/images/encode.spec.js +++ b/api/server/services/Files/images/encode.spec.js @@ -109,3 +109,56 @@ describe('encodeAndFormat - request memory guard', () => { expect(result.image_urls[0].image_url.url).toBe(`data:image/png;base64,${localBase64}`); }); }); + +describe('encodeAndFormat - image detail', () => { + const imageFile = () => ({ + source: FileSources.s3, + height: 10, + width: 10, + type: 'image/png', + file_id: 'f-detail', + filepath: 'bucket/a.png', + filename: 'a.png', + bytes: 128, + }); + + beforeEach(() => { + mockGetDownloadStream.mockResolvedValue(Readable.from([Buffer.from('image-bytes')])); + }); + + it('falls back to auto when no detail is configured anywhere', async () => { + const result = await encodeAndFormat(makeReq(), [imageFile()], { endpoint: 'openai' }); + + expect(result.image_urls[0].image_url.detail).toBe('auto'); + }); + + it('uses the conversation-level detail from the request body', async () => { + const req = makeReq(); + req.body.imageDetail = 'low'; + + const result = await encodeAndFormat(req, [imageFile()], { endpoint: 'openai' }); + + expect(result.image_urls[0].image_url.detail).toBe('low'); + }); + + it('uses the detail resolved by the caller on routes with no body setting', async () => { + const result = await encodeAndFormat(makeReq(), [imageFile()], { + endpoint: 'agents', + imageDetail: 'high', + }); + + expect(result.image_urls[0].image_url.detail).toBe('high'); + }); + + it('prefers the caller-resolved detail over the request body', async () => { + const req = makeReq(); + req.body.imageDetail = 'low'; + + const result = await encodeAndFormat(req, [imageFile()], { + endpoint: 'agents', + imageDetail: 'high', + }); + + expect(result.image_urls[0].image_url.detail).toBe('high'); + }); +}); diff --git a/packages/api/src/agents/initialize.ts b/packages/api/src/agents/initialize.ts index 0dba003e2b..097aef6f32 100644 --- a/packages/api/src/agents/initialize.ts +++ b/packages/api/src/agents/initialize.ts @@ -20,10 +20,11 @@ import type { AgentToolOptions, TEndpointOption, ReasoningResponseKey, + StatefulCodeEnvironment, + ImageDetail, TFile, Agent, TUser, - StatefulCodeEnvironment, } from 'librechat-data-provider'; import type { GenericTool, LCToolRegistry, ToolMap, LCTool } from '@librechat/agents'; import type { IMongoFile, FileOwnerScope } from '@librechat/data-schemas'; @@ -373,6 +374,9 @@ export type InitializedAgent = Agent & { baseContextTokens?: number; useLegacyContent: boolean; resendFiles: boolean; + /** Detail level LibreChat encodes image content blocks with, from the agent's + * model parameters. Absent when the agent does not configure one. */ + imageDetail?: ImageDetail; tool_resources?: AgentToolResources; userMCPAuthMap?: Record>; /** Tool map for ToolNode to use when executing tools (required for PTC) */ @@ -950,7 +954,7 @@ export async function initializeAgent( ), ); - const { resendFiles, maxContextTokens, modelOptions } = extractLibreChatParams( + const { resendFiles, maxContextTokens, imageDetail, modelOptions } = extractLibreChatParams( _modelOptions as Record, ); @@ -1776,6 +1780,7 @@ export async function initializeAgent( const initializedAgent: InitializedAgent = { ...agent, resendFiles, + imageDetail, toolRegistry, mcpAvailableTools, requestScopedConnections, diff --git a/packages/api/src/utils/llm.test.ts b/packages/api/src/utils/llm.test.ts index c4d270f400..3c652d4cce 100644 --- a/packages/api/src/utils/llm.test.ts +++ b/packages/api/src/utils/llm.test.ts @@ -1,3 +1,4 @@ +import { ImageDetail } from 'librechat-data-provider'; import { extractLibreChatParams } from './llm'; describe('extractLibreChatParams', () => { @@ -30,6 +31,7 @@ describe('extractLibreChatParams', () => { maxContextTokens: 4096, fileTokenLimit: 50000, modelLabel: 'GPT-4', + imageDetail: ImageDetail.high, model: 'gpt-4', temperature: 0.7, max_tokens: 1000, @@ -42,6 +44,7 @@ describe('extractLibreChatParams', () => { expect(result.maxContextTokens).toBe(4096); expect(result.fileTokenLimit).toBe(50000); expect(result.modelLabel).toBe('GPT-4'); + expect(result.imageDetail).toBe(ImageDetail.high); expect(result.modelOptions).toEqual({ model: 'gpt-4', temperature: 0.7, @@ -49,6 +52,21 @@ describe('extractLibreChatParams', () => { }); }); + it('should keep imageDetail out of the model options sent to the provider', () => { + const result = extractLibreChatParams({ model: 'gpt-4o', imageDetail: ImageDetail.low }); + + expect(result.imageDetail).toBe(ImageDetail.low); + expect(result.modelOptions).toEqual({ model: 'gpt-4o' }); + expect('imageDetail' in result.modelOptions).toBe(false); + }); + + it('should leave imageDetail undefined when the agent configures none', () => { + const result = extractLibreChatParams({ model: 'gpt-4o' }); + + expect(result.imageDetail).toBeUndefined(); + expect(result.modelOptions).toEqual({ model: 'gpt-4o' }); + }); + it('should handle null values for LibreChat params', () => { const options = { resendFiles: true, diff --git a/packages/api/src/utils/llm.ts b/packages/api/src/utils/llm.ts index d1e1d9a43b..bd2eb73ba2 100644 --- a/packages/api/src/utils/llm.ts +++ b/packages/api/src/utils/llm.ts @@ -1,5 +1,5 @@ import { librechat } from 'librechat-data-provider'; -import type { DynamicSettingProps } from 'librechat-data-provider'; +import type { DynamicSettingProps, ImageDetail } from 'librechat-data-provider'; type LibreChatKeys = keyof typeof librechat; @@ -10,6 +10,7 @@ type LibreChatParams = { maxContextTokens?: number; fileTokenLimit?: number; modelLabel?: string | null; + imageDetail?: ImageDetail; }; /** @@ -35,6 +36,7 @@ export function extractLibreChatParams( const maxContextTokens = (delete modelOptions.maxContextTokens, options.maxContextTokens); const fileTokenLimit = (delete modelOptions.fileTokenLimit, options.fileTokenLimit); const modelLabel = (delete modelOptions.modelLabel, options.modelLabel); + const imageDetail = (delete modelOptions.imageDetail, options.imageDetail); return { modelOptions: modelOptions as Omit< @@ -46,5 +48,6 @@ export function extractLibreChatParams( promptPrefix, resendFiles, modelLabel, + imageDetail, }; } diff --git a/packages/data-provider/src/parameterSettings.ts b/packages/data-provider/src/parameterSettings.ts index 1a6a2cad66..e6898c1074 100644 --- a/packages/data-provider/src/parameterSettings.ts +++ b/packages/data-provider/src/parameterSettings.ts @@ -71,24 +71,6 @@ const baseDefinitions: Record = { minTags: 0, maxTags: 4, }, - imageDetail: { - key: 'imageDetail', - label: 'com_endpoint_plug_image_detail', - labelCode: true, - description: 'com_endpoint_openai_detail', - descriptionCode: true, - type: 'enum', - default: ImageDetail.auto, - component: 'slider', - options: [ImageDetail.low, ImageDetail.auto, ImageDetail.high], - enumMappings: { - [ImageDetail.low]: 'com_ui_low', - [ImageDetail.auto]: 'com_ui_auto', - [ImageDetail.high]: 'com_ui_high', - }, - optionType: 'conversation', - columnSpan: 2, - }, }; const createDefinition = ( @@ -147,6 +129,26 @@ export const librechat = { placeholderCode: true, optionType: 'model', } as const, + /** Controls how LibreChat encodes image content blocks, not a provider request + * parameter — so it belongs to this group and is stripped from model options. */ + imageDetail: { + key: 'imageDetail', + label: 'com_endpoint_plug_image_detail', + labelCode: true, + description: 'com_endpoint_openai_detail', + descriptionCode: true, + type: 'enum', + default: ImageDetail.auto, + component: 'slider', + options: [ImageDetail.low, ImageDetail.auto, ImageDetail.high], + enumMappings: { + [ImageDetail.low]: 'com_ui_low', + [ImageDetail.auto]: 'com_ui_auto', + [ImageDetail.high]: 'com_ui_high', + }, + optionType: 'conversation', + columnSpan: 2, + } as SettingDefinition, fileTokenLimit: { key: 'fileTokenLimit', label: 'com_ui_file_token_limit', @@ -887,7 +889,7 @@ const openAI: SettingsConfiguration = [ openAIParams.presence_penalty, baseDefinitions.stop, librechat.resendFiles, - baseDefinitions.imageDetail, + librechat.imageDetail, openAIParams.web_search, openAIParams.reasoning_effort, openAIParams.useResponsesApi, @@ -920,7 +922,7 @@ const openAICol2: SettingsConfiguration = [ openAIParams.presence_penalty, baseDefinitions.stop, librechat.resendFiles, - baseDefinitions.imageDetail, + librechat.imageDetail, openAIParams.reasoning_effort, openAIParams.reasoning_summary, openAIParams.reasoning_mode,