🖍️ fix: Honor Agent Image Detail Settings (#15407)

* 🖼️ fix: Apply an Agent's Configured Image Detail

The Agent Builder exposes an Image Detail slider and saves it to
`model_parameters.imageDetail`, but nothing ever read it back: `encodeAndFormat`
resolves the level from `req.body.imageDetail`, which the agents route never
populates, so every image was encoded as `detail: "auto"`. Providers that reject
`auto` — MiniMax's OpenAI-compatible API among them — fail every image upload
through an agent, and the setting that would fix it has no effect.

The root cause is a classification error rather than a missing argument.
`extractLibreChatParams` splits LibreChat's own settings out of an agent's model
parameters, and its key set is derived from the `librechat` group in
`parameterSettings`. `imageDetail` sat in `baseDefinitions` instead, so it was
neither extracted for LibreChat's use nor stripped from the options handed to the
provider.

It is now grouped with `resendFiles` and the other LibreChat-level parameters,
which gives it the same path those already take: extracted during agent
initialization, carried on the initialized agent, passed to the client, and read
by `encodeAndFormat`. `AgentClient.getSaveOptions` already expected
`options.imageDetail` — nothing had ever populated it.

The request body still wins where it is set, so the classic endpoints keep
resolving detail from conversation settings exactly as before.

Fixes #15359

* 🔤 fix: Type Image Detail as Its Enum, Not a Bare String

`TConversation.imageDetail` is the `ImageDetail` enum, so the extractor's return
type and the initialized agent's field are now declared as that enum rather than
widened to `string`, and the tests pass enum members instead of raw literals.

Caught by CI's type check, which I had run before adding the tests and not after —
jest compiles the specs through babel, so the suite stayed green over a type error.

api/server/controllers/agents/client.imageDetail.spec.js

@@ -0,0 +1,55 @@
+const mockEncodeAndFormat = jest.fn().mockResolvedValue({ files: [], image_urls: [] });
+
+jest.mock('~/server/services/Files/images/encode', () => ({
+  encodeAndFormat: (...args) => mockEncodeAndFormat(...args),
+}));
+
+const AgentClient = require('./client');
+
+describe('AgentClient.addImageURLs - image detail', () => {
+  const buildClient = (options) => {
+    const client = Object.create(AgentClient.prototype);
+    client.options = {
+      req: { body: {} },
+      endpoint: 'agents',
+      agent: { id: 'agent_1', provider: 'openai' },
+      ...options,
+    };
+    return client;
+  };
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it("forwards the agent's configured detail to the encoder", async () => {
+    const client = buildClient({ imageDetail: 'high' });
+
+    await client.addImageURLs({}, [{ file_id: 'f_1' }]);
+
+    expect(mockEncodeAndFormat).toHaveBeenCalledTimes(1);
+    expect(mockEncodeAndFormat.mock.calls[0][2]).toEqual(
+      expect.objectContaining({ imageDetail: 'high' }),
+    );
+  });
+
+  it('leaves the detail undefined when the agent configures none, so the encoder falls back', async () => {
+    const client = buildClient({});
+
+    await client.addImageURLs({}, [{ file_id: 'f_1' }]);
+
+    expect(mockEncodeAndFormat.mock.calls[0][2].imageDetail).toBeUndefined();
+  });
+
+  it('still passes the provider and endpoint alongside the detail', async () => {
+    const client = buildClient({ imageDetail: 'low' });
+
+    await client.addImageURLs({}, [{ file_id: 'f_1' }]);
+
+    expect(mockEncodeAndFormat.mock.calls[0][2]).toEqual({
+      provider: 'openai',
+      endpoint: 'agents',
+      imageDetail: 'low',
+    });
+  });
+});

api/server/controllers/agents/client.js

@@ -1675,6 +1675,7 @@ class AgentClient extends BaseClient {
       {
         provider: this.options.agent.provider,
         endpoint: this.options.endpoint,
+        imageDetail: this.options.imageDetail,
       },
       VisionModes.agents,
     );

api/server/services/Endpoints/agents/initialize.js

@@ -1567,6 +1567,7 @@ const initializeClient = async ({
     agentContextAttachmentsByAgentId,
     endpointType: endpointOption.endpointType,
     resendFiles: primaryConfig.resendFiles ?? true,
+    imageDetail: primaryConfig.imageDetail,
     maxContextTokens: primaryConfig.maxContextTokens,
     endpoint: isEphemeralAgentId(primaryConfig.id) ? primaryConfig.endpoint : EModelEndpoint.agents,
     subagentAggregatorsByToolCallId,

api/server/services/Files/images/encode.js

@@ -94,6 +94,8 @@ const blobStorageSources = new Set([
  * @param {object} params - Object containing provider/endpoint information
  * @param {Providers | EModelEndpoint | string} [params.provider] - The provider for the image
  * @param {string} [params.endpoint] - Optional: The endpoint for the image
+ * @param {string} [params.imageDetail] - Optional: Detail level resolved by the caller, used
+ *   where the request body carries no conversation-level setting (the agents route).
  * @param {string} [mode] - Optional: The endpoint mode for the image.
  * @returns {Promise<{ files: MongoFile[]; image_urls: MessageContentImageUrl[] }>} - A promise that resolves to the result object containing the encoded images and file details.
  */
@@ -159,7 +161,7 @@ async function encodeAndFormat(req, files, params, mode) {
     promises.push(preparePayload(req, file));
   }
 
-  const detail = req.body.imageDetail ?? ImageDetail.auto;
+  const detail = params.imageDetail ?? req.body.imageDetail ?? ImageDetail.auto;
 
   /** @type {Array<[MongoFile, string]>} */
   const formattedImages = await Promise.all(promises);

api/server/services/Files/images/encode.spec.js

@@ -109,3 +109,56 @@ describe('encodeAndFormat - request memory guard', () => {
     expect(result.image_urls[0].image_url.url).toBe(`data:image/png;base64,${localBase64}`);
   });
 });
+
+describe('encodeAndFormat - image detail', () => {
+  const imageFile = () => ({
+    source: FileSources.s3,
+    height: 10,
+    width: 10,
+    type: 'image/png',
+    file_id: 'f-detail',
+    filepath: 'bucket/a.png',
+    filename: 'a.png',
+    bytes: 128,
+  });
+
+  beforeEach(() => {
+    mockGetDownloadStream.mockResolvedValue(Readable.from([Buffer.from('image-bytes')]));
+  });
+
+  it('falls back to auto when no detail is configured anywhere', async () => {
+    const result = await encodeAndFormat(makeReq(), [imageFile()], { endpoint: 'openai' });
+
+    expect(result.image_urls[0].image_url.detail).toBe('auto');
+  });
+
+  it('uses the conversation-level detail from the request body', async () => {
+    const req = makeReq();
+    req.body.imageDetail = 'low';
+
+    const result = await encodeAndFormat(req, [imageFile()], { endpoint: 'openai' });
+
+    expect(result.image_urls[0].image_url.detail).toBe('low');
+  });
+
+  it('uses the detail resolved by the caller on routes with no body setting', async () => {
+    const result = await encodeAndFormat(makeReq(), [imageFile()], {
+      endpoint: 'agents',
+      imageDetail: 'high',
+    });
+
+    expect(result.image_urls[0].image_url.detail).toBe('high');
+  });
+
+  it('prefers the caller-resolved detail over the request body', async () => {
+    const req = makeReq();
+    req.body.imageDetail = 'low';
+
+    const result = await encodeAndFormat(req, [imageFile()], {
+      endpoint: 'agents',
+      imageDetail: 'high',
+    });
+
+    expect(result.image_urls[0].image_url.detail).toBe('high');
+  });
+});

packages/api/src/agents/initialize.ts

@@ -20,10 +20,11 @@ import type {
   AgentToolOptions,
   TEndpointOption,
   ReasoningResponseKey,
+  StatefulCodeEnvironment,
+  ImageDetail,
   TFile,
   Agent,
   TUser,
-  StatefulCodeEnvironment,
 } from 'librechat-data-provider';
 import type { GenericTool, LCToolRegistry, ToolMap, LCTool } from '@librechat/agents';
 import type { IMongoFile, FileOwnerScope } from '@librechat/data-schemas';
@@ -373,6 +374,9 @@ export type InitializedAgent = Agent & {
   baseContextTokens?: number;
   useLegacyContent: boolean;
   resendFiles: boolean;
+  /** Detail level LibreChat encodes image content blocks with, from the agent's
+   * model parameters. Absent when the agent does not configure one. */
+  imageDetail?: ImageDetail;
   tool_resources?: AgentToolResources;
   userMCPAuthMap?: Record<string, Record<string, string>>;
   /** Tool map for ToolNode to use when executing tools (required for PTC) */
@@ -950,7 +954,7 @@ export async function initializeAgent(
     ),
   );
 
-  const { resendFiles, maxContextTokens, modelOptions } = extractLibreChatParams(
+  const { resendFiles, maxContextTokens, imageDetail, modelOptions } = extractLibreChatParams(
     _modelOptions as Record<string, unknown>,
   );
 
@@ -1776,6 +1780,7 @@ export async function initializeAgent(
   const initializedAgent: InitializedAgent = {
     ...agent,
     resendFiles,
+    imageDetail,
     toolRegistry,
     mcpAvailableTools,
     requestScopedConnections,

packages/api/src/utils/llm.test.ts

@@ -1,3 +1,4 @@
+import { ImageDetail } from 'librechat-data-provider';
 import { extractLibreChatParams } from './llm';
 
 describe('extractLibreChatParams', () => {
@@ -30,6 +31,7 @@ describe('extractLibreChatParams', () => {
       maxContextTokens: 4096,
       fileTokenLimit: 50000,
       modelLabel: 'GPT-4',
+      imageDetail: ImageDetail.high,
       model: 'gpt-4',
       temperature: 0.7,
       max_tokens: 1000,
@@ -42,6 +44,7 @@ describe('extractLibreChatParams', () => {
     expect(result.maxContextTokens).toBe(4096);
     expect(result.fileTokenLimit).toBe(50000);
     expect(result.modelLabel).toBe('GPT-4');
+    expect(result.imageDetail).toBe(ImageDetail.high);
     expect(result.modelOptions).toEqual({
       model: 'gpt-4',
       temperature: 0.7,
@@ -49,6 +52,21 @@ describe('extractLibreChatParams', () => {
     });
   });
 
+  it('should keep imageDetail out of the model options sent to the provider', () => {
+    const result = extractLibreChatParams({ model: 'gpt-4o', imageDetail: ImageDetail.low });
+
+    expect(result.imageDetail).toBe(ImageDetail.low);
+    expect(result.modelOptions).toEqual({ model: 'gpt-4o' });
+    expect('imageDetail' in result.modelOptions).toBe(false);
+  });
+
+  it('should leave imageDetail undefined when the agent configures none', () => {
+    const result = extractLibreChatParams({ model: 'gpt-4o' });
+
+    expect(result.imageDetail).toBeUndefined();
+    expect(result.modelOptions).toEqual({ model: 'gpt-4o' });
+  });
+
   it('should handle null values for LibreChat params', () => {
     const options = {
       resendFiles: true,

packages/api/src/utils/llm.ts

@@ -1,5 +1,5 @@
 import { librechat } from 'librechat-data-provider';
-import type { DynamicSettingProps } from 'librechat-data-provider';
+import type { DynamicSettingProps, ImageDetail } from 'librechat-data-provider';
 
 type LibreChatKeys = keyof typeof librechat;
 
@@ -10,6 +10,7 @@ type LibreChatParams = {
   maxContextTokens?: number;
   fileTokenLimit?: number;
   modelLabel?: string | null;
+  imageDetail?: ImageDetail;
 };
 
 /**
@@ -35,6 +36,7 @@ export function extractLibreChatParams(
   const maxContextTokens = (delete modelOptions.maxContextTokens, options.maxContextTokens);
   const fileTokenLimit = (delete modelOptions.fileTokenLimit, options.fileTokenLimit);
   const modelLabel = (delete modelOptions.modelLabel, options.modelLabel);
+  const imageDetail = (delete modelOptions.imageDetail, options.imageDetail);
 
   return {
     modelOptions: modelOptions as Omit<
@@ -46,5 +48,6 @@ export function extractLibreChatParams(
     promptPrefix,
     resendFiles,
     modelLabel,
+    imageDetail,
   };
 }

packages/data-provider/src/parameterSettings.ts

@@ -71,24 +71,6 @@ const baseDefinitions: Record<string, SettingDefinition> = {
     minTags: 0,
     maxTags: 4,
   },
-  imageDetail: {
-    key: 'imageDetail',
-    label: 'com_endpoint_plug_image_detail',
-    labelCode: true,
-    description: 'com_endpoint_openai_detail',
-    descriptionCode: true,
-    type: 'enum',
-    default: ImageDetail.auto,
-    component: 'slider',
-    options: [ImageDetail.low, ImageDetail.auto, ImageDetail.high],
-    enumMappings: {
-      [ImageDetail.low]: 'com_ui_low',
-      [ImageDetail.auto]: 'com_ui_auto',
-      [ImageDetail.high]: 'com_ui_high',
-    },
-    optionType: 'conversation',
-    columnSpan: 2,
-  },
 };
 
 const createDefinition = (
@@ -147,6 +129,26 @@ export const librechat = {
     placeholderCode: true,
     optionType: 'model',
   } as const,
+  /** Controls how LibreChat encodes image content blocks, not a provider request
+   * parameter — so it belongs to this group and is stripped from model options. */
+  imageDetail: {
+    key: 'imageDetail',
+    label: 'com_endpoint_plug_image_detail',
+    labelCode: true,
+    description: 'com_endpoint_openai_detail',
+    descriptionCode: true,
+    type: 'enum',
+    default: ImageDetail.auto,
+    component: 'slider',
+    options: [ImageDetail.low, ImageDetail.auto, ImageDetail.high],
+    enumMappings: {
+      [ImageDetail.low]: 'com_ui_low',
+      [ImageDetail.auto]: 'com_ui_auto',
+      [ImageDetail.high]: 'com_ui_high',
+    },
+    optionType: 'conversation',
+    columnSpan: 2,
+  } as SettingDefinition,
   fileTokenLimit: {
     key: 'fileTokenLimit',
     label: 'com_ui_file_token_limit',
@@ -887,7 +889,7 @@ const openAI: SettingsConfiguration = [
   openAIParams.presence_penalty,
   baseDefinitions.stop,
   librechat.resendFiles,
-  baseDefinitions.imageDetail,
+  librechat.imageDetail,
   openAIParams.web_search,
   openAIParams.reasoning_effort,
   openAIParams.useResponsesApi,
@@ -920,7 +922,7 @@ const openAICol2: SettingsConfiguration = [
   openAIParams.presence_penalty,
   baseDefinitions.stop,
   librechat.resendFiles,
-  baseDefinitions.imageDetail,
+  librechat.imageDetail,
   openAIParams.reasoning_effort,
   openAIParams.reasoning_summary,
   openAIParams.reasoning_mode,