From 06a6c424356bbc0e9ca5bdc691e04d709bed95c2 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 29 May 2026 08:09:32 -0700 Subject: [PATCH] =?UTF-8?q?=E2=99=8A=20feat:=20Model-Aware=20Max=20Output?= =?UTF-8?q?=20Tokens=20for=20Google/Gemini=20(#13390)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 📤 feat: Model-Aware Max Output Tokens for Google/Gemini Resolves #13384. Current Gemini text models (2.5 and 3+, including Gemini 3.5 Flash) support 64K output tokens, but LibreChat defaulted every Google model to the legacy 8K value — most visibly in the Agents model-parameter panel. - Add model-aware `reset`/`set` to `googleSettings.maxOutputTokens`, mirroring the Anthropic pattern: Gemini 2.5/3+ -> 65536, legacy (2.0 and earlier) and Gemma -> 8192. - Resolve the default server-side in `getGoogleConfig` and in the Agents, preset, and standard Google settings panels via a shared `applyModelAwareDefaults` helper. - Make `compactGoogleSchema` and `generateGoogleSchema` model-aware so explicit user values are preserved and not overwritten. * 🛡️ fix: Cap Google max output at Vertex-safe limits Addresses Codex review (P1) on #13390. Vertex AI caps current Gemini text models at 65,535 output tokens (vs 65,536 on AI Studio) and image models at 32,768, so an unconditional 65,536 default could make otherwise-default Vertex requests fail validation. - Lower the modern text default/ceiling to 65535 (valid on both Vertex and AI Studio). - Resolve Gemini image models (e.g. gemini-2.5-flash-image) to 32768. - Add reset/set + getGoogleConfig tests for image models and the Vertex default path. * 🧮 fix: Respect configured Google defaults and legacy image caps Addresses Codex review round 2 on #13390 (one P2, two P3). - P2 (llm.ts): apply the model-aware maxOutputTokens default as the final fallback instead of pre-filling it, so an explicit value, `defaultParams`, and `addParams` all take precedence and `dropParams` is honored. Empty-string values stay stripped (preserves prior Gemini empty-payload handling). - P3 (panels): pass the resolved params endpoint (`overriddenEndpointKey`) to `applyModelAwareDefaults`, so custom endpoints with `defaultParamsEndpoint: 'google'` also surface the model-aware default. - P3 (schemas): nest the image-model check inside the 2.5+/3+ version check, so legacy image IDs (e.g. gemini-2.0-flash-preview-image-generation) keep the 8K cap instead of being treated as 32K models. - Add tests for defaultParams precedence, dropParams, legacy image models, and the Vertex default path. * 🧭 fix: Base Google defaults on final model and configured overrides Addresses Codex review round 3 on #13390 (two P2). - llm.ts: resolve the model-aware maxOutputTokens default from the final `llmConfig.model` (after defaultParams/addParams) instead of the model captured from modelOptions, so a model forced via addParams/paramDefinitions on a Google-compatible custom endpoint gets its correct limit. - Panels: apply model-aware defaults to the built-in settings first, then overlay `customParams.paramDefinitions`, so an admin-configured maxOutputTokens default wins in the UI (consistent with backend precedence). - Add parameterSettings.spec for applyModelAwareDefaults (incl. override precedence) and a getGoogleConfig final-model test. --- .../components/Endpoints/Settings/Google.tsx | 7 +- .../SidePanel/Agents/ModelPanel.tsx | 12 +- .../components/SidePanel/Parameters/Panel.tsx | 12 +- packages/api/src/endpoints/google/llm.spec.ts | 81 +++++++++++++ packages/api/src/endpoints/google/llm.ts | 20 +++ packages/data-provider/src/generate.ts | 4 +- .../src/parameterSettings.spec.ts | 57 +++++++++ .../data-provider/src/parameterSettings.ts | 20 +++ packages/data-provider/src/schemas.spec.ts | 114 +++++++++++++++++- packages/data-provider/src/schemas.ts | 36 +++++- 10 files changed, 349 insertions(+), 14 deletions(-) create mode 100644 packages/data-provider/src/parameterSettings.spec.ts diff --git a/client/src/components/Endpoints/Settings/Google.tsx b/client/src/components/Endpoints/Settings/Google.tsx index 18bf95a1d0..caf4e100be 100644 --- a/client/src/components/Endpoints/Settings/Google.tsx +++ b/client/src/components/Endpoints/Settings/Google.tsx @@ -49,6 +49,7 @@ export default function Settings({ conversation, setOption, models, readonly }: const setTopP = setOption('topP'); const setTopK = setOption('topK'); const setMaxOutputTokens = setOption('maxOutputTokens'); + const maxOutputTokensDefault = google.maxOutputTokens.reset(model ?? ''); return (
@@ -267,7 +268,7 @@ export default function Settings({ conversation, setOption, models, readonly }: ( {localize('com_endpoint_default_with_num', { - 0: google.maxOutputTokens.default + '', + 0: maxOutputTokensDefault + '', })} ) @@ -292,9 +293,9 @@ export default function Settings({ conversation, setOption, models, readonly }:
setMaxOutputTokens(value[0])} - onDoubleClick={() => setMaxOutputTokens(google.maxOutputTokens.default)} + onDoubleClick={() => setMaxOutputTokens(maxOutputTokensDefault)} max={google.maxOutputTokens.max} min={google.maxOutputTokens.min} step={google.maxOutputTokens.step} diff --git a/client/src/components/SidePanel/Agents/ModelPanel.tsx b/client/src/components/SidePanel/Agents/ModelPanel.tsx index 312e2c52c5..d1bf137505 100644 --- a/client/src/components/SidePanel/Agents/ModelPanel.tsx +++ b/client/src/components/SidePanel/Agents/ModelPanel.tsx @@ -11,6 +11,7 @@ import { LocalStorageKeys, SettingDefinition, agentParamSettings, + applyModelAwareDefaults, } from 'librechat-data-provider'; import type * as t from 'librechat-data-provider'; import type { AgentForm, AgentModelPanelProps, StringOption } from '~/common'; @@ -82,9 +83,14 @@ export default function ModelPanel({ agentParamSettings[combinedKey] ?? agentParamSettings[overriddenEndpointKey] ?? []; const overriddenParams = endpointsConfig[provider]?.customParams?.paramDefinitions ?? []; const overriddenParamsMap = keyBy(overriddenParams, 'key'); - return defaultParams - .filter((param) => param != null) - .map((param) => (overriddenParamsMap[param.key] as SettingDefinition) ?? param); + const modelAwareParams = applyModelAwareDefaults( + defaultParams.filter((param) => param != null), + overriddenEndpointKey, + model ?? '', + ); + return modelAwareParams.map( + (param) => (overriddenParamsMap[param.key] as SettingDefinition) ?? param, + ); }, [endpointType, endpointsConfig, model, provider]); const setOption = (optionKey: keyof t.AgentModelParameters) => (value: t.AgentParameterValue) => { diff --git a/client/src/components/SidePanel/Parameters/Panel.tsx b/client/src/components/SidePanel/Parameters/Panel.tsx index a62ca5834a..cd33937af7 100644 --- a/client/src/components/SidePanel/Parameters/Panel.tsx +++ b/client/src/components/SidePanel/Parameters/Panel.tsx @@ -8,6 +8,7 @@ import { getEndpointField, SettingDefinition, tConvoUpdateSchema, + applyModelAwareDefaults, } from 'librechat-data-provider'; import type { TPreset } from 'librechat-data-provider'; import { SaveAsPresetDialog } from '~/components/Endpoints'; @@ -45,9 +46,14 @@ export default function Parameters() { const defaultParams = paramSettings[combinedKey] ?? paramSettings[overriddenEndpointKey] ?? []; const overriddenParams = endpointsConfig[provider]?.customParams?.paramDefinitions ?? []; const overriddenParamsMap = keyBy(overriddenParams, 'key'); - return defaultParams - .filter((param) => param != null) - .map((param) => (overriddenParamsMap[param.key] as SettingDefinition) ?? param); + const modelAwareParams = applyModelAwareDefaults( + defaultParams.filter((param) => param != null), + overriddenEndpointKey, + model, + ); + return modelAwareParams.map( + (param) => (overriddenParamsMap[param.key] as SettingDefinition) ?? param, + ); }, [endpointType, endpointsConfig, model, provider]); useEffect(() => { diff --git a/packages/api/src/endpoints/google/llm.spec.ts b/packages/api/src/endpoints/google/llm.spec.ts index 6443939a9b..a078842543 100644 --- a/packages/api/src/endpoints/google/llm.spec.ts +++ b/packages/api/src/endpoints/google/llm.spec.ts @@ -99,6 +99,87 @@ describe('getGoogleConfig', () => { }); }); + describe('Model-aware maxOutputTokens default', () => { + const credentials = { + [AuthKeys.GOOGLE_API_KEY]: 'test-api-key', + }; + const vertexCredentials = { + [AuthKeys.GOOGLE_SERVICE_KEY]: { + project_id: 'test-project', + client_email: 'test@test-project.iam.gserviceaccount.com', + private_key: 'test-private-key', + }, + }; + + it('defaults current Gemini models to 65535 when unset', () => { + const result = getGoogleConfig(credentials, { + modelOptions: { model: 'gemini-2.5-pro' }, + }); + expect(result.llmConfig).toHaveProperty('maxOutputTokens', 65535); + }); + + it('defaults Gemini 3.5 Flash to 65535 when unset', () => { + const result = getGoogleConfig(credentials, { + modelOptions: { model: 'gemini-3.5-flash' }, + }); + expect(result.llmConfig).toHaveProperty('maxOutputTokens', 65535); + }); + + it('defaults Gemini image models to 32768 when unset', () => { + const result = getGoogleConfig(credentials, { + modelOptions: { model: 'gemini-2.5-flash-image' }, + }); + expect(result.llmConfig).toHaveProperty('maxOutputTokens', 32768); + }); + + it('defaults legacy Gemini models to 8192 when unset', () => { + const result = getGoogleConfig(credentials, { + modelOptions: { model: 'gemini-2.0-flash' }, + }); + expect(result.llmConfig).toHaveProperty('maxOutputTokens', 8192); + }); + + it('keeps the Vertex default within the model output limit', () => { + const result = getGoogleConfig(vertexCredentials, { + modelOptions: { model: 'gemini-2.5-flash' }, + }); + expect(result.provider).toBe(Providers.VERTEXAI); + expect(result.llmConfig).toHaveProperty('maxOutputTokens', 65535); + }); + + it('preserves an explicit maxOutputTokens value', () => { + const result = getGoogleConfig(credentials, { + modelOptions: { model: 'gemini-2.5-pro', maxOutputTokens: 1024 }, + }); + expect(result.llmConfig).toHaveProperty('maxOutputTokens', 1024); + }); + + it('lets a configured defaultParams maxOutputTokens take precedence over the model default', () => { + const result = getGoogleConfig(credentials, { + modelOptions: { model: 'gemini-2.5-pro' }, + defaultParams: { maxOutputTokens: 2048 }, + }); + expect(result.llmConfig).toHaveProperty('maxOutputTokens', 2048); + }); + + it('omits maxOutputTokens when listed in dropParams', () => { + const result = getGoogleConfig(credentials, { + modelOptions: { model: 'gemini-2.5-pro' }, + dropParams: ['maxOutputTokens'], + }); + expect(result.llmConfig).not.toHaveProperty('maxOutputTokens'); + }); + + it('bases the default on the final model after an addParams override', () => { + const result = getGoogleConfig(credentials, { + modelOptions: { model: 'gemini-1.5-flash' }, + addParams: { model: 'gemini-2.5-pro' }, + }); + expect(result.llmConfig).toHaveProperty('model', 'gemini-2.5-pro'); + expect(result.llmConfig).toHaveProperty('maxOutputTokens', 65535); + }); + }); + describe('Empty String Handling (Issue Fix)', () => { it('should remove empty string maxOutputTokens from config', () => { const credentials = { diff --git a/packages/api/src/endpoints/google/llm.ts b/packages/api/src/endpoints/google/llm.ts index e9133e60b3..880f3ef5d6 100644 --- a/packages/api/src/endpoints/google/llm.ts +++ b/packages/api/src/endpoints/google/llm.ts @@ -533,6 +533,26 @@ export function getGoogleConfig( }); } + /** + * Apply the model-aware `maxOutputTokens` default last, so an explicit value, + * `defaultParams`, and `addParams` all take precedence and a `dropParams` entry + * is respected. Only fill in when the field is genuinely unset (`undefined`/`null`); + * an empty-string value stays stripped per Gemini empty-payload handling. Without + * this, current Gemini models would inherit the legacy 8K default instead of their + * documented limit. + */ + const maxOutputDropped = + Array.isArray(options.dropParams) && options.dropParams.includes('maxOutputTokens'); + if ( + !maxOutputDropped && + modelOptions?.maxOutputTokens == null && + (llmConfig as Record).maxOutputTokens == null + ) { + const resolvedModel = (llmConfig as { model?: string }).model || modelName; + (llmConfig as GoogleClientOptions).maxOutputTokens = + googleSettings.maxOutputTokens.reset(resolvedModel); + } + applyGemini35FlashOverrides({ config: llmConfig, provider, diff --git a/packages/data-provider/src/generate.ts b/packages/data-provider/src/generate.ts index 21f63a34d9..c184983505 100644 --- a/packages/data-provider/src/generate.ts +++ b/packages/data-provider/src/generate.ts @@ -614,7 +614,9 @@ export const generateGoogleSchema = (customGoogle: GoogleSettings) => { promptPrefix: obj.promptPrefix ?? null, examples: obj.examples ?? [{ input: { content: '' }, output: { content: '' } }], temperature: obj.temperature ?? defaults.temperature.default, - maxOutputTokens: obj.maxOutputTokens ?? defaults.maxOutputTokens.default, + maxOutputTokens: + obj.maxOutputTokens ?? + defaults.maxOutputTokens.reset(obj.model ?? defaults.model.default), topP: obj.topP ?? defaults.topP.default, topK: obj.topK ?? defaults.topK.default, maxContextTokens: obj.maxContextTokens ?? undefined, diff --git a/packages/data-provider/src/parameterSettings.spec.ts b/packages/data-provider/src/parameterSettings.spec.ts new file mode 100644 index 0000000000..a8e269525a --- /dev/null +++ b/packages/data-provider/src/parameterSettings.spec.ts @@ -0,0 +1,57 @@ +import { EModelEndpoint } from './types'; +import { applyModelAwareDefaults, paramSettings } from './parameterSettings'; +import type { SettingDefinition } from './generate'; + +const googleParams = paramSettings[EModelEndpoint.google] as SettingDefinition[]; +const maxOut = (params: SettingDefinition[]) => params.find((p) => p.key === 'maxOutputTokens'); + +describe('applyModelAwareDefaults', () => { + it('resolves the Google maxOutputTokens default for current Gemini models', () => { + const result = applyModelAwareDefaults(googleParams, EModelEndpoint.google, 'gemini-2.5-pro'); + expect(maxOut(result)?.default).toBe(65535); + }); + + it('resolves the legacy default for older Gemini models', () => { + const result = applyModelAwareDefaults(googleParams, EModelEndpoint.google, 'gemini-1.5-flash'); + expect(maxOut(result)?.default).toBe(8192); + }); + + it('resolves the image default for Gemini image models', () => { + const result = applyModelAwareDefaults( + googleParams, + EModelEndpoint.google, + 'gemini-2.5-flash-image', + ); + expect(maxOut(result)?.default).toBe(32768); + }); + + it('returns settings unchanged for non-Google endpoints', () => { + const result = applyModelAwareDefaults( + googleParams, + EModelEndpoint.anthropic, + 'gemini-2.5-pro', + ); + expect(result).toBe(googleParams); + }); + + it('returns settings unchanged when no model is provided', () => { + expect(applyModelAwareDefaults(googleParams, EModelEndpoint.google, '')).toBe(googleParams); + }); + + it('does not mutate the original settings', () => { + const before = maxOut(googleParams)?.default; + applyModelAwareDefaults(googleParams, EModelEndpoint.google, 'gemini-2.5-pro'); + expect(maxOut(googleParams)?.default).toBe(before); + }); + + it('lets a configured override applied afterward take precedence', () => { + const modelAware = applyModelAwareDefaults( + googleParams, + EModelEndpoint.google, + 'gemini-2.5-pro', + ); + const override = { ...maxOut(modelAware), default: 2048 } as SettingDefinition; + const final = modelAware.map((p) => (p.key === 'maxOutputTokens' ? override : p)); + expect(maxOut(final)?.default).toBe(2048); + }); +}); diff --git a/packages/data-provider/src/parameterSettings.ts b/packages/data-provider/src/parameterSettings.ts index e9b4f36f11..f3ff5deedc 100644 --- a/packages/data-provider/src/parameterSettings.ts +++ b/packages/data-provider/src/parameterSettings.ts @@ -1141,3 +1141,23 @@ export const agentParamSettings: Record + setting.key === 'maxOutputTokens' + ? { ...setting, default: googleSettings.maxOutputTokens.reset(model) } + : setting, + ); +} diff --git a/packages/data-provider/src/schemas.spec.ts b/packages/data-provider/src/schemas.spec.ts index 34cd72a346..1ce3a596fc 100644 --- a/packages/data-provider/src/schemas.spec.ts +++ b/packages/data-provider/src/schemas.spec.ts @@ -1,4 +1,10 @@ -import { AnthropicEffort, anthropicSettings, eAnthropicEffortSchema } from './schemas'; +import { + AnthropicEffort, + googleSettings, + anthropicSettings, + compactGoogleSchema, + eAnthropicEffortSchema, +} from './schemas'; describe('anthropicSettings', () => { describe('maxOutputTokens.reset()', () => { @@ -354,6 +360,112 @@ describe('anthropicSettings', () => { }); }); +describe('googleSettings', () => { + describe('maxOutputTokens.reset()', () => { + const { reset } = googleSettings.maxOutputTokens; + + describe('current Gemini text models (64K, Vertex-safe)', () => { + it.each([ + 'gemini-2.5-pro', + 'gemini-2.5-flash', + 'gemini-2.5-flash-lite', + 'gemini-2.5-pro-preview-05-06', + 'gemini-3', + 'gemini-3-pro', + 'gemini-3.1', + 'gemini-3.1-flash-lite', + 'gemini-3.5-flash', + 'models/gemini-3.5-flash', + 'gemini-4-pro', + 'gemini-10-flash', + ])('returns 65535 for %s', (model) => { + expect(reset(model)).toBe(65535); + }); + }); + + describe('Gemini image models (32K)', () => { + it.each(['gemini-2.5-flash-image', 'gemini-3-pro-image'])('returns 32768 for %s', (model) => { + expect(reset(model)).toBe(32768); + }); + }); + + describe('legacy/deprecated Gemini and Gemma models (8K)', () => { + it.each([ + 'gemini', + 'gemini-pro', + 'gemini-pro-vision', + 'gemini-1.0-pro', + 'gemini-1.5-pro', + 'gemini-1.5-flash', + 'gemini-1.5-flash-latest', + 'gemini-1.5-flash-8b', + 'gemini-2.0-flash', + 'gemini-2.0-flash-lite', + 'gemini-2.0-flash-preview-image-generation', + 'gemini-exp-1206', + 'gemma-3-27b', + ])('returns 8192 for %s', (model) => { + expect(reset(model)).toBe(8192); + }); + }); + }); + + describe('maxOutputTokens.set()', () => { + const { set } = googleSettings.maxOutputTokens; + + it('caps current Gemini models at 65535', () => { + expect(set(100000, 'gemini-2.5-pro')).toBe(65535); + expect(set(100000, 'gemini-3.5-flash')).toBe(65535); + }); + + it('allows values within the current 64K limit', () => { + expect(set(32000, 'gemini-2.5-flash')).toBe(32000); + expect(set(65535, 'gemini-3.5-flash')).toBe(65535); + }); + + it('caps Gemini image models at 32768', () => { + expect(set(65535, 'gemini-2.5-flash-image')).toBe(32768); + expect(set(32768, 'gemini-2.5-flash-image')).toBe(32768); + }); + + it('caps legacy Gemini models at 8192', () => { + expect(set(65535, 'gemini-2.0-flash')).toBe(8192); + expect(set(20000, 'gemini-1.5-flash')).toBe(8192); + }); + + it('allows values within the legacy 8K limit', () => { + expect(set(4096, 'gemini-1.5-flash')).toBe(4096); + expect(set(8192, 'gemini-2.0-flash')).toBe(8192); + }); + }); + + describe('compactGoogleSchema (model-aware maxOutputTokens)', () => { + it('strips the model default for current Gemini models', () => { + const result = compactGoogleSchema.parse({ + model: 'gemini-2.5-pro', + maxOutputTokens: 65535, + }); + expect(result.maxOutputTokens).toBeUndefined(); + }); + + it('preserves a deliberate below-default value for current Gemini models', () => { + const result = compactGoogleSchema.parse({ + model: 'gemini-2.5-pro', + maxOutputTokens: 8192, + }); + expect(result.maxOutputTokens).toBe(8192); + }); + + it('strips the legacy default for legacy Gemini models', () => { + const result = compactGoogleSchema.parse({ + model: 'gemini-1.5-flash', + maxOutputTokens: 8192, + }); + expect(result.maxOutputTokens).toBeUndefined(); + }); + }); +}); + describe('AnthropicEffort', () => { it('exposes xhigh between high and max in the enum', () => { expect(AnthropicEffort.xhigh).toBe('xhigh'); diff --git a/packages/data-provider/src/schemas.ts b/packages/data-provider/src/schemas.ts index 7fa1826e59..c3cb4cef39 100644 --- a/packages/data-provider/src/schemas.ts +++ b/packages/data-provider/src/schemas.ts @@ -366,15 +366,45 @@ export const openAISettings = { }, }; +/** + * `65535` (not 65536) is the value valid on both Google AI Studio and Vertex AI: + * Vertex caps current Gemini text models at 65,535 output tokens, so defaulting to + * 65,536 would make otherwise-default Vertex requests fail validation. + */ +const GOOGLE_MAX_OUTPUT = 65535 as const; +const GOOGLE_IMAGE_MAX_OUTPUT = 32768 as const; +const GOOGLE_LEGACY_MAX_OUTPUT = 8192 as const; + +/** + * Resolves the documented max output-token limit for a Google/Gemini model. + * Current Gemini text models (2.5 and 3+) support 64K output tokens; their image + * variants (e.g. `gemini-2.5-flash-image`) cap at 32K; legacy/deprecated models + * (2.0 and earlier, including legacy image models) and Gemma retain the 8K limit. + */ +const getGoogleMaxOutputTokens = (modelName: string): number => { + if (/gemini-(?:2\.5|[3-9]|\d{2,})/i.test(modelName)) { + if (/image/i.test(modelName)) { + return GOOGLE_IMAGE_MAX_OUTPUT; + } + return GOOGLE_MAX_OUTPUT; + } + return GOOGLE_LEGACY_MAX_OUTPUT; +}; + export const googleSettings = { model: { default: 'gemini-1.5-flash-latest' as const, }, maxOutputTokens: { min: 1 as const, - max: 65536 as const, + max: GOOGLE_MAX_OUTPUT, step: 1 as const, - default: 8192 as const, + default: GOOGLE_LEGACY_MAX_OUTPUT, + reset: (modelName: string): number => getGoogleMaxOutputTokens(modelName), + set: (value: number, modelName: string): number => { + const max = getGoogleMaxOutputTokens(modelName); + return value > max ? max : value; + }, }, temperature: { min: 0 as const, @@ -1265,7 +1295,7 @@ export const compactGoogleSchema = googleBaseSchema if (newObj.temperature === google.temperature.default) { delete newObj.temperature; } - if (newObj.maxOutputTokens === google.maxOutputTokens.default) { + if (newObj.maxOutputTokens === google.maxOutputTokens.reset(newObj.model ?? '')) { delete newObj.maxOutputTokens; } if (newObj.topP === google.topP.default) {