From 4f5486c9328638d7adecd8ae749b21026b344f71 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 23 May 2026 16:55:57 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=9B=B0=EF=B8=8F=20feat:=20Support=20Gemin?= =?UTF-8?q?i=20Tool=20Combinations=20(#13273)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: support Gemini tool combinations * test: align Gemini provider tool shape * fix: avoid duplicate native web search tools * fix: gate Gemini tool combinations by model --- .../src/agents/__tests__/initialize.test.ts | 162 ++++++++++++++++-- packages/api/src/agents/initialize.ts | 48 +++++- 2 files changed, 187 insertions(+), 23 deletions(-) diff --git a/packages/api/src/agents/__tests__/initialize.test.ts b/packages/api/src/agents/__tests__/initialize.test.ts index 5b229b1f58..35a8c91ffd 100644 --- a/packages/api/src/agents/__tests__/initialize.test.ts +++ b/packages/api/src/agents/__tests__/initialize.test.ts @@ -225,6 +225,17 @@ function countNamedWebSearchTools(tools: unknown[] | undefined): number { ); } +function countGoogleSearchTools(tools: unknown[] | undefined): number { + return ( + tools?.filter((tool) => { + if (tool == null || typeof tool !== 'object') { + return false; + } + return 'googleSearch' in tool || 'googleSearchRetrieval' in tool; + }).length ?? 0 + ); +} + function countWebSearchDefinitions( toolDefinitions: Array<{ name: string }> | undefined, ): number { @@ -308,16 +319,22 @@ describe('initializeAgent — custom provider token lookup', () => { }); }); -describe('initializeAgent — Anthropic web_search precedence', () => { +describe('initializeAgent — provider web_search precedence', () => { const nativeWebSearchTool = { type: 'web_search_20250305', name: Tools.web_search, }; + const nativeGoogleSearchTool = { googleSearch: {} }; const libreChatWebSearchDefinition = { name: Tools.web_search, description: 'Search the web', parameters: { type: 'object', properties: {} }, }; + const mcpToolDefinition = { + name: 'mcp_lookup', + description: 'Lookup context', + parameters: { type: 'object', properties: {} }, + }; beforeEach(() => { jest.clearAllMocks(); @@ -398,7 +415,85 @@ describe('initializeAgent — Anthropic web_search precedence', () => { expect(countWebSearchDefinitions(result.toolDefinitions)).toBe(1); }); - it('leaves non-Anthropic providers unchanged', async () => { + it('keeps Google native search when LibreChat web_search is not selected', async () => { + const { agent, req, res, loadTools, db } = createMocks({ + provider: Providers.GOOGLE, + model: 'gemini-3.5-flash', + providerTools: [nativeGoogleSearchTool], + loadedToolDefinitions: [mcpToolDefinition], + }); + agent.tools = ['mcp_lookup']; + + const result = await initializeAgent( + { + req, + res, + agent, + loadTools, + endpointOption: { endpoint: EModelEndpoint.agents }, + allowedProviders: new Set([Providers.GOOGLE]), + isInitialAgent: true, + }, + db, + ); + + expect(result.tools).toEqual([nativeGoogleSearchTool]); + expect(countGoogleSearchTools(result.tools)).toBe(1); + expect(result.toolDefinitions).toContain(mcpToolDefinition); + }); + + it('rejects Google native search with external tools for unsupported Gemini models', async () => { + const { agent, req, res, loadTools, db } = createMocks({ + provider: Providers.GOOGLE, + model: 'gemini-2.5-flash', + providerTools: [nativeGoogleSearchTool], + loadedToolDefinitions: [mcpToolDefinition], + }); + agent.tools = ['mcp_lookup']; + + await expect( + initializeAgent( + { + req, + res, + agent, + loadTools, + endpointOption: { endpoint: EModelEndpoint.agents }, + allowedProviders: new Set([Providers.GOOGLE]), + isInitialAgent: true, + }, + db, + ), + ).rejects.toThrow(/google_tool_conflict/); + }); + + it('prefers LibreChat web_search when Google native search is also enabled', async () => { + const { agent, req, res, loadTools, db } = createMocks({ + provider: Providers.GOOGLE, + providerTools: [nativeGoogleSearchTool], + loadedToolDefinitions: [libreChatWebSearchDefinition], + }); + agent.tools = [Tools.web_search]; + + const result = await initializeAgent( + { + req, + res, + agent, + loadTools, + endpointOption: { endpoint: EModelEndpoint.agents }, + allowedProviders: new Set([Providers.GOOGLE]), + isInitialAgent: true, + }, + db, + ); + + expect(result.tools).toEqual([]); + expect(countGoogleSearchTools(result.tools)).toBe(0); + expect(countWebSearchDefinitions(result.toolDefinitions)).toBe(1); + }); + + it('leaves providers without a native web search conflict unchanged', async () => { const { agent, req, res, loadTools, db } = createMocks({ provider: Providers.OPENAI, providerTools: [nativeWebSearchTool], @@ -1456,27 +1551,21 @@ describe('initializeAgent — execute_code capability expansion', () => { expect(neitherAgent.codeEnvAvailable).toBe(false); }); - it('trips GOOGLE_TOOL_CONFLICT on Google/Vertex when execute_code expands alongside provider tools', async () => { - /* Pre-Phase 8, an `execute_code`-only agent on Google/Vertex with - `options.tools` populated would throw GOOGLE_TOOL_CONFLICT because - `CodeExecutionToolDefinition` populated `toolDefinitions` and - `hasAgentTools` was true. After dropping that registry entry, the - check is now gated on the runtime-expanded `bash_tool` + `read_file` - pair — so the expansion MUST happen before `hasAgentTools` is - computed or the guard silently goes away for this scenario. */ + it('allows Google provider tools alongside execute_code definitions', async () => { const { agent, req, res, loadTools, db } = createMocks({ provider: Providers.GOOGLE, overrideProvider: Providers.GOOGLE, + model: 'gemini-3.5-flash', }); agent.tools = ['execute_code']; /* Surface an options.tools array from the provider config — this is - the `google_search` / `url_context` built-in LLM tooling that + the `googleSearch` / `urlContext` built-in LLM tooling that Google/Vertex exposes via provider options. */ mockGetProviderConfig.mockReturnValue({ getOptions: jest.fn().mockResolvedValue({ - llmConfig: { model: 'test-model', maxTokens: 4096 }, - tools: [{ google_search: {} }], + llmConfig: { model: 'gemini-3.5-flash', maxTokens: 4096 }, + tools: [{ googleSearch: {} }], } satisfies InitializeResultBase), overrideProvider: Providers.GOOGLE, }); @@ -1495,7 +1584,52 @@ describe('initializeAgent — execute_code capability expansion', () => { }, db, ), - ).rejects.toThrow(/google_tool_conflict/); + ).resolves.toEqual( + expect.objectContaining({ + tools: [{ googleSearch: {} }], + toolDefinitions: expect.arrayContaining([ + expect.objectContaining({ name: 'bash_tool' }), + expect.objectContaining({ name: 'read_file' }), + ]), + }), + ); + }); + + it('combines Google provider tools with structured external tools', async () => { + const structuredTool = { + name: 'weather', + description: 'Get weather', + schema: { type: 'object', properties: {} }, + }; + const providerTool = { googleSearch: {} }; + const { agent, req, res, loadTools, db } = createMocks({ + provider: Providers.GOOGLE, + overrideProvider: Providers.GOOGLE, + model: 'gemini-3.5-flash', + providerTools: [providerTool], + structuredTools: [structuredTool], + }); + agent.tools = ['weather']; + + await expect( + initializeAgent( + { + req, + res, + agent, + loadTools, + endpointOption: { endpoint: EModelEndpoint.agents }, + allowedProviders: new Set([Providers.GOOGLE]), + isInitialAgent: true, + codeEnvAvailable: false, + }, + db, + ), + ).resolves.toEqual( + expect.objectContaining({ + tools: [structuredTool, providerTool], + }), + ); }); }); diff --git a/packages/api/src/agents/initialize.ts b/packages/api/src/agents/initialize.ts index cfe32fedd3..85152a9afd 100644 --- a/packages/api/src/agents/initialize.ts +++ b/packages/api/src/agents/initialize.ts @@ -77,7 +77,22 @@ function hasToolDefinition(toolDefinitions: LCTool[] | undefined, name: string): return toolDefinitions?.some((toolDefinition) => toolDefinition.name === name) === true; } -function resolveAnthropicToolConflicts({ +function hasGoogleSearchTool(tool: unknown): boolean { + if (tool == null || typeof tool !== 'object') { + return false; + } + return 'googleSearch' in tool || 'googleSearchRetrieval' in tool; +} + +function supportsGoogleToolCombination(model: unknown): boolean { + if (typeof model !== 'string') { + return false; + } + const normalized = model.toLowerCase().split('/').pop() ?? model.toLowerCase(); + return normalized.startsWith('gemini-3'); +} + +function resolveProviderToolConflicts({ provider, tools, toolDefinitions, @@ -86,7 +101,7 @@ function resolveAnthropicToolConflicts({ tools?: unknown[]; toolDefinitions?: LCTool[]; }): unknown[] | undefined { - if (provider !== Providers.ANTHROPIC || !tools?.length) { + if (!tools?.length) { return tools; } @@ -94,9 +109,19 @@ function resolveAnthropicToolConflicts({ return tools; } + const shouldRemoveTool = (tool: unknown): boolean => { + if (provider === Providers.ANTHROPIC) { + return getToolName(tool) === Tools.web_search; + } + if (provider === Providers.GOOGLE || provider === Providers.VERTEXAI) { + return hasGoogleSearchTool(tool); + } + return false; + }; + let removed = 0; const resolvedTools = tools.filter((tool) => { - const shouldRemove = getToolName(tool) === Tools.web_search; + const shouldRemove = shouldRemoveTool(tool); if (shouldRemove) { removed += 1; } @@ -105,7 +130,7 @@ function resolveAnthropicToolConflicts({ if (removed > 0) { logger.debug( - `[initializeAgent] Removed ${removed} Anthropic native web_search tool(s); LibreChat web_search is enabled.`, + `[initializeAgent] Removed ${removed} ${provider} native web search tool(s); LibreChat web_search is enabled.`, ); } @@ -847,9 +872,9 @@ export async function initializeAgent( * never accidentally registers `bash_tool` or primes sandbox files * just because the admin globally enabled code execution. * - * Done BEFORE the `hasAgentTools` / GOOGLE_TOOL_CONFLICT gate so - * execute-code-only agents on Google/Vertex still trip the conflict - * guard when provider-specific tools are also configured. Also before + * Done before provider-tool merging so execute-code-only agents on + * Google/Vertex still surface as external function definitions when + * provider-specific tools are also configured. Also before * `injectSkillCatalog` so the skill path's own * `registerCodeExecutionTools` call upgrades `read_file` from the * code-only description to the skill-aware description without adding a @@ -882,7 +907,7 @@ export async function initializeAgent( /** Check for tool presence from either full instances or definitions (event-driven mode) */ const hasAgentTools = (structuredTools?.length ?? 0) > 0 || (toolDefinitions?.length ?? 0) > 0; - const providerTools = resolveAnthropicToolConflicts({ + const providerTools = resolveProviderToolConflicts({ provider: agent.provider, tools: options.tools, toolDefinitions, @@ -898,7 +923,12 @@ export async function initializeAgent( hasProviderTools && hasAgentTools ) { - throw new Error(`{ "type": "${ErrorTypes.GOOGLE_TOOL_CONFLICT}"}`); + if (!supportsGoogleToolCombination(llmConfig.model)) { + throw new Error(`{ "type": "${ErrorTypes.GOOGLE_TOOL_CONFLICT}"}`); + } + if (structuredTools?.length) { + tools = structuredTools.concat(providerTools as GenericTool[]); + } } else if ( (agent.provider === Providers.OPENAI || agent.provider === Providers.AZURE ||