diff --git a/.env.example b/.env.example index b63c9605c2..9b9cea96ed 100644 --- a/.env.example +++ b/.env.example @@ -1302,6 +1302,14 @@ OPENWEATHER_API_KEY= # Tavily (Search Provider and/or Scraper) # TAVILY_API_KEY=your_tavily_api_key +# Keenable (Search Provider and/or Scraper; keyless by default, a key only lifts +# rate limits and covers both search and page fetch) +# KEENABLE_API_KEY=your_keenable_api_key +# Optional: Custom Keenable search API URL +# KEENABLE_API_URL=your_keenable_api_url +# Optional: Custom Keenable fetch API URL (used when scraperProvider is keenable) +# KEENABLE_FETCH_URL=your_keenable_fetch_api_url + # Scraper (Required) # FIRECRAWL_API_KEY=your_firecrawl_api_key # Optional: Custom Firecrawl API URL diff --git a/api/app/clients/tools/util/handleTools.js b/api/app/clients/tools/util/handleTools.js index 1eb28acd2c..aa20ba06c8 100644 --- a/api/app/clients/tools/util/handleTools.js +++ b/api/app/clients/tools/util/handleTools.js @@ -418,6 +418,10 @@ const loadTools = async ({ loadAuthValues, webSearchConfig: webSearch, }); + if (!result.authenticated) { + logger.warn('[handleTools] Skipping web search because authentication is incomplete.'); + continue; + } const { onSearchResults, onGetHighlights } = options?.[Tools.web_search] ?? {}; const { httpAgent, httpsAgent } = resolveWebSearchSSRFAgents( result.authResult, diff --git a/api/app/clients/tools/util/handleTools.test.js b/api/app/clients/tools/util/handleTools.test.js index 63953a1ec3..6ceaa578cf 100644 --- a/api/app/clients/tools/util/handleTools.test.js +++ b/api/app/clients/tools/util/handleTools.test.js @@ -14,6 +14,7 @@ const mockGetAccessibleMcpServerNames = jest.fn(async () => []); const mockCreateSearchTool = jest.fn(() => ({ name: 'web_search' })); const mockLoadWebSearchAuth = jest.fn(async () => ({ + authenticated: true, authResult: { searchProvider: 'serper', searxngInstanceUrl: 'http://searxng.internal:8080' }, })); @@ -895,5 +896,23 @@ describe('Tool Handlers', () => { loadWebSearchConfig({ allowedAddresses: { '10.0.0.5:11434': true } }), ).resolves.toBeDefined(); }); + + it('does not construct web search when authentication is incomplete', async () => { + mockLoadWebSearchAuth.mockResolvedValueOnce({ + authenticated: false, + authResult: { searchProvider: 'keenable' }, + }); + + const toolMap = await loadTools({ + user: fakeUser._id.toString(), + tools: [Tools.web_search], + returnMap: true, + webSearch: {}, + options: { req: buildReq() }, + }); + + expect(toolMap[Tools.web_search]).toBeUndefined(); + expect(mockCreateSearchTool).not.toHaveBeenCalled(); + }); }); }); diff --git a/api/package.json b/api/package.json index b31d6287dc..a8600fa1b0 100644 --- a/api/package.json +++ b/api/package.json @@ -46,7 +46,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.8.0", "@keyv/redis": "5.1.6", - "@librechat/agents": "^3.7.5", + "@librechat/agents": "^3.7.6", "@librechat/api": "*", "@librechat/data-schemas": "*", "@microsoft/microsoft-graph-client": "^3.0.7", diff --git a/api/server/controllers/UserController.js b/api/server/controllers/UserController.js index 040b2ae21b..d41e27a072 100644 --- a/api/server/controllers/UserController.js +++ b/api/server/controllers/UserController.js @@ -1,5 +1,5 @@ const mongoose = require('mongoose'); -const { logger, getTenantId, webSearchKeys } = require('@librechat/data-schemas'); +const { logger, getTenantId } = require('@librechat/data-schemas'); const { getNewS3URL, needsRefresh, @@ -8,7 +8,8 @@ const { MCPTokenStorage, getAppConfigOptionsFromUser, normalizeHttpError, - extractWebSearchEnvVars, + getWebSearchInstallEntries, + getWebSearchUninstallFields, deleteAgentCheckpoints, deleteAllSharedLinksWithCleanup, } = require('@librechat/api'); @@ -233,18 +234,17 @@ const updateUserPluginsController = async (req, res) => { return res.status(200).send(); } - let keys = Object.keys(auth); - const values = Object.values(auth); // Used in 'install' block + let authEntries = Object.entries(auth); const isMCPTool = pluginKey.startsWith('mcp_') || pluginKey.includes(Constants.mcp_delimiter); // Early exit condition: - // If keys are empty (meaning auth: {} was likely sent for uninstall, or auth was empty for install) - // AND it's not web_search (which has special key handling to populate `keys` for uninstall) + // If auth is empty (meaning auth: {} was likely sent for uninstall or install) + // AND it's not web_search (which expands its uninstall fields) // AND it's NOT (an uninstall action FOR an MCP tool - we need to proceed for this case to clear all its auth) // THEN return. if ( - keys.length === 0 && + authEntries.length === 0 && pluginKey !== Tools.web_search && !(action === 'uninstall' && isMCPTool) ) { @@ -261,23 +261,29 @@ const updateUserPluginsController = async (req, res) => { if (pluginKey === Tools.web_search) { /** @type {TCustomConfig['webSearch']} */ const webSearchConfig = appConfig?.webSearch; - keys = extractWebSearchEnvVars({ - keys: action === 'install' ? keys : webSearchKeys, - config: webSearchConfig, - }); + authEntries = + action === 'install' + ? getWebSearchInstallEntries({ auth, config: webSearchConfig }) + : getWebSearchUninstallFields(webSearchConfig).map((field) => [field, '']); } if (action === 'install') { - for (let i = 0; i < keys.length; i++) { - authService = await updateUserPluginAuth(user.id, keys[i], pluginKey, values[i]); + for (const [field, value] of authEntries) { + authService = + pluginKey === Tools.web_search && value === '' + ? await deleteUserPluginAuth(user.id, field) + : await updateUserPluginAuth(user.id, field, pluginKey, value); if (authService instanceof Error) { logger.error('[authService]', authService); ({ status, message } = normalizeHttpError(authService)); + if (pluginKey === Tools.web_search) { + break; + } } } } else if (action === 'uninstall') { // const isMCPTool was defined earlier - if (isMCPTool && keys.length === 0) { + if (isMCPTool && authEntries.length === 0) { // This handles the case where auth: {} is sent for an MCP tool uninstall. // It means "delete all credentials associated with this MCP pluginKey". authService = await deleteUserPluginAuth(user.id, null, true, pluginKey); @@ -299,12 +305,11 @@ const updateUserPluginsController = async (req, res) => { } } else { // This handles: - // 1. Web_search uninstall (keys will be populated with all webSearchKeys if auth was {}). - // 2. Other tools uninstall (if keys were provided). - // 3. MCP tool uninstall if specific keys were provided in `auth` (not current frontend behavior). - // If keys is empty for non-MCP tools (and not web_search), this loop won't run, and nothing is deleted. - for (let i = 0; i < keys.length; i++) { - authService = await deleteUserPluginAuth(user.id, keys[i]); // Deletes by authField name + // 1. Web_search uninstall (entries include every configured field). + // 2. Other tools uninstall (if auth fields were provided). + // 3. MCP tool uninstall if specific fields were provided in `auth`. + for (const [field] of authEntries) { + authService = await deleteUserPluginAuth(user.id, field); // Deletes by authField name if (authService instanceof Error) { logger.error('[authService] Error deleting specific auth key:', authService); ({ status, message } = normalizeHttpError(authService)); diff --git a/api/server/controllers/UserController.spec.js b/api/server/controllers/UserController.spec.js index 89af705a56..f6fbf8cdbd 100644 --- a/api/server/controllers/UserController.spec.js +++ b/api/server/controllers/UserController.spec.js @@ -10,6 +10,7 @@ const mockPurgeAgentTriggerDeliveriesForUser = jest.fn().mockResolvedValue(undef const mockCancelAndDrainSubagentThreads = jest.fn().mockResolvedValue(undefined); const mockQuiesceUserSchedules = jest.fn().mockResolvedValue(true); const mockRestoreUserSchedules = jest.fn().mockResolvedValue(undefined); +const mockGetWebSearchInstallEntries = jest.fn(); jest.mock('@librechat/data-schemas', () => { const actual = jest.requireActual('@librechat/data-schemas'); @@ -86,6 +87,7 @@ jest.mock('@librechat/api', () => ({ ...jest.requireActual('@librechat/api'), needsRefresh: jest.fn(), getNewS3URL: jest.fn(), + getWebSearchInstallEntries: (...args) => mockGetWebSearchInstallEntries(...args), GenerationJobManager: { getCleanupBlockingJobIdsForUser: (...args) => mockGetActiveJobIdsForUser(...args), abortJob: (...args) => mockAbortJob(...args), @@ -146,6 +148,7 @@ const { deleteUserController, getUserController, acceptTermsController, + updateUserPluginsController, resendVerificationController, verifyEmailController, } = require('./UserController'); @@ -159,6 +162,97 @@ const { cancelAgentTriggerUserDeletion, } = require('~/models'); const { verifyEmail, resendVerificationEmail } = require('~/server/services/AuthService'); +const { updateUserPluginAuth, deleteUserPluginAuth } = require('~/server/services/PluginService'); +const { webSearchSelectionFields } = require('@librechat/data-schemas'); + +describe('updateUserPluginsController', () => { + const mockRes = { + status: jest.fn().mockReturnThis(), + send: jest.fn().mockReturnThis(), + json: jest.fn().mockReturnThis(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('does not persist web-search selections after a credential write fails', async () => { + mockGetWebSearchInstallEntries.mockReturnValue([ + ['KEENABLE_API_KEY', 'new-key'], + [webSearchSelectionFields.selectedProvider, 'keenable'], + [webSearchSelectionFields.selectedScraper, 'keenable'], + [webSearchSelectionFields.selectedReranker, 'none'], + ]); + updateUserPluginAuth.mockResolvedValueOnce(new Error('credential write failed')); + + await updateUserPluginsController( + { + config: { + webSearch: { + keenableApiKey: '${KEENABLE_API_KEY}', + keenableApiUrl: '${KEENABLE_API_URL}', + }, + }, + user: { id: 'user-id', _id: 'user-id', plugins: [] }, + body: { + pluginKey: 'web_search', + action: 'install', + isEntityTool: true, + auth: { + selectedProvider: 'keenable', + selectedScraper: 'keenable', + selectedReranker: 'none', + keenableApiKey: 'new-key', + }, + }, + }, + mockRes, + ); + + expect(updateUserPluginAuth).toHaveBeenCalledTimes(1); + expect(updateUserPluginAuth).toHaveBeenCalledWith( + 'user-id', + 'KEENABLE_API_KEY', + 'web_search', + 'new-key', + ); + for (const selectionField of Object.values(webSearchSelectionFields)) { + expect(updateUserPluginAuth).not.toHaveBeenCalledWith( + 'user-id', + selectionField, + 'web_search', + expect.anything(), + ); + } + }); + + it('deletes explicitly cleared web-search credentials', async () => { + mockGetWebSearchInstallEntries.mockReturnValue([['KEENABLE_API_URL', '']]); + + await updateUserPluginsController( + { + config: { + webSearch: { + keenableApiUrl: '${KEENABLE_API_URL}', + }, + }, + user: { id: 'user-id', _id: 'user-id', plugins: [] }, + body: { + pluginKey: 'web_search', + action: 'install', + isEntityTool: true, + auth: { + keenableApiUrl: '', + }, + }, + }, + mockRes, + ); + + expect(deleteUserPluginAuth).toHaveBeenCalledWith('user-id', 'KEENABLE_API_URL'); + expect(updateUserPluginAuth).not.toHaveBeenCalled(); + }); +}); describe('verifyEmailController', () => { const mockRes = { diff --git a/api/server/controllers/__tests__/tools.verifyToolAuth.spec.js b/api/server/controllers/__tests__/tools.verifyToolAuth.spec.js index 03965021c4..3200039470 100644 --- a/api/server/controllers/__tests__/tools.verifyToolAuth.spec.js +++ b/api/server/controllers/__tests__/tools.verifyToolAuth.spec.js @@ -32,6 +32,7 @@ jest.mock('~/app/clients/tools/util', () => ({ })); const { Tools, AuthType } = require('librechat-data-provider'); +const { loadWebSearchAuth } = require('@librechat/api'); const { verifyToolAuth } = require('../tools'); /** @@ -100,3 +101,49 @@ describe('verifyToolAuth — execute_code system-auth contract', () => { expect(payload.message).not.toBe(AuthType.USER_PROVIDED); }); }); + +describe('verifyToolAuth — web search selection contract', () => { + const makeRes = () => { + const res = {}; + res.status = jest.fn().mockReturnValue(res); + res.json = jest.fn().mockReturnValue(res); + return res; + }; + + it('returns the resolved provider selections with the auth state', async () => { + loadWebSearchAuth.mockResolvedValue({ + authenticated: true, + authTypes: { + providers: AuthType.USER_PROVIDED, + scrapers: AuthType.USER_PROVIDED, + rerankers: AuthType.SYSTEM_DEFINED, + }, + authResult: { + searchProvider: 'keenable', + scraperProvider: 'keenable', + rerankerType: 'none', + }, + }); + const req = { + params: { toolId: Tools.web_search }, + user: { id: 'user-1' }, + config: { webSearch: {} }, + }; + const res = makeRes(); + + await verifyToolAuth(req, res); + + expect(res.status).toHaveBeenCalledWith(200); + expect(res.json).toHaveBeenCalledWith({ + authenticated: true, + authTypes: { + providers: AuthType.USER_PROVIDED, + scrapers: AuthType.USER_PROVIDED, + rerankers: AuthType.SYSTEM_DEFINED, + }, + searchProvider: 'keenable', + scraperProvider: 'keenable', + rerankerType: 'none', + }); + }); +}); diff --git a/api/server/controllers/tools.js b/api/server/controllers/tools.js index 89509e5140..c2298c5fb4 100644 --- a/api/server/controllers/tools.js +++ b/api/server/controllers/tools.js @@ -56,6 +56,9 @@ const verifyWebSearchAuth = async (req, res) => { return res.status(200).json({ authenticated: result.authenticated, authTypes: result.authTypes, + searchProvider: result.authResult.searchProvider, + scraperProvider: result.authResult.scraperProvider, + rerankerType: result.authResult.rerankerType, }); } catch (error) { console.error('Error in verifyWebSearchAuth:', error); diff --git a/api/server/services/PluginService.js b/api/server/services/PluginService.js index 10422d49d2..71941b9fc5 100644 --- a/api/server/services/PluginService.js +++ b/api/server/services/PluginService.js @@ -42,7 +42,9 @@ const getUserPluginAuthValue = async (userId, authField, throwError = true, plug const pluginAuth = await findOnePluginAuth(searchParams); if (!pluginAuth) { const pluginInfo = pluginKey ? ` for plugin ${pluginKey}` : ''; - throw new Error(`No plugin auth ${authField} found for user ${userId}${pluginInfo}`); + const error = new Error(`No plugin auth ${authField} found for user ${userId}${pluginInfo}`); + error.code = 'PLUGIN_AUTH_NOT_FOUND'; + throw error; } const decryptedValue = await decrypt(pluginAuth.value); @@ -51,7 +53,9 @@ const getUserPluginAuthValue = async (userId, authField, throwError = true, plug if (!throwError) { return null; } - logger.error('[getUserPluginAuthValue]', err); + if (err?.code !== 'PLUGIN_AUTH_NOT_FOUND') { + logger.error('[getUserPluginAuthValue]', err); + } throw err; } }; diff --git a/api/server/services/PluginService.spec.js b/api/server/services/PluginService.spec.js new file mode 100644 index 0000000000..c4858e6885 --- /dev/null +++ b/api/server/services/PluginService.spec.js @@ -0,0 +1,46 @@ +const mockLoggerError = jest.fn(); +const mockFindOnePluginAuth = jest.fn(); + +jest.mock('@librechat/data-schemas', () => ({ + logger: { error: mockLoggerError }, +})); + +jest.mock('@librechat/api', () => ({ + encrypt: jest.fn(), + decrypt: jest.fn(), +})); + +jest.mock('~/models', () => ({ + findOnePluginAuth: (...args) => mockFindOnePluginAuth(...args), + updatePluginAuth: jest.fn(), + deletePluginAuth: jest.fn(), +})); + +const { getUserPluginAuthValue } = require('./PluginService'); + +describe('getUserPluginAuthValue', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('does not log an expected missing auth value', async () => { + mockFindOnePluginAuth.mockResolvedValue(null); + + await expect( + getUserPluginAuthValue('user-id', 'WEB_SEARCH_SELECTED_PROVIDER'), + ).rejects.toMatchObject({ code: 'PLUGIN_AUTH_NOT_FOUND' }); + + expect(mockLoggerError).not.toHaveBeenCalled(); + }); + + it('still logs credential-store failures', async () => { + const error = new Error('database unavailable'); + mockFindOnePluginAuth.mockRejectedValue(error); + + await expect(getUserPluginAuthValue('user-id', 'WEB_SEARCH_SELECTED_PROVIDER')).rejects.toBe( + error, + ); + + expect(mockLoggerError).toHaveBeenCalledWith('[getUserPluginAuthValue]', error); + }); +}); diff --git a/api/server/services/Tools/credentials.js b/api/server/services/Tools/credentials.js index a289134a5a..11b232227e 100644 --- a/api/server/services/Tools/credentials.js +++ b/api/server/services/Tools/credentials.js @@ -8,9 +8,16 @@ const { getUserPluginAuthValue } = require('~/server/services/PluginService'); * @param {string[]} params.authFields * @param {Set} [params.optional] * @param {boolean} [params.throwError] + * @param {boolean} [params.failOnOptionalError] * @returns */ -const loadAuthValues = async ({ userId, authFields, optional, throwError = true }) => { +const loadAuthValues = async ({ + userId, + authFields, + optional, + throwError = true, + failOnOptionalError = false, +}) => { let authValues = {}; /** @@ -28,7 +35,9 @@ const loadAuthValues = async ({ userId, authFields, optional, throwError = true try { value = await getUserPluginAuthValue(userId, field, throwError); } catch (err) { - if (optional && optional.has(field)) { + const isOptional = optional && optional.has(field); + const isMissingOptional = isOptional && err?.code === 'PLUGIN_AUTH_NOT_FOUND'; + if (isOptional && (!failOnOptionalError || isMissingOptional)) { return { authField: field, authValue: undefined }; } if (field === fields[fields.length - 1]) { @@ -42,9 +51,11 @@ const loadAuthValues = async ({ userId, authFields, optional, throwError = true return null; }; - for (let authField of authFields) { - const fields = authField.split('||'); - const result = await findAuthValue(fields); + const results = await Promise.all( + authFields.map((authField) => findAuthValue(authField.split('||'))), + ); + + for (const result of results) { if (result) { authValues[result.authField] = result.authValue; } diff --git a/api/server/services/Tools/credentials.spec.js b/api/server/services/Tools/credentials.spec.js index 727edbd9f3..fc396ce05a 100644 --- a/api/server/services/Tools/credentials.spec.js +++ b/api/server/services/Tools/credentials.spec.js @@ -94,6 +94,35 @@ describe('loadAuthValues', () => { expect(getUserPluginAuthValue).not.toHaveBeenCalled(); }); + it('should load independent authentication fields in parallel', async () => { + delete process.env.FIRST_KEY; + delete process.env.SECOND_KEY; + let resolveFirst; + const firstValue = new Promise((resolve) => { + resolveFirst = resolve; + }); + getUserPluginAuthValue.mockImplementation((_userId, field) => { + if (field === 'FIRST_KEY') { + return firstValue; + } + return Promise.resolve('second-value'); + }); + + const resultPromise = loadAuthValues({ + userId: 'user1', + authFields: ['FIRST_KEY', 'SECOND_KEY'], + }); + await Promise.resolve(); + const callCountBeforeFirstResolved = getUserPluginAuthValue.mock.calls.length; + resolveFirst('first-value'); + + await expect(resultPromise).resolves.toEqual({ + FIRST_KEY: 'first-value', + SECOND_KEY: 'second-value', + }); + expect(callCountBeforeFirstResolved).toBe(2); + }); + it('should return real env value from first matching field in fallback chain', async () => { process.env.GEMINI_API_KEY = 'gemini-key'; process.env.GOOGLE_KEY = 'google-key'; @@ -120,6 +149,34 @@ describe('loadAuthValues', () => { expect(result).toEqual({ GOOGLE_KEY: undefined }); }); + it('should distinguish a missing optional credential from a credential-store failure', async () => { + delete process.env.KEENABLE_API_URL; + const missingError = Object.assign(new Error('No auth found'), { + code: 'PLUGIN_AUTH_NOT_FOUND', + }); + getUserPluginAuthValue.mockRejectedValueOnce(missingError); + + await expect( + loadAuthValues({ + userId: 'user1', + authFields: ['KEENABLE_API_URL'], + optional: new Set(['KEENABLE_API_URL']), + failOnOptionalError: true, + }), + ).resolves.toEqual({ KEENABLE_API_URL: undefined }); + + getUserPluginAuthValue.mockRejectedValueOnce(new Error('Database unavailable')); + + await expect( + loadAuthValues({ + userId: 'user1', + authFields: ['KEENABLE_API_URL'], + optional: new Set(['KEENABLE_API_URL']), + failOnOptionalError: true, + }), + ).rejects.toThrow('Database unavailable'); + }); + it('should not leak sentinel through catch path when DB lookup throws', async () => { process.env.GOOGLE_KEY = AuthType.USER_PROVIDED; getUserPluginAuthValue.mockRejectedValue(new Error('No auth found')); diff --git a/client/src/components/Chat/Input/ToolDialogs.tsx b/client/src/components/Chat/Input/ToolDialogs.tsx index 86d383b639..9d3b929562 100644 --- a/client/src/components/Chat/Input/ToolDialogs.tsx +++ b/client/src/components/Chat/Input/ToolDialogs.tsx @@ -32,10 +32,14 @@ function ToolDialogs() { isOpen={searchDialogOpen} onRevoke={searchHandleRevoke} register={searchMethods.register} + setValue={searchMethods.setValue} onOpenChange={setSearchDialogOpen} handleSubmit={searchMethods.handleSubmit} triggerRefs={[searchMenuTriggerRef, searchBadgeTriggerRef]} isToolAuthenticated={webSearchAuthData?.authenticated ?? false} + searchProvider={webSearchAuthData?.searchProvider} + scraperProvider={webSearchAuthData?.scraperProvider} + rerankerType={webSearchAuthData?.rerankerType} /> ); } diff --git a/client/src/components/SidePanel/Agents/Search/Action.tsx b/client/src/components/SidePanel/Agents/Search/Action.tsx index 5cd627393b..de70e609ff 100644 --- a/client/src/components/SidePanel/Agents/Search/Action.tsx +++ b/client/src/components/SidePanel/Agents/Search/Action.tsx @@ -2,7 +2,13 @@ import { useRef } from 'react'; import { Button } from '@librechat/client'; import { useFormContext } from 'react-hook-form'; import { KeyRound, CircleCheck } from 'lucide-react'; -import { AuthType, AgentCapabilities } from 'librechat-data-provider'; +import { + AuthType, + RerankerTypes, + SearchProviders, + ScraperProviders, + AgentCapabilities, +} from 'librechat-data-provider'; import type { AgentForm } from '~/common'; import { useLocalize, useSearchApiKeyForm } from '~/hooks'; import ApiKeyDialog from './ApiKeyDialog'; @@ -10,9 +16,15 @@ import ApiKeyDialog from './ApiKeyDialog'; export default function Action({ authTypes = [], isToolAuthenticated = false, + searchProvider, + scraperProvider, + rerankerType, }: { authTypes?: [string, AuthType][]; isToolAuthenticated?: boolean; + searchProvider?: SearchProviders; + scraperProvider?: ScraperProviders; + rerankerType?: RerankerTypes; }) { const localize = useLocalize(); const { setValue } = useFormContext(); @@ -68,9 +80,13 @@ export default function Action({ onRevoke={handleRevokeApiKey} onOpenChange={setIsDialogOpen} register={keyFormMethods.register} + setValue={keyFormMethods.setValue} isToolAuthenticated={isToolAuthenticated} handleSubmit={keyFormMethods.handleSubmit} triggerRef={apiKeyButtonRef} + searchProvider={searchProvider} + scraperProvider={scraperProvider} + rerankerType={rerankerType} /> ); diff --git a/client/src/components/SidePanel/Agents/Search/ApiKeyDialog.test.tsx b/client/src/components/SidePanel/Agents/Search/ApiKeyDialog.test.tsx index c8d8f9e331..8db1f888c1 100644 --- a/client/src/components/SidePanel/Agents/Search/ApiKeyDialog.test.tsx +++ b/client/src/components/SidePanel/Agents/Search/ApiKeyDialog.test.tsx @@ -1,8 +1,14 @@ import React from 'react'; import { render, screen, fireEvent } from '@testing-library/react'; -import ApiKeyDialog from './ApiKeyDialog'; -import { AuthType, SearchCategories, RerankerTypes } from 'librechat-data-provider'; +import { + AuthType, + RerankerTypes, + SearchProviders, + SearchCategories, + ScraperProviders, +} from 'librechat-data-provider'; import { useGetStartupConfig } from '~/data-provider'; +import ApiKeyDialog from './ApiKeyDialog'; // Mock useLocalize to just return the key jest.mock('~/hooks', () => ({ @@ -32,6 +38,7 @@ const defaultProps = { ], isToolAuthenticated: false, register: mockRegister as any, + setValue: jest.fn(), handleSubmit: (fn: any) => (e: any) => fn(e), }; @@ -62,6 +69,39 @@ describe('ApiKeyDialog', () => { expect(screen.getByPlaceholderText('com_ui_web_search_cohere_key')).toBeInTheDocument(); }); + it('restores per-user provider and scraper selections', () => { + mockUseGetStartupConfig.mockReturnValue({ data: {} }); + render( + , + ); + + expect( + screen.getByRole('button', { name: 'com_ui_web_search_provider_keenable' }), + ).toBeInTheDocument(); + expect( + screen.getByRole('button', { name: 'com_ui_web_search_scraper_keenable' }), + ).toBeInTheDocument(); + }); + + it('writes keyless provider and reranker selections into the submitted form', () => { + const setValue = jest.fn(); + mockUseGetStartupConfig.mockReturnValue({ data: {} }); + render(); + + fireEvent.click(screen.getByText('com_ui_web_search_provider_keenable')); + fireEvent.click(screen.getByText('com_ui_web_search_reranker_none')); + + expect(setValue).toHaveBeenCalledWith('selectedProvider', SearchProviders.KEENABLE); + expect(setValue).toHaveBeenCalledWith('selectedReranker', RerankerTypes.NONE); + expect(screen.getByPlaceholderText('com_ui_web_search_keenable_url')).toBeInTheDocument(); + expect(screen.queryByPlaceholderText('com_ui_web_search_jina_key')).not.toBeInTheDocument(); + expect(screen.queryByPlaceholderText('com_ui_web_search_cohere_key')).not.toBeInTheDocument(); + }); + it('shows static text for provider and only provider input if provider is set', () => { mockUseGetStartupConfig.mockReturnValue({ data: { webSearch: { searchProvider: 'serper' } } }); render(); @@ -70,6 +110,15 @@ describe('ApiKeyDialog', () => { expect(screen.queryByRole('button', { name: /provider/i })).not.toBeInTheDocument(); }); + it('shows the custom URL input when Keenable is pinned but user-configurable', () => { + mockUseGetStartupConfig.mockReturnValue({ + data: { webSearch: { searchProvider: SearchProviders.KEENABLE } }, + }); + render(); + + expect(screen.getByPlaceholderText('com_ui_web_search_keenable_url')).toBeInTheDocument(); + }); + it('shows only Jina reranker field if rerankerType is set to jina', () => { mockUseGetStartupConfig.mockReturnValue({ data: { webSearch: { rerankerType: RerankerTypes.JINA } }, @@ -114,6 +163,66 @@ describe('ApiKeyDialog', () => { expect(screen.getByText('com_ui_web_search_reranker')).toBeInTheDocument(); }); + it('does not submit selections for system-defined categories', () => { + const onSubmit = jest.fn(); + const formData = { + selectedProvider: SearchProviders.SERPER, + selectedScraper: ScraperProviders.KEENABLE, + selectedReranker: RerankerTypes.NONE, + keenableApiKey: '', + } as any; + mockUseGetStartupConfig.mockReturnValue({ data: {} }); + render( + () => fn(formData)} + />, + ); + + fireEvent.click(screen.getByRole('button', { name: 'com_ui_save' })); + + expect(onSubmit).toHaveBeenCalledWith({ + selectedScraper: ScraperProviders.KEENABLE, + keenableApiKey: '', + }); + }); + + it('does not submit selections for admin-pinned user-auth categories', () => { + const onSubmit = jest.fn(); + const formData = { + selectedProvider: SearchProviders.KEENABLE, + selectedScraper: ScraperProviders.KEENABLE, + selectedReranker: RerankerTypes.NONE, + keenableApiKey: 'user-key', + } as any; + mockUseGetStartupConfig.mockReturnValue({ + data: { + webSearch: { + searchProvider: SearchProviders.KEENABLE, + scraperProvider: ScraperProviders.KEENABLE, + rerankerType: RerankerTypes.NONE, + }, + }, + }); + render( + () => fn(formData)} + />, + ); + + fireEvent.click(screen.getByRole('button', { name: 'com_ui_save' })); + + expect(onSubmit).toHaveBeenCalledWith({ keenableApiKey: 'user-key' }); + }); + it('does not render scraper section if SYSTEM_DEFINED', () => { mockUseGetStartupConfig.mockReturnValue({ data: {} }); const props = { diff --git a/client/src/components/SidePanel/Agents/Search/ApiKeyDialog.tsx b/client/src/components/SidePanel/Agents/Search/ApiKeyDialog.tsx index e2277f0ec6..c7f7a86e39 100644 --- a/client/src/components/SidePanel/Agents/Search/ApiKeyDialog.tsx +++ b/client/src/components/SidePanel/Agents/Search/ApiKeyDialog.tsx @@ -1,4 +1,4 @@ -import { useState } from 'react'; +import { useEffect, useState } from 'react'; import { Button, OGDialog, OGDialogTemplate } from '@librechat/client'; import { AuthType, @@ -7,7 +7,7 @@ import { ScraperProviders, SearchCategories, } from 'librechat-data-provider'; -import type { UseFormRegister, UseFormHandleSubmit } from 'react-hook-form'; +import type { UseFormRegister, UseFormHandleSubmit, UseFormSetValue } from 'react-hook-form'; import type { SearchApiKeyFormData } from '~/hooks/Plugins/useAuthSearchTool'; import InputSection, { type DropdownOption } from './InputSection'; import { useGetStartupConfig } from '~/data-provider'; @@ -21,9 +21,13 @@ export default function ApiKeyDialog({ authTypes, isToolAuthenticated, register, + setValue, handleSubmit, triggerRef, triggerRefs, + searchProvider, + scraperProvider, + rerankerType, }: { isOpen: boolean; onOpenChange: (open: boolean) => void; @@ -32,23 +36,54 @@ export default function ApiKeyDialog({ authTypes: [string, AuthType][]; isToolAuthenticated: boolean; register: UseFormRegister; + setValue: UseFormSetValue; handleSubmit: UseFormHandleSubmit; triggerRef?: React.RefObject; triggerRefs?: React.RefObject[]; + searchProvider?: SearchProviders; + scraperProvider?: ScraperProviders; + rerankerType?: RerankerTypes; }) { const localize = useLocalize(); const { data: config } = useGetStartupConfig(); const [selectedProvider, setSelectedProvider] = useState( - config?.webSearch?.searchProvider || SearchProviders.SERPER, + config?.webSearch?.searchProvider || searchProvider || SearchProviders.SERPER, ); const [selectedReranker, setSelectedReranker] = useState( - config?.webSearch?.rerankerType || RerankerTypes.JINA, + config?.webSearch?.rerankerType || rerankerType || RerankerTypes.JINA, ); const [selectedScraper, setSelectedScraper] = useState( - config?.webSearch?.scraperProvider || ScraperProviders.FIRECRAWL, + config?.webSearch?.scraperProvider || scraperProvider || ScraperProviders.FIRECRAWL, ); + useEffect(() => { + if (!isOpen) { + return; + } + setSelectedProvider( + config?.webSearch?.searchProvider || searchProvider || SearchProviders.SERPER, + ); + setSelectedScraper( + config?.webSearch?.scraperProvider || scraperProvider || ScraperProviders.FIRECRAWL, + ); + setSelectedReranker(config?.webSearch?.rerankerType || rerankerType || RerankerTypes.JINA); + }, [ + config?.webSearch?.rerankerType, + config?.webSearch?.scraperProvider, + config?.webSearch?.searchProvider, + isOpen, + rerankerType, + scraperProvider, + searchProvider, + ]); + + useEffect(() => { + setValue('selectedProvider', selectedProvider); + setValue('selectedScraper', selectedScraper); + setValue('selectedReranker', selectedReranker); + }, [selectedProvider, selectedReranker, selectedScraper, setValue]); + const providerOptions: DropdownOption[] = [ { key: SearchProviders.SERPER, @@ -92,9 +127,31 @@ export default function ApiKeyDialog({ }, }, }, + { + key: SearchProviders.KEENABLE, + label: localize('com_ui_web_search_provider_keenable'), + inputs: { + keenableApiKey: { + placeholder: localize('com_ui_enter_api_key_optional'), + type: 'password' as const, + link: { + url: 'https://keenable.ai', + text: localize('com_ui_web_search_provider_keenable_key'), + }, + }, + keenableApiUrl: { + placeholder: localize('com_ui_web_search_keenable_url'), + type: 'text' as const, + }, + }, + }, ]; const rerankerOptions: DropdownOption[] = [ + { + key: RerankerTypes.NONE, + label: localize('com_ui_web_search_reranker_none'), + }, { key: RerankerTypes.JINA, label: localize('com_ui_web_search_reranker_jina'), @@ -180,6 +237,20 @@ export default function ApiKeyDialog({ }, }, }, + { + key: ScraperProviders.KEENABLE, + label: localize('com_ui_web_search_scraper_keenable'), + inputs: { + keenableApiKey: { + placeholder: localize('com_ui_enter_api_key_optional'), + type: 'password' as const, + link: { + url: 'https://keenable.ai', + text: localize('com_ui_web_search_provider_keenable_key'), + }, + }, + }, + }, ]; const [dropdownOpen, setDropdownOpen] = useState({ @@ -204,6 +275,20 @@ export default function ApiKeyDialog({ setSelectedScraper(key as ScraperProviders); }; + const handleVisibleSubmit = (data: SearchApiKeyFormData) => { + const visibleData = { ...data }; + if (providerAuthType === AuthType.SYSTEM_DEFINED || config?.webSearch?.searchProvider != null) { + delete visibleData.selectedProvider; + } + if (scraperAuthType === AuthType.SYSTEM_DEFINED || config?.webSearch?.scraperProvider != null) { + delete visibleData.selectedScraper; + } + if (rerankerAuthType === AuthType.SYSTEM_DEFINED || config?.webSearch?.rerankerType != null) { + delete visibleData.selectedReranker; + } + onSubmit(visibleData); + }; + return (
{localize('com_ui_web_search')}
-
+ {/* Provider Section */} {providerAuthType !== AuthType.SYSTEM_DEFINED && ( } selection={{ - selectHandler: handleSubmit(onSubmit), + selectHandler: handleSubmit(handleVisibleSubmit), selectClasses: 'bg-surface-submit hover:bg-surface-submit-hover text-white', selectText: localize('com_ui_save'), }} diff --git a/client/src/components/SidePanel/Agents/Tools/ItemDialog/sections/BuiltinSection.tsx b/client/src/components/SidePanel/Agents/Tools/ItemDialog/sections/BuiltinSection.tsx index 9a897da794..16fc9be55b 100644 --- a/client/src/components/SidePanel/Agents/Tools/ItemDialog/sections/BuiltinSection.tsx +++ b/client/src/components/SidePanel/Agents/Tools/ItemDialog/sections/BuiltinSection.tsx @@ -114,7 +114,15 @@ function MemoryConfig({ value, onChange }: MemoryConfigProps) { function WebSearchConfig() { const { data } = useVerifyAgentToolAuth({ toolId: Tools.web_search }, { retry: 1 }); - return ; + return ( + + ); } export default function BuiltinSection({ diff --git a/client/src/hooks/Plugins/__tests__/useAuthSearchTool.test.ts b/client/src/hooks/Plugins/__tests__/useAuthSearchTool.test.ts new file mode 100644 index 0000000000..5fdfeaf4ad --- /dev/null +++ b/client/src/hooks/Plugins/__tests__/useAuthSearchTool.test.ts @@ -0,0 +1,75 @@ +import { Tools } from 'librechat-data-provider'; +import { act, renderHook } from '@testing-library/react'; +import useAuthSearchTool, { type SearchApiKeyFormData } from '../useAuthSearchTool'; + +const mockMutate = jest.fn(); +const mockInvalidateQueries = jest.fn(); +const mockSetQueryData = jest.fn(); + +jest.mock('@tanstack/react-query', () => ({ + useQueryClient: () => ({ + invalidateQueries: mockInvalidateQueries, + setQueryData: mockSetQueryData, + }), +})); + +jest.mock('librechat-data-provider/react-query', () => ({ + useUpdateUserPluginsMutation: () => ({ mutate: mockMutate }), +})); + +const emptyForm = (): SearchApiKeyFormData => ({ + serperApiKey: '', + searxngInstanceUrl: '', + searxngApiKey: '', + firecrawlApiKey: '', + firecrawlApiUrl: '', + tavilyApiKey: '', + keenableApiKey: '', + keenableApiUrl: '', + jinaApiKey: '', + jinaApiUrl: '', + cohereApiKey: '', +}); + +describe('useAuthSearchTool', () => { + it('forwards an explicitly empty Keenable URL as a clear operation', () => { + const { result } = renderHook(() => useAuthSearchTool()); + + act(() => result.current.installTool(emptyForm(), { keenableApiUrl: true })); + + expect(mockMutate).toHaveBeenCalledWith({ + pluginKey: Tools.web_search, + action: 'install', + auth: { keenableApiUrl: '' }, + isEntityTool: true, + }); + }); + + it('omits a pristine empty Keenable URL', () => { + const { result } = renderHook(() => useAuthSearchTool()); + + act(() => result.current.installTool(emptyForm())); + + expect(mockMutate).toHaveBeenCalledWith({ + pluginKey: Tools.web_search, + action: 'install', + auth: {}, + isEntityTool: true, + }); + }); + + it('omits an undefined Keenable URL even when marked dirty', () => { + const data = emptyForm(); + data.keenableApiUrl = undefined as unknown as string; + const { result } = renderHook(() => useAuthSearchTool()); + + act(() => result.current.installTool(data, { keenableApiUrl: true })); + + expect(mockMutate).toHaveBeenCalledWith({ + pluginKey: Tools.web_search, + action: 'install', + auth: {}, + isEntityTool: true, + }); + }); +}); diff --git a/client/src/hooks/Plugins/useAuthSearchTool.ts b/client/src/hooks/Plugins/useAuthSearchTool.ts index ffb156dba1..1cb65ede0f 100644 --- a/client/src/hooks/Plugins/useAuthSearchTool.ts +++ b/client/src/hooks/Plugins/useAuthSearchTool.ts @@ -1,13 +1,20 @@ import { useCallback } from 'react'; import { useQueryClient } from '@tanstack/react-query'; -import { AuthType, Tools, QueryKeys } from 'librechat-data-provider'; import { useUpdateUserPluginsMutation } from 'librechat-data-provider/react-query'; +import { + Tools, + AuthType, + QueryKeys, + RerankerTypes, + SearchProviders, + ScraperProviders, +} from 'librechat-data-provider'; export type SearchApiKeyFormData = { // Selected options - selectedProvider: string; - selectedReranker: string; - selectedScraper: string; + selectedProvider?: SearchProviders; + selectedReranker?: RerankerTypes; + selectedScraper?: ScraperProviders; // API keys and URLs serperApiKey: string; searxngInstanceUrl: string; @@ -15,11 +22,15 @@ export type SearchApiKeyFormData = { firecrawlApiKey: string; firecrawlApiUrl: string; tavilyApiKey: string; + keenableApiKey: string; + keenableApiUrl: string; jinaApiKey: string; jinaApiUrl: string; cohereApiKey: string; }; +export type SearchApiKeyDirtyFields = Partial>; + const useAuthSearchTool = (options?: { isEntityTool: boolean }) => { const queryClient = useQueryClient(); const isEntityTool = options?.isEntityTool ?? true; @@ -28,6 +39,9 @@ const useAuthSearchTool = (options?: { isEntityTool: boolean }) => { queryClient.setQueryData([QueryKeys.toolAuth, Tools.web_search], () => { return { authenticated: vars.action === 'install', + searchProvider: vars.auth?.selectedProvider, + scraperProvider: vars.auth?.selectedScraper, + rerankerType: vars.auth?.selectedReranker, authTypes: vars.action === 'install' ? [ @@ -48,20 +62,27 @@ const useAuthSearchTool = (options?: { isEntityTool: boolean }) => { }); const installTool = useCallback( - (data: SearchApiKeyFormData) => { + (data: SearchApiKeyFormData, dirtyFields: SearchApiKeyDirtyFields = {}) => { const auth = Object.entries({ + selectedProvider: data.selectedProvider, + selectedScraper: data.selectedScraper, + selectedReranker: data.selectedReranker, serperApiKey: data.serperApiKey, searxngInstanceUrl: data.searxngInstanceUrl, searxngApiKey: data.searxngApiKey, firecrawlApiKey: data.firecrawlApiKey, firecrawlApiUrl: data.firecrawlApiUrl, tavilyApiKey: data.tavilyApiKey, + keenableApiKey: data.keenableApiKey, + keenableApiUrl: data.keenableApiUrl, jinaApiKey: data.jinaApiKey, jinaApiUrl: data.jinaApiUrl, cohereApiKey: data.cohereApiKey, }).reduce( (acc, [key, value]) => { - if (value) { + const wasExplicitlyCleared = + value === '' && dirtyFields[key as keyof SearchApiKeyFormData] === true; + if (value || wasExplicitlyCleared) { acc[key] = value; } return acc; diff --git a/client/src/hooks/Plugins/useSearchApiKeyForm.ts b/client/src/hooks/Plugins/useSearchApiKeyForm.ts index 9133abf3af..a8744a798d 100644 --- a/client/src/hooks/Plugins/useSearchApiKeyForm.ts +++ b/client/src/hooks/Plugins/useSearchApiKeyForm.ts @@ -1,7 +1,7 @@ import { useRef, useState, useCallback } from 'react'; import { useForm } from 'react-hook-form'; -import useAuthSearchTool from '~/hooks/Plugins/useAuthSearchTool'; import type { SearchApiKeyFormData } from '~/hooks/Plugins/useAuthSearchTool'; +import useAuthSearchTool from '~/hooks/Plugins/useAuthSearchTool'; export default function useSearchApiKeyForm({ onSubmit, @@ -15,15 +15,18 @@ export default function useSearchApiKeyForm({ const badgeTriggerRef = useRef(null); const [isDialogOpen, setIsDialogOpen] = useState(false); const { installTool, removeTool } = useAuthSearchTool({ isEntityTool: true }); - const { reset } = methods; + const { + reset, + formState: { dirtyFields }, + } = methods; const onSubmitHandler = useCallback( (data: SearchApiKeyFormData) => { - installTool(data); + installTool(data, dirtyFields); setIsDialogOpen(false); onSubmit?.(); }, - [onSubmit, installTool], + [dirtyFields, onSubmit, installTool], ); const handleRevokeApiKey = useCallback(() => { diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.json index 0563fd6dee..98abf4fae9 100644 --- a/client/src/locales/en/translation.json +++ b/client/src/locales/en/translation.json @@ -1233,6 +1233,7 @@ "com_ui_endpoint_menu": "LLM Endpoint Menu", "com_ui_enter": "Enter", "com_ui_enter_api_key": "Enter API Key", + "com_ui_enter_api_key_optional": "Enter API Key (optional)", "com_ui_enter_description": "Enter description (optional)", "com_ui_enter_full_screen": "Enter full screen", "com_ui_enter_key": "Enter key", @@ -2424,8 +2425,11 @@ "com_ui_web_search_firecrawl_url": "Firecrawl API URL (optional)", "com_ui_web_search_jina_key": "Enter Jina API Key", "com_ui_web_search_jina_url": "Jina API URL (optional)", + "com_ui_web_search_keenable_url": "Keenable API URL (optional)", "com_ui_web_search_processing": "Processing results", "com_ui_web_search_provider": "Search Provider", + "com_ui_web_search_provider_keenable": "Keenable API", + "com_ui_web_search_provider_keenable_key": "Get your Keenable API key (optional)", "com_ui_web_search_provider_searxng": "SearXNG", "com_ui_web_search_provider_serper": "Serper API", "com_ui_web_search_provider_serper_key": "Get your Serper API key", @@ -2436,11 +2440,13 @@ "com_ui_web_search_reranker_cohere": "Cohere", "com_ui_web_search_reranker_cohere_key": "Get your Cohere API key", "com_ui_web_search_reranker_jina": "Jina AI", + "com_ui_web_search_reranker_none": "None", "com_ui_web_search_reranker_jina_key": "Get your Jina API key", "com_ui_web_search_reranker_jina_url_help": "Learn about Jina Rerank API", "com_ui_web_search_scraper": "Scraper", "com_ui_web_search_scraper_firecrawl": "Firecrawl API", "com_ui_web_search_scraper_firecrawl_key": "Get your Firecrawl API key", + "com_ui_web_search_scraper_keenable": "Keenable Fetch API", "com_ui_web_search_scraper_serper": "Serper Scrape API", "com_ui_web_search_scraper_serper_key": "Get your Serper API key", "com_ui_web_search_scraper_tavily": "Tavily Extract API", diff --git a/librechat.example.yaml b/librechat.example.yaml index f0eef9fc6b..41ff2f702f 100644 --- a/librechat.example.yaml +++ b/librechat.example.yaml @@ -1027,6 +1027,8 @@ endpoints: # searxngApiKey: '${SEARXNG_API_KEY}' # # Tavily (search provider and/or scraper) # tavilyApiKey: '${TAVILY_API_KEY}' +# # Keenable (search provider; keyless by default, a key only lifts rate limits) +# keenableApiKey: '${KEENABLE_API_KEY}' # # Content scrapers # firecrawlApiKey: '${FIRECRAWL_API_KEY}' # firecrawlApiUrl: '${FIRECRAWL_API_URL}' @@ -1090,6 +1092,40 @@ endpoints: # # format: markdown # 'markdown' (default) or 'text' (plain text, may increase latency) # # timeout: 15000 # HTTP request timeout in milliseconds (max 120000); Tavily Extract receives seconds clamped to 1-60 # +# Keenable as both search provider and scraper example — a fully keyless web +# search stack: no API key for any of the three categories. The public search and +# fetch endpoints are rate-limited; a key only lifts the cap. +# webSearch: +# searchProvider: keenable +# scraperProvider: keenable +# rerankerType: none # Rerankers all require a key; 'none' skips reranking +# # keenableApiKey: '${KEENABLE_API_KEY}' # Optional: a key only lifts rate limits (public endpoints used when unset) +# # keenableApiUrl: '${KEENABLE_API_URL}' # Optional: search API URL override (fetch uses KEENABLE_FETCH_URL) +# keenableSearchOptions: +# maxResults: 5 # 1-20 results per search (default: 8) +# # site: example.com # Restrict search to a single domain +# # attributionTitle: LibreChat # X-Keenable-Title header sent with each request +# # timeout: 15000 # HTTP request timeout in milliseconds (max 120000) +# keenableScraperOptions: +# # attributionTitle: LibreChat # X-Keenable-Title header sent with each fetch +# timeout: 15000 # HTTP request timeout in milliseconds (max 120000) +# +# Keenable search paired with another scraper (Firecrawl here) also works; each +# category is configured independently: +# webSearch: +# searchProvider: keenable +# scraperProvider: firecrawl +# firecrawlApiKey: '${FIRECRAWL_API_KEY}' +# +# The reverse (someone else's search, Keenable as the scraper) also works. Pin +# the stack for every user as shown below, or leave the providers unpinned so +# each user can save this same keyless combination in the API-key dialog: +# webSearch: +# searchProvider: searxng +# searxngInstanceUrl: '${SEARXNG_INSTANCE_URL}' +# scraperProvider: keenable +# rerankerType: none +# # SearXNG as the search provider example: # webSearch: # searchProvider: searxng diff --git a/package-lock.json b/package-lock.json index 81615aa893..9c407b901c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -63,7 +63,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.8.0", "@keyv/redis": "5.1.6", - "@librechat/agents": "^3.7.5", + "@librechat/agents": "^3.7.6", "@librechat/api": "*", "@librechat/data-schemas": "*", "@microsoft/microsoft-graph-client": "^3.0.7", @@ -10633,9 +10633,9 @@ } }, "node_modules/@librechat/agents": { - "version": "3.7.5", - "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-3.7.5.tgz", - "integrity": "sha512-ymIhkegUnO3mIdSjbDb+d8iUNj9YgLGf82kllWFojKhmphTQQbsxNrDaauroxptc/0Ie1XRpqdQthWnmqQKkBA==", + "version": "3.7.6", + "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-3.7.6.tgz", + "integrity": "sha512-v+Oe3RUn4J5CG7+7Y3uGZ+MlC5zReDp0BnvGkyZd9oqlkypR8dbK+lsV3s41kOJ6VEL5rxerZ36b+5TmV/2mww==", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "^0.115.0", @@ -42838,7 +42838,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.8.0", "@keyv/redis": "5.1.6", - "@librechat/agents": "^3.7.5", + "@librechat/agents": "^3.7.6", "@librechat/data-schemas": "*", "@modelcontextprotocol/sdk": "^1.30.0", "@opentelemetry/api": "^1.9.0", diff --git a/packages/api/package.json b/packages/api/package.json index fb79c578e0..4b651b88b6 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -113,7 +113,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.8.0", "@keyv/redis": "5.1.6", - "@librechat/agents": "^3.7.5", + "@librechat/agents": "^3.7.6", "@librechat/data-schemas": "*", "@modelcontextprotocol/sdk": "^1.30.0", "@opentelemetry/api": "^1.9.0", diff --git a/packages/api/src/web/agent.spec.ts b/packages/api/src/web/agent.spec.ts index f75dab16fe..20b35c9553 100644 --- a/packages/api/src/web/agent.spec.ts +++ b/packages/api/src/web/agent.spec.ts @@ -1,3 +1,4 @@ +import { ScraperProviders } from 'librechat-data-provider'; import { resolveWebSearchSSRFAgents } from './agent'; import { isAddressAllowed } from '../auth'; @@ -78,6 +79,40 @@ describe('resolveWebSearchSSRFAgents', () => { expect(isAddressAllowed('proxy.internal', ['proxy.internal:3128'], '3128')).toBe(true); }); + it('derives proxy exemptions from a custom Keenable search URL', () => { + process.env.HTTP_PROXY = 'http://10.11.12.13:3128'; + + const { httpAgent } = resolveWebSearchSSRFAgents({ + keenableApiUrl: 'http://keenable-search.internal:8080', + }); + + expect(() => connect(httpAgent, '10.11.12.13', 3128)).not.toThrow(); + }); + + it('derives proxy exemptions from the Keenable fetch URL environment override', () => { + process.env.HTTP_PROXY = 'http://10.11.12.14:3128'; + process.env.KEENABLE_FETCH_URL = 'http://keenable-fetch.internal:8080'; + + const { httpAgent } = resolveWebSearchSSRFAgents({ + scraperProvider: ScraperProviders.KEENABLE, + }); + + expect(() => connect(httpAgent, '10.11.12.14', 3128)).not.toThrow(); + }); + + it('does not exempt the Keenable fetch proxy for another scraper', () => { + process.env.HTTP_PROXY = 'http://10.11.12.15:3128'; + process.env.KEENABLE_FETCH_URL = 'http://keenable-fetch.internal:8080'; + + const { httpAgent } = resolveWebSearchSSRFAgents({ + scraperProvider: ScraperProviders.FIRECRAWL, + }); + + expect(() => connect(httpAgent, '10.11.12.15', 3128)).toThrow( + expect.objectContaining({ code: 'ESSRF' }), + ); + }); + it('scopes the proxy exemption to its port, so another private port stays blocked', () => { expect(isAddressAllowed('proxy.internal', ['proxy.internal:3128'], '9')).toBe(false); }); diff --git a/packages/api/src/web/agent.ts b/packages/api/src/web/agent.ts index f864175703..5e73700d9f 100644 --- a/packages/api/src/web/agent.ts +++ b/packages/api/src/web/agent.ts @@ -1,4 +1,5 @@ import { getProxyForUrl } from 'proxy-from-env'; +import { ScraperProviders } from 'librechat-data-provider'; import type { TWebSearchConfig } from 'librechat-data-provider'; import type https from 'node:https'; import type http from 'node:http'; @@ -16,6 +17,7 @@ const WEB_SEARCH_URL_KEYS = [ 'jinaApiUrl', 'tavilySearchUrl', 'tavilyExtractUrl', + 'keenableApiUrl', ] as const; /** @@ -31,8 +33,13 @@ const WEB_SEARCH_URL_KEYS = [ */ function getProxyExemptions(authResult: Partial): string[] { const entries = new Set(); - for (const key of WEB_SEARCH_URL_KEYS) { - const destination = authResult[key]; + const destinations = [ + ...WEB_SEARCH_URL_KEYS.map((key) => authResult[key]), + ...(authResult.scraperProvider === ScraperProviders.KEENABLE + ? [process.env.KEENABLE_FETCH_URL] + : []), + ]; + for (const destination of destinations) { if (typeof destination !== 'string' || destination.length === 0) { continue; } diff --git a/packages/api/src/web/web.spec.ts b/packages/api/src/web/web.spec.ts index e10a7f72de..be0748b779 100644 --- a/packages/api/src/web/web.spec.ts +++ b/packages/api/src/web/web.spec.ts @@ -1,13 +1,19 @@ -import { webSearchAuth } from '@librechat/data-schemas'; -import { SafeSearchTypes, AuthType } from 'librechat-data-provider'; -import type { - ScraperProviders, - TWebSearchConfig, - SearchProviders, - TCustomConfig, +import { webSearchAuth, webSearchSelectionFields } from '@librechat/data-schemas'; +import { + AuthType, RerankerTypes, + SafeSearchTypes, + SearchCategories, + SearchProviders, + ScraperProviders, } from 'librechat-data-provider'; -import { loadWebSearchAuth, extractWebSearchEnvVars } from './web'; +import type { TWebSearchConfig, TCustomConfig } from 'librechat-data-provider'; +import { + loadWebSearchAuth, + extractWebSearchEnvVars, + getWebSearchInstallEntries, + getWebSearchUninstallFields, +} from './web'; // Mock the extractVariableName function jest.mock('../utils', () => ({ @@ -75,6 +81,65 @@ describe('web.ts', () => { }); }); + describe('web search plugin auth mapping', () => { + it('keeps credential values paired with fields after unsupported keys are filtered', () => { + const config = { + keenableApiKey: '${KEENABLE_API_KEY}', + } as TWebSearchConfig; + + const entries = getWebSearchInstallEntries({ + auth: { + selectedProvider: 'keenable', + serperApiKey: 'serper-key', + keenableApiKey: 'keenable-key', + }, + config, + }); + + expect(entries).toEqual([ + ['KEENABLE_API_KEY', 'keenable-key'], + [webSearchSelectionFields.selectedProvider, 'keenable'], + ]); + }); + + it('rejects reranker selections that have no runtime service', () => { + const entries = getWebSearchInstallEntries({ + auth: { + selectedReranker: 'infinity', + }, + config: {} as TWebSearchConfig, + }); + + expect(entries).toEqual([]); + }); + + it('preserves explicit empty credential values as clear operations', () => { + const entries = getWebSearchInstallEntries({ + auth: { + keenableApiUrl: '', + }, + config: { + keenableApiUrl: '${KEENABLE_API_URL}', + } as TWebSearchConfig, + }); + + expect(entries).toEqual([['KEENABLE_API_URL', '']]); + }); + + it('clears persisted provider choices with web-search credentials', () => { + const fields = getWebSearchUninstallFields({ + keenableApiKey: '${KEENABLE_API_KEY}', + } as TWebSearchConfig); + + expect(fields).toEqual([ + 'KEENABLE_API_KEY', + webSearchSelectionFields.selectedProvider, + webSearchSelectionFields.selectedScraper, + webSearchSelectionFields.selectedReranker, + ]); + }); + }); + describe('loadWebSearchAuth', () => { // Common test variables const userId = 'test-user-id'; @@ -450,6 +515,612 @@ describe('web.ts', () => { expect(result.authResult).toHaveProperty('safeSearch', SafeSearchTypes.OFF); }); + it('should authenticate Keenable keyless (no API key configured)', async () => { + // Keenable works against the public endpoint with no key. Return nothing for + // the KEENABLE_* fields; other categories still authenticate normally. + mockLoadAuthValues.mockImplementation(({ authFields }) => { + const result: Record = {}; + authFields.forEach((field: string) => { + if (field.startsWith('KEENABLE_')) { + return; + } + result[field] = + field === 'FIRECRAWL_API_URL' ? 'https://api.firecrawl.dev' : 'test-api-key'; + }); + return Promise.resolve(result); + }); + + const keenableConfig = { + ...webSearchConfig, + keenableApiKey: '${KEENABLE_API_KEY}', + keenableApiUrl: '${KEENABLE_API_URL}', + searchProvider: 'keenable' as SearchProviders, + keenableSearchOptions: { maxResults: 7 }, + } as TWebSearchConfig; + + const result = await loadWebSearchAuth({ + userId, + webSearchConfig: keenableConfig, + loadAuthValues: mockLoadAuthValues, + }); + + expect(result.authResult.searchProvider).toBe('keenable' as SearchProviders); + // No key was resolved, so it should not be set on the result. + expect(result.authResult.keenableApiKey).toBeUndefined(); + // Provider-specific options are passed through untouched. + expect(result.authResult.keenableSearchOptions).toEqual({ maxResults: 7 }); + }); + + it('should pick up a configured Keenable API key', async () => { + mockLoadAuthValues.mockImplementation(({ authFields }) => { + const result: Record = {}; + authFields.forEach((field: string) => { + if (field === 'KEENABLE_API_KEY') { + result[field] = 'user-keenable-key'; + } else if (field === 'KEENABLE_API_URL') { + return; + } else { + result[field] = + field === 'FIRECRAWL_API_URL' ? 'https://api.firecrawl.dev' : 'test-api-key'; + } + }); + return Promise.resolve(result); + }); + + const keenableConfig = { + ...webSearchConfig, + keenableApiKey: '${KEENABLE_API_KEY}', + keenableApiUrl: '${KEENABLE_API_URL}', + searchProvider: 'keenable' as SearchProviders, + } as TWebSearchConfig; + + const result = await loadWebSearchAuth({ + userId, + webSearchConfig: keenableConfig, + loadAuthValues: mockLoadAuthValues, + }); + + expect(result.authResult.searchProvider).toBe('keenable' as SearchProviders); + expect(result.authResult.keenableApiKey).toBe('user-keenable-key'); + }); + + it('should keep Keenable editable when the URL is user-provided but not saved yet', async () => { + const originalApiKey = process.env.KEENABLE_API_KEY; + const originalApiUrl = process.env.KEENABLE_API_URL; + process.env.KEENABLE_API_KEY = 'system-keenable-key'; + process.env.KEENABLE_API_URL = AuthType.USER_PROVIDED; + mockLoadAuthValues.mockResolvedValue({ + KEENABLE_API_KEY: 'system-keenable-key', + }); + + try { + const result = await loadWebSearchAuth({ + userId, + webSearchConfig: { + keenableApiKey: '${KEENABLE_API_KEY}', + keenableApiUrl: '${KEENABLE_API_URL}', + searchProvider: SearchProviders.KEENABLE, + scraperProvider: ScraperProviders.KEENABLE, + rerankerType: RerankerTypes.NONE, + safeSearch: SafeSearchTypes.MODERATE, + } as TWebSearchConfig, + loadAuthValues: mockLoadAuthValues, + }); + + expect(result.authTypes).toEqual([ + [SearchCategories.PROVIDERS, AuthType.USER_PROVIDED], + [SearchCategories.SCRAPERS, AuthType.USER_PROVIDED], + [SearchCategories.RERANKERS, AuthType.SYSTEM_DEFINED], + ]); + } finally { + if (originalApiKey == null) { + delete process.env.KEENABLE_API_KEY; + } else { + process.env.KEENABLE_API_KEY = originalApiKey; + } + if (originalApiUrl == null) { + delete process.env.KEENABLE_API_URL; + } else { + process.env.KEENABLE_API_URL = originalApiUrl; + } + } + }); + + it('should not send a system Keenable key to a user-provided API URL', async () => { + const originalApiKey = process.env.KEENABLE_API_KEY; + const originalApiUrl = process.env.KEENABLE_API_URL; + process.env.KEENABLE_API_KEY = 'system-keenable-key'; + delete process.env.KEENABLE_API_URL; + mockLoadAuthValues.mockResolvedValue({ + KEENABLE_API_KEY: 'system-keenable-key', + KEENABLE_API_URL: 'https://user-keenable.example.com', + }); + + try { + const result = await loadWebSearchAuth({ + userId, + webSearchConfig: { + keenableApiKey: '${KEENABLE_API_KEY}', + keenableApiUrl: '${KEENABLE_API_URL}', + searchProvider: 'keenable' as SearchProviders, + scraperProvider: 'keenable' as ScraperProviders, + rerankerType: 'none' as RerankerTypes, + safeSearch: SafeSearchTypes.MODERATE, + } as TWebSearchConfig, + loadAuthValues: mockLoadAuthValues, + }); + + expect(result.authResult.keenableApiUrl).toBe('https://user-keenable.example.com'); + expect(result.authResult.keenableApiKey).toBeUndefined(); + expect(result.authTypes).toEqual([ + ['providers', AuthType.USER_PROVIDED], + ['scrapers', AuthType.USER_PROVIDED], + ['rerankers', AuthType.SYSTEM_DEFINED], + ]); + } finally { + if (originalApiKey == null) { + delete process.env.KEENABLE_API_KEY; + } else { + process.env.KEENABLE_API_KEY = originalApiKey; + } + if (originalApiUrl == null) { + delete process.env.KEENABLE_API_URL; + } else { + process.env.KEENABLE_API_URL = originalApiUrl; + } + } + }); + + it('should treat a whitespace-only Keenable URL env value as user-provided', async () => { + const originalApiKey = process.env.KEENABLE_API_KEY; + const originalApiUrl = process.env.KEENABLE_API_URL; + process.env.KEENABLE_API_KEY = 'system-keenable-key'; + process.env.KEENABLE_API_URL = ' '; + mockLoadAuthValues.mockResolvedValue({ + KEENABLE_API_KEY: 'system-keenable-key', + KEENABLE_API_URL: 'https://user-keenable.example.com', + }); + + try { + const result = await loadWebSearchAuth({ + userId, + webSearchConfig: { + keenableApiKey: '${KEENABLE_API_KEY}', + keenableApiUrl: '${KEENABLE_API_URL}', + searchProvider: 'keenable' as SearchProviders, + scraperProvider: 'keenable' as ScraperProviders, + rerankerType: 'none' as RerankerTypes, + safeSearch: SafeSearchTypes.MODERATE, + } as TWebSearchConfig, + loadAuthValues: mockLoadAuthValues, + }); + + expect(result.authResult.keenableApiUrl).toBe('https://user-keenable.example.com'); + expect(result.authResult.keenableApiKey).toBeUndefined(); + } finally { + if (originalApiKey == null) { + delete process.env.KEENABLE_API_KEY; + } else { + process.env.KEENABLE_API_KEY = originalApiKey; + } + if (originalApiUrl == null) { + delete process.env.KEENABLE_API_URL; + } else { + process.env.KEENABLE_API_URL = originalApiUrl; + } + } + }); + + it('should reject selected Keenable auth when its user API URL is blocked', async () => { + mockIsSSRFTarget.mockReturnValueOnce(true); + mockLoadAuthValues.mockResolvedValue({ + KEENABLE_API_KEY: 'user-keenable-key', + KEENABLE_API_URL: 'http://127.0.0.1:8080', + }); + + const result = await loadWebSearchAuth({ + userId, + webSearchConfig: { + keenableApiKey: '${KEENABLE_API_KEY}', + keenableApiUrl: '${KEENABLE_API_URL}', + searchProvider: 'keenable' as SearchProviders, + scraperProvider: 'keenable' as ScraperProviders, + rerankerType: 'none' as RerankerTypes, + safeSearch: SafeSearchTypes.MODERATE, + } as TWebSearchConfig, + loadAuthValues: mockLoadAuthValues, + }); + + expect(result.authenticated).toBe(false); + expect(result.authResult.searchProvider).toBe(SearchProviders.KEENABLE); + expect(result.authResult.scraperProvider).toBe(ScraperProviders.KEENABLE); + expect(result.authResult.keenableApiKey).toBeUndefined(); + expect(result.authResult.keenableApiUrl).toBeUndefined(); + }); + + it('should authenticate a fully keyless Keenable stack (search + scrape + no reranker)', async () => { + // Nothing is authenticated anywhere: every field resolves empty. + mockLoadAuthValues.mockImplementation(() => Promise.resolve({})); + + const keenableConfig = { + keenableApiKey: '${KEENABLE_API_KEY}', + keenableApiUrl: '${KEENABLE_API_URL}', + firecrawlApiKey: '${FIRECRAWL_API_KEY}', + jinaApiKey: '${JINA_API_KEY}', + searchProvider: 'keenable' as SearchProviders, + scraperProvider: 'keenable' as ScraperProviders, + rerankerType: 'none' as RerankerTypes, + safeSearch: SafeSearchTypes.MODERATE, + } as TWebSearchConfig; + + const result = await loadWebSearchAuth({ + userId, + webSearchConfig: keenableConfig, + loadAuthValues: mockLoadAuthValues, + }); + + expect(result.authenticated).toBe(true); + expect(result.authResult.searchProvider).toBe('keenable' as SearchProviders); + expect(result.authResult.scraperProvider).toBe('keenable' as ScraperProviders); + expect(result.authResult.rerankerType).toBe('none' as RerankerTypes); + // Keyless: no credential ends up on the result. + expect(result.authResult.keenableApiKey).toBeUndefined(); + expect(result.authTypes).toEqual([ + ['providers', AuthType.USER_PROVIDED], + ['scrapers', AuthType.USER_PROVIDED], + ['rerankers', AuthType.SYSTEM_DEFINED], + ]); + }); + + it('should fail closed when optional Keenable credential lookup fails', async () => { + mockLoadAuthValues.mockImplementation(({ authFields, failOnOptionalError }) => { + if (failOnOptionalError && authFields.includes('KEENABLE_API_URL')) { + return Promise.reject(new Error('Credential store unavailable')); + } + return Promise.resolve({}); + }); + + const result = await loadWebSearchAuth({ + userId, + webSearchConfig: { + keenableApiKey: '${KEENABLE_API_KEY}', + keenableApiUrl: '${KEENABLE_API_URL}', + searchProvider: SearchProviders.KEENABLE, + scraperProvider: ScraperProviders.KEENABLE, + rerankerType: RerankerTypes.NONE, + safeSearch: SafeSearchTypes.MODERATE, + } as TWebSearchConfig, + loadAuthValues: mockLoadAuthValues, + }); + + expect(result.authenticated).toBe(false); + expect(result.authResult.keenableApiKey).toBeUndefined(); + expect(result.authResult.keenableApiUrl).toBeUndefined(); + expect(mockLoadAuthValues).toHaveBeenCalledWith( + expect.objectContaining({ + authFields: ['KEENABLE_API_KEY', 'KEENABLE_API_URL'], + throwError: true, + failOnOptionalError: true, + }), + ); + }); + + it('should restore a fully keyless Keenable stack from per-user selections', async () => { + mockLoadAuthValues.mockImplementation(({ authFields }) => { + const result: Record = {}; + authFields.forEach((field: string) => { + if (field === webSearchSelectionFields.selectedProvider) { + result[field] = 'keenable'; + } else if (field === webSearchSelectionFields.selectedScraper) { + result[field] = 'keenable'; + } else if (field === webSearchSelectionFields.selectedReranker) { + result[field] = 'none'; + } + }); + return Promise.resolve(result); + }); + + const result = await loadWebSearchAuth({ + userId, + webSearchConfig: { + keenableApiKey: '${KEENABLE_API_KEY}', + keenableApiUrl: '${KEENABLE_API_URL}', + safeSearch: SafeSearchTypes.MODERATE, + } as TWebSearchConfig, + loadAuthValues: mockLoadAuthValues, + }); + + expect(result.authenticated).toBe(true); + expect(result.authResult.searchProvider).toBe('keenable'); + expect(result.authResult.scraperProvider).toBe('keenable'); + expect(result.authResult.rerankerType).toBe('none'); + expect(result.authTypes.every(([, authType]) => authType === AuthType.USER_PROVIDED)).toBe( + true, + ); + }); + + it('should fail closed when persisted-selection lookup fails', async () => { + mockLoadAuthValues.mockImplementation(({ authFields, failOnOptionalError }) => { + if (failOnOptionalError && authFields.includes(webSearchSelectionFields.selectedProvider)) { + return Promise.reject(new Error('Credential store unavailable')); + } + return Promise.resolve( + Object.fromEntries(authFields.map((field: string) => [field, 'system-value'])), + ); + }); + + const result = await loadWebSearchAuth({ + userId, + webSearchConfig: { + serperApiKey: '${SERPER_API_KEY}', + firecrawlApiKey: '${FIRECRAWL_API_KEY}', + jinaApiKey: '${JINA_API_KEY}', + safeSearch: SafeSearchTypes.MODERATE, + } as TWebSearchConfig, + loadAuthValues: mockLoadAuthValues, + }); + + expect(result.authenticated).toBe(false); + expect(result.authResult.searchProvider).toBeUndefined(); + expect(result.authResult.scraperProvider).toBeUndefined(); + expect(result.authResult.rerankerType).toBeUndefined(); + expect(mockLoadAuthValues).toHaveBeenCalledWith( + expect.objectContaining({ + authFields: [ + webSearchSelectionFields.selectedProvider, + webSearchSelectionFields.selectedScraper, + webSearchSelectionFields.selectedReranker, + ], + throwError: true, + failOnOptionalError: true, + }), + ); + }); + + it('should return a persisted provider selection when its credential is missing', async () => { + mockLoadAuthValues.mockImplementation(({ authFields }) => { + if (authFields.includes(webSearchSelectionFields.selectedProvider)) { + return Promise.resolve({ + [webSearchSelectionFields.selectedProvider]: SearchProviders.TAVILY, + }); + } + return Promise.resolve({}); + }); + + const result = await loadWebSearchAuth({ + userId, + webSearchConfig: { + tavilyApiKey: '${TAVILY_API_KEY}', + firecrawlApiKey: '${FIRECRAWL_API_KEY}', + safeSearch: SafeSearchTypes.MODERATE, + } as TWebSearchConfig, + loadAuthValues: mockLoadAuthValues, + }); + + expect(result.authenticated).toBe(false); + expect(result.authResult.searchProvider).toBe(SearchProviders.TAVILY); + expect(result.authTypes).toContainEqual(['providers', AuthType.USER_PROVIDED]); + }); + + it('should pass Keenable scraper options through and honor their timeout', async () => { + mockLoadAuthValues.mockImplementation(() => Promise.resolve({})); + + const keenableConfig = { + keenableApiKey: '${KEENABLE_API_KEY}', + searchProvider: 'keenable' as SearchProviders, + scraperProvider: 'keenable' as ScraperProviders, + rerankerType: 'none' as RerankerTypes, + keenableScraperOptions: { timeout: 12000, attributionTitle: 'LibreChat' }, + safeSearch: SafeSearchTypes.MODERATE, + } as TWebSearchConfig; + + const result = await loadWebSearchAuth({ + userId, + webSearchConfig: keenableConfig, + loadAuthValues: mockLoadAuthValues, + }); + + expect(result.authResult.keenableScraperOptions).toEqual({ + timeout: 12000, + attributionTitle: 'LibreChat', + }); + expect(result.authResult.scraperTimeout).toBe(12000); + }); + + it('should select Keenable when a key is supplied but no searchProvider is pinned', async () => { + // The API-key dialog can submit credentials but cannot pin a provider, and + // Keenable has no required auth field, so the generic loop would skip it. + mockLoadAuthValues.mockImplementation(({ authFields }) => { + const result: Record = {}; + authFields.forEach((field: string) => { + if (field === 'KEENABLE_API_KEY') { + result[field] = 'user-keenable-key'; + } + }); + return Promise.resolve(result); + }); + + const unpinnedConfig = { + serperApiKey: '${SERPER_API_KEY}', + firecrawlApiKey: '${FIRECRAWL_API_KEY}', + keenableApiKey: '${KEENABLE_API_KEY}', + jinaApiKey: '${JINA_API_KEY}', + safeSearch: SafeSearchTypes.MODERATE, + } as TWebSearchConfig; + + const result = await loadWebSearchAuth({ + userId, + webSearchConfig: unpinnedConfig, + loadAuthValues: mockLoadAuthValues, + }); + + expect(result.authenticated).toBe(true); + expect(result.authResult.searchProvider).toBe('keenable' as SearchProviders); + expect(result.authResult.keenableApiKey).toBe('user-keenable-key'); + // The scraper completes the Keenable stack, since none of the keyed + // scrapers authenticated and no scraperProvider was pinned. + expect(result.authResult.scraperProvider).toBe('keenable' as ScraperProviders); + expect(result.authResult.rerankerType).toBe('none' as RerankerTypes); + }); + + it('should not select Keenable when nothing is configured for it', async () => { + // Guard against the keyless fallback hijacking installs that configured no + // provider at all: without a Keenable value, the category stays unauthenticated. + mockLoadAuthValues.mockImplementation(() => Promise.resolve({})); + + const emptyConfig = { + serperApiKey: '${SERPER_API_KEY}', + firecrawlApiKey: '${FIRECRAWL_API_KEY}', + keenableApiKey: '${KEENABLE_API_KEY}', + jinaApiKey: '${JINA_API_KEY}', + safeSearch: SafeSearchTypes.MODERATE, + } as TWebSearchConfig; + + const result = await loadWebSearchAuth({ + userId, + webSearchConfig: emptyConfig, + loadAuthValues: mockLoadAuthValues, + }); + + expect(result.authenticated).toBe(false); + expect(result.authResult.searchProvider).toBeUndefined(); + expect(result.authResult.scraperProvider).toBeUndefined(); + }); + + it('should scrape with Keenable for another search provider when a key is supplied', async () => { + // SearXNG stays the search provider while the user submits a Keenable key + // for the scraper. The dialog cannot pin `scraperProvider`, so a supplied + // Keenable value is the only signal that Keenable was chosen here. + mockLoadAuthValues.mockImplementation(({ authFields }) => { + const result: Record = {}; + authFields.forEach((field: string) => { + if (field === 'SEARXNG_INSTANCE_URL') { + result[field] = 'https://searx.example.com'; + } + if (field === 'KEENABLE_API_KEY') { + result[field] = 'user-keenable-key'; + } + }); + return Promise.resolve(result); + }); + + const scraperOnlyConfig = { + searxngInstanceUrl: '${SEARXNG_INSTANCE_URL}', + firecrawlApiKey: '${FIRECRAWL_API_KEY}', + keenableApiKey: '${KEENABLE_API_KEY}', + searchProvider: 'searxng' as SearchProviders, + rerankerType: 'none' as RerankerTypes, + safeSearch: SafeSearchTypes.MODERATE, + } as TWebSearchConfig; + + const result = await loadWebSearchAuth({ + userId, + webSearchConfig: scraperOnlyConfig, + loadAuthValues: mockLoadAuthValues, + }); + + expect(result.authResult.searchProvider).toBe('searxng' as SearchProviders); + expect(result.authResult.scraperProvider).toBe('keenable' as ScraperProviders); + expect(result.authResult.keenableApiKey).toBe('user-keenable-key'); + expect(result.authenticated).toBe(true); + }); + + it('should ignore a saved Keenable search URL for scraper-only Keenable', async () => { + const originalApiKey = process.env.KEENABLE_API_KEY; + const originalSerperKey = process.env.SERPER_API_KEY; + process.env.KEENABLE_API_KEY = 'system-keenable-key'; + process.env.SERPER_API_KEY = 'system-serper-key'; + mockIsSSRFTarget.mockReturnValue(true); + mockLoadAuthValues.mockImplementation(({ authFields }) => { + const result: Record = {}; + for (const field of authFields) { + if (field === 'KEENABLE_API_KEY') { + result[field] = 'system-keenable-key'; + } else if (field === 'KEENABLE_API_URL') { + result[field] = 'http://127.0.0.1:8080'; + } else if (field === 'SERPER_API_KEY') { + result[field] = 'system-serper-key'; + } + } + return Promise.resolve(result); + }); + + try { + const result = await loadWebSearchAuth({ + userId, + webSearchConfig: { + serperApiKey: '${SERPER_API_KEY}', + keenableApiKey: '${KEENABLE_API_KEY}', + keenableApiUrl: '${KEENABLE_API_URL}', + searchProvider: SearchProviders.SERPER, + scraperProvider: ScraperProviders.KEENABLE, + rerankerType: RerankerTypes.NONE, + safeSearch: SafeSearchTypes.MODERATE, + } as TWebSearchConfig, + loadAuthValues: mockLoadAuthValues, + }); + + expect(result.authenticated).toBe(true); + expect(result.authResult.searchProvider).toBe(SearchProviders.SERPER); + expect(result.authResult.scraperProvider).toBe(ScraperProviders.KEENABLE); + expect(result.authResult.keenableApiKey).toBe('system-keenable-key'); + expect(result.authResult.keenableApiUrl).toBeUndefined(); + expect(mockLoadAuthValues).toHaveBeenCalledWith( + expect.objectContaining({ authFields: ['KEENABLE_API_KEY'] }), + ); + expect(mockLoadAuthValues).not.toHaveBeenCalledWith( + expect.objectContaining({ authFields: ['KEENABLE_API_KEY', 'KEENABLE_API_URL'] }), + ); + expect(mockIsSSRFTarget).not.toHaveBeenCalled(); + } finally { + mockIsSSRFTarget.mockReturnValue(false); + if (originalApiKey == null) { + delete process.env.KEENABLE_API_KEY; + } else { + process.env.KEENABLE_API_KEY = originalApiKey; + } + if (originalSerperKey == null) { + delete process.env.SERPER_API_KEY; + } else { + process.env.SERPER_API_KEY = originalSerperKey; + } + } + }); + + it('should not scrape with Keenable for another search provider without a Keenable value', async () => { + // SearXNG authenticates as the provider (it is not also a scraper); with no + // scraper key configured and no Keenable value the scrapers category must + // stay unauthenticated rather than fall back to Keenable. + mockLoadAuthValues.mockImplementation(({ authFields }) => { + const result: Record = {}; + authFields.forEach((field: string) => { + if (field === 'SEARXNG_INSTANCE_URL') { + result[field] = 'https://searx.example.com'; + } + }); + return Promise.resolve(result); + }); + + const mixedConfig = { + searxngInstanceUrl: '${SEARXNG_INSTANCE_URL}', + firecrawlApiKey: '${FIRECRAWL_API_KEY}', + keenableApiKey: '${KEENABLE_API_KEY}', + searchProvider: 'searxng' as SearchProviders, + rerankerType: 'none' as RerankerTypes, + safeSearch: SafeSearchTypes.MODERATE, + } as TWebSearchConfig; + + const result = await loadWebSearchAuth({ + userId, + webSearchConfig: mixedConfig, + loadAuthValues: mockLoadAuthValues, + }); + + expect(result.authResult.searchProvider).toBe('searxng' as SearchProviders); + expect(result.authResult.scraperProvider).toBeUndefined(); + expect(result.authenticated).toBe(false); + }); + it('should set the correct service types in authResult', async () => { // Mock successful authentication mockLoadAuthValues.mockImplementation(({ authFields }) => { diff --git a/packages/api/src/web/web.ts b/packages/api/src/web/web.ts index 3933936518..b4ba2949ad 100644 --- a/packages/api/src/web/web.ts +++ b/packages/api/src/web/web.ts @@ -1,14 +1,15 @@ -import { webSearchAuth } from '@librechat/data-schemas'; +import { webSearchAuth, webSearchKeys, webSearchSelectionFields } from '@librechat/data-schemas'; import { AuthType, + RerankerTypes, SafeSearchTypes, SearchCategories, SearchProviders, ScraperProviders, extractVariableName, } from 'librechat-data-provider'; -import type { RerankerTypes, TCustomConfig, TWebSearchConfig } from 'librechat-data-provider'; import type { TWebSearchKeys, TWebSearchCategories } from '@librechat/data-schemas'; +import type { TCustomConfig, TWebSearchConfig } from 'librechat-data-provider'; import { isSSRFTarget, resolveHostnameSSRF, getEffectivePort } from '../auth'; /** @@ -28,6 +29,10 @@ const USER_PROVIDED_OPT_IN_URL_KEYS = new Set([ 'tavilyExtractUrl', ]); +const SEARCH_PROVIDER_VALUES = new Set(Object.values(SearchProviders)); +const SCRAPER_PROVIDER_VALUES = new Set(Object.values(ScraperProviders)); +const RERANKER_VALUES = new Set(Object.values(RerankerTypes)); + function isUserProvidedEnabled(field: string): boolean { return process.env[field] === AuthType.USER_PROVIDED; } @@ -82,6 +87,59 @@ export function extractWebSearchEnvVars({ return authFields; } +function mapWebSearchSelection(key: string, value: string): [string, string] | undefined { + if (key === 'selectedProvider' && SEARCH_PROVIDER_VALUES.has(value)) { + return [webSearchSelectionFields.selectedProvider, value]; + } + if (key === 'selectedScraper' && SCRAPER_PROVIDER_VALUES.has(value)) { + return [webSearchSelectionFields.selectedScraper, value]; + } + if (key === 'selectedReranker' && RERANKER_VALUES.has(value)) { + return [webSearchSelectionFields.selectedReranker, value]; + } + return undefined; +} + +export function getWebSearchInstallEntries({ + auth, + config, +}: { + auth: Partial>; + config: TCustomConfig['webSearch']; +}): [string, string][] { + const credentialEntries: [string, string][] = []; + const selectionEntries: [string, string][] = []; + + for (const [key, value] of Object.entries(auth)) { + if (typeof value !== 'string') { + continue; + } + + const selection = mapWebSearchSelection(key, value); + if (selection) { + selectionEntries.push(selection); + continue; + } + + const [authField] = extractWebSearchEnvVars({ + keys: [key as TWebSearchKeys], + config, + }); + if (authField) { + credentialEntries.push([authField, value]); + } + } + + return [...credentialEntries, ...selectionEntries]; +} + +export function getWebSearchUninstallFields(config: TCustomConfig['webSearch']): string[] { + return [ + ...extractWebSearchEnvVars({ keys: webSearchKeys, config }), + ...Object.values(webSearchSelectionFields), + ]; +} + /** * Type for web search authentication result */ @@ -94,6 +152,13 @@ export interface WebSearchAuthResult { authResult: Partial; } +interface KeenableAuthResolution { + isUserProvided: boolean; + hasSystemApiKey: boolean; + rejectedUserApiUrl: boolean; + lookupFailed: boolean; +} + /** * Loads and verifies web search authentication values * @param params - Authentication parameters @@ -112,12 +177,194 @@ export async function loadWebSearchAuth({ authFields: string[]; optional?: Set; throwError?: boolean; + failOnOptionalError?: boolean; }) => Promise>; throwError?: boolean; }): Promise { let authenticated = true; const authResult: Partial = {}; + /** + * Keenable is keyless by default: both its search and its page fetch work + * against public endpoints with no key, so neither category needs a secret to + * authenticate. This resolves the optional key/URL overrides once (a key only + * lifts rate limits) and reports whether any of them came from the user. + */ + const keenableAuth = new Map>(); + function resolveKeenableAuth(includeApiUrl: boolean): Promise { + const cached = keenableAuth.get(includeApiUrl); + if (cached) { + return cached; + } + + const resolution = (async () => { + let keenableUserProvided = false; + let hasSystemApiKey = false; + let rejectedUserApiUrl = false; + const keenableKeys: TWebSearchKeys[] = includeApiUrl + ? ['keenableApiKey', 'keenableApiUrl'] + : ['keenableApiKey']; + const authEntries: Array<{ key: TWebSearchKeys; field: string }> = []; + + for (const originalKey of keenableKeys) { + const [field] = extractWebSearchEnvVars({ + keys: [originalKey], + config: webSearchConfig, + }); + if (field) { + authEntries.push({ key: originalKey, field }); + } + } + + let authValues: Record = {}; + try { + const authFields = authEntries.map(({ field }) => field); + authValues = await loadAuthValues({ + userId, + authFields, + optional: new Set(authFields), + throwError: true, + failOnOptionalError: true, + }); + } catch { + return { + isUserProvided: false, + hasSystemApiKey: false, + rejectedUserApiUrl: false, + lookupFailed: true, + }; + } + + const resolvedEntries: Array<{ + key: TWebSearchKeys; + value: string; + isFieldUserProvided: boolean; + }> = []; + for (const { key: originalKey, field } of authEntries) { + const value = authValues[field]; + const envValue = process.env[field]; + const normalizedEnvValue = envValue?.trim(); + const isFieldUserProvided = + normalizedEnvValue == null || + normalizedEnvValue === '' || + normalizedEnvValue === AuthType.USER_PROVIDED; + if (isFieldUserProvided) { + // The category stays editable even before the user saves a value. + // Otherwise a system key would hide a separately user-provided URL. + keenableUserProvided = true; + } + if (!value) { + continue; + } + if ( + originalKey === 'keenableApiUrl' && + isFieldUserProvided && + (await isSSRFUrl(value, webSearchConfig?.allowedAddresses)) + ) { + rejectedUserApiUrl = true; + continue; + } + resolvedEntries.push({ key: originalKey, value, isFieldUserProvided }); + } + + if (rejectedUserApiUrl) { + return { + isUserProvided: true, + hasSystemApiKey: false, + rejectedUserApiUrl: true, + lookupFailed: false, + }; + } + + const hasUserProvidedApiUrl = resolvedEntries.some( + ({ key, isFieldUserProvided }) => key === 'keenableApiUrl' && isFieldUserProvided, + ); + for (const { key: originalKey, value, isFieldUserProvided } of resolvedEntries) { + // Never forward an administrator's secret to an endpoint controlled by + // the user. Keenable remains functional without the key at that URL. + if (originalKey === 'keenableApiKey' && !isFieldUserProvided && hasUserProvidedApiUrl) { + continue; + } + authResult[originalKey] = value; + if (originalKey === 'keenableApiKey' && !isFieldUserProvided) { + hasSystemApiKey = true; + } + } + + return { + isUserProvided: keenableUserProvided, + hasSystemApiKey, + rejectedUserApiUrl: false, + lookupFailed: false, + }; + })(); + keenableAuth.set(includeApiUrl, resolution); + return resolution; + } + + let userSelections: + | Promise<{ + searchProvider?: SearchProviders; + scraperProvider?: ScraperProviders; + rerankerType?: RerankerTypes; + lookupFailed?: boolean; + }> + | undefined; + function resolveUserSelections(): Promise<{ + searchProvider?: SearchProviders; + scraperProvider?: ScraperProviders; + rerankerType?: RerankerTypes; + lookupFailed?: boolean; + }> { + userSelections ??= (async () => { + const fields: string[] = []; + if (!webSearchConfig?.searchProvider) { + fields.push(webSearchSelectionFields.selectedProvider); + } + if (!webSearchConfig?.scraperProvider) { + fields.push(webSearchSelectionFields.selectedScraper); + } + if (!webSearchConfig?.rerankerType) { + fields.push(webSearchSelectionFields.selectedReranker); + } + if (fields.length === 0) { + return {}; + } + + let values: Record; + try { + values = await loadAuthValues({ + userId, + authFields: fields, + optional: new Set(fields), + throwError: true, + failOnOptionalError: true, + }); + } catch { + return { lookupFailed: true }; + } + + const searchProvider = values[webSearchSelectionFields.selectedProvider]; + const scraperProvider = values[webSearchSelectionFields.selectedScraper]; + const rerankerType = values[webSearchSelectionFields.selectedReranker]; + return { + searchProvider: + searchProvider != null && SEARCH_PROVIDER_VALUES.has(searchProvider) + ? (searchProvider as SearchProviders) + : undefined, + scraperProvider: + scraperProvider != null && SCRAPER_PROVIDER_VALUES.has(scraperProvider) + ? (scraperProvider as ScraperProviders) + : undefined, + rerankerType: + rerankerType != null && RERANKER_VALUES.has(rerankerType) + ? (rerankerType as RerankerTypes) + : undefined, + }; + })(); + return userSelections; + } + /** Type-safe iterator for the category-service combinations */ async function checkAuth( category: C, @@ -133,12 +380,54 @@ export async function loadWebSearchAuth({ specificService = webSearchConfig.scraperProvider as unknown as ServiceType; } else if (category === SearchCategories.RERANKERS && webSearchConfig?.rerankerType) { specificService = webSearchConfig.rerankerType as unknown as ServiceType; + } - // Special case: skipping the reranker means skipping auth as well - if (specificService === 'none') { - authResult.rerankerType = specificService as RerankerTypes; - return [true, false]; + if (!specificService) { + const selections = await resolveUserSelections(); + if (selections.lookupFailed) { + return [false, true]; } + if (category === SearchCategories.PROVIDERS && selections.searchProvider) { + specificService = selections.searchProvider as unknown as ServiceType; + authResult.searchProvider = selections.searchProvider; + isUserProvided = true; + } else if (category === SearchCategories.SCRAPERS && selections.scraperProvider) { + specificService = selections.scraperProvider as unknown as ServiceType; + authResult.scraperProvider = selections.scraperProvider; + isUserProvided = true; + } else if (category === SearchCategories.RERANKERS && selections.rerankerType) { + specificService = selections.rerankerType as unknown as ServiceType; + authResult.rerankerType = selections.rerankerType; + isUserProvided = true; + } + } + + if (category === SearchCategories.RERANKERS && specificService === 'none') { + authResult.rerankerType = specificService as RerankerTypes; + return [true, isUserProvided]; + } + + // Special case: Keenable is keyless by default. The public endpoints need no + // key, so a pinned Keenable authenticates even when nothing is configured — + // as a search provider and as a scraper alike. + if (category === SearchCategories.PROVIDERS && specificService === SearchProviders.KEENABLE) { + const keenable = await resolveKeenableAuth(true); + authResult.searchProvider = SearchProviders.KEENABLE; + if (keenable.lookupFailed || keenable.rejectedUserApiUrl) { + return [false, true]; + } + return [true, isUserProvided || keenable.isUserProvided || !keenable.hasSystemApiKey]; + } + if (category === SearchCategories.SCRAPERS && specificService === ScraperProviders.KEENABLE) { + const searchUsesKeenable = + authResult.searchProvider === SearchProviders.KEENABLE || + webSearchConfig?.searchProvider === SearchProviders.KEENABLE; + const keenable = await resolveKeenableAuth(searchUsesKeenable); + authResult.scraperProvider = ScraperProviders.KEENABLE; + if (keenable.lookupFailed || keenable.rejectedUserApiUrl) { + return [false, true]; + } + return [true, isUserProvided || keenable.isUserProvided || !keenable.hasSystemApiKey]; } // If a specific service is specified, only check that one @@ -255,10 +544,61 @@ export async function loadWebSearchAuth({ continue; } } - if (category === SearchCategories.RERANKERS && !webSearchConfig?.rerankerType) { + if ( + category === SearchCategories.RERANKERS && + !webSearchConfig?.rerankerType && + !specificService + ) { authResult.rerankerType = 'none' as RerankerTypes; return [true, false]; } + + /** + * Keyless fallback, reached only when no keyed service in the category + * authenticated. The loop above skips Keenable whenever it isn't pinned, + * because none of its auth fields are required (`requiredKeys.length === 0`), + * so a legacy Keenable credential saved before provider selections were + * persisted would otherwise leave the category unauthenticated. Gating the + * compatibility path on an actual Keenable value preserves those installs + * without making Keenable the implicit default for new users. + */ + if (category === SearchCategories.PROVIDERS && !specificService) { + const keenable = await resolveKeenableAuth(true); + if (keenable.lookupFailed || keenable.rejectedUserApiUrl) { + return [false, true]; + } + if (authResult.keenableApiKey || authResult.keenableApiUrl) { + authResult.searchProvider = SearchProviders.KEENABLE; + return [true, keenable.isUserProvided]; + } + } + + /** + * Same for the scraper. Reached only when no keyed scraper authenticated, so + * the alternative is web search staying disabled entirely. Two triggers: + * Keenable being the resolved search provider (providers are checked first), + * which is what completes a fully keyless stack, or a Keenable value actually + * being present, which is how the API-key dialog expresses "scrape with + * Keenable" for a deployment whose search runs on someone else. With neither, + * it stays unauthenticated rather than silently scraping for a provider that + * was never told to use Keenable. + */ + if (category === SearchCategories.SCRAPERS && !specificService) { + const searchUsesKeenable = authResult.searchProvider === SearchProviders.KEENABLE; + const keenable = await resolveKeenableAuth(searchUsesKeenable); + if (keenable.lookupFailed || keenable.rejectedUserApiUrl) { + return [false, true]; + } + if ( + authResult.searchProvider === SearchProviders.KEENABLE || + authResult.keenableApiKey || + authResult.keenableApiUrl + ) { + authResult.scraperProvider = ScraperProviders.KEENABLE; + return [true, keenable.isUserProvided]; + } + } + return [false, isUserProvided]; } @@ -285,6 +625,8 @@ export async function loadWebSearchAuth({ scraperOptionsTimeout = webSearchConfig?.tavilyScraperOptions?.timeout; } else if (scraperProvider === ScraperProviders.FIRECRAWL) { scraperOptionsTimeout = webSearchConfig?.firecrawlOptions?.timeout; + } else if (scraperProvider === ScraperProviders.KEENABLE) { + scraperOptionsTimeout = webSearchConfig?.keenableScraperOptions?.timeout; } const searchProvider = authResult.searchProvider ?? webSearchConfig?.searchProvider; @@ -296,6 +638,8 @@ export async function loadWebSearchAuth({ authResult.searxngSearchOptions = webSearchConfig?.searxngSearchOptions; authResult.tavilySearchOptions = webSearchConfig?.tavilySearchOptions; authResult.tavilyScraperOptions = webSearchConfig?.tavilyScraperOptions; + authResult.keenableSearchOptions = webSearchConfig?.keenableSearchOptions; + authResult.keenableScraperOptions = webSearchConfig?.keenableScraperOptions; return { authTypes, diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts index a508935916..be69d24140 100644 --- a/packages/data-provider/src/config.spec.ts +++ b/packages/data-provider/src/config.spec.ts @@ -737,6 +737,66 @@ describe('webSearchSchema', () => { ).toThrow(); }); + it('accepts Keenable search options', () => { + const result = webSearchSchema.parse({ + keenableSearchOptions: { + maxResults: 7, + site: 'example.com', + attributionTitle: 'LibreChat', + timeout: 15000, + }, + }); + + expect(result.keenableSearchOptions?.maxResults).toBe(7); + expect(result.keenableSearchOptions?.site).toBe('example.com'); + expect(result.keenableSearchOptions?.attributionTitle).toBe('LibreChat'); + expect(result.keenableSearchOptions?.timeout).toBe(15000); + }); + + it('rejects invalid Keenable search options', () => { + expect(() => + webSearchSchema.parse({ + keenableSearchOptions: { + maxResults: 0, + }, + }), + ).toThrow(); + + expect(() => + webSearchSchema.parse({ + keenableSearchOptions: { + timeout: 120001, + }, + }), + ).toThrow(); + }); + + it('accepts Keenable as a scraper provider with its options', () => { + const result = webSearchSchema.parse({ + searchProvider: 'keenable', + scraperProvider: 'keenable', + rerankerType: 'none', + keenableScraperOptions: { + attributionTitle: 'LibreChat', + timeout: 15000, + }, + }); + + expect(result.scraperProvider).toBe('keenable'); + expect(result.keenableScraperOptions?.attributionTitle).toBe('LibreChat'); + expect(result.keenableScraperOptions?.timeout).toBe(15000); + }); + + it('rejects invalid Keenable scraper options', () => { + expect(() => + webSearchSchema.parse({ + keenableScraperOptions: { + timeout: 120001, + }, + }), + ).toThrow(); + }); + it('accepts SearXNG search options', () => { const result = webSearchSchema.parse({ searxngSearchOptions: { diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 1d517f5889..bb35b4e0f7 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -1887,12 +1887,14 @@ export enum SearchProviders { SERPER = 'serper', SEARXNG = 'searxng', TAVILY = 'tavily', + KEENABLE = 'keenable', } export enum ScraperProviders { FIRECRAWL = 'firecrawl', SERPER = 'serper', TAVILY = 'tavily', + KEENABLE = 'keenable', } export enum RerankerTypes { @@ -1938,6 +1940,8 @@ export const webSearchSchema = z.object({ tavilyApiKeyPreview: apiKeyPreviewSchema, tavilySearchUrl: z.string().optional().default('${TAVILY_SEARCH_URL}'), tavilyExtractUrl: z.string().optional().default('${TAVILY_EXTRACT_URL}'), + keenableApiKey: z.string().optional().default('${KEENABLE_API_KEY}'), + keenableApiUrl: z.string().optional().default('${KEENABLE_API_URL}'), jinaApiKey: z.string().optional().default('${JINA_API_KEY}'), jinaApiKeyPreview: apiKeyPreviewSchema, jinaApiUrl: z.string().optional().default('${JINA_API_URL}'), @@ -2019,6 +2023,20 @@ export const webSearchSchema = z.object({ timeout: z.number().int().nonnegative().max(120000).optional(), }) .optional(), + keenableSearchOptions: z + .object({ + maxResults: z.number().int().min(1).max(20).optional(), + site: z.string().optional(), + attributionTitle: z.string().optional(), + timeout: z.number().int().nonnegative().max(120000).optional(), + }) + .optional(), + keenableScraperOptions: z + .object({ + attributionTitle: z.string().optional(), + timeout: z.number().int().nonnegative().max(120000).optional(), + }) + .optional(), }); export type TWebSearchConfig = DeepPartial>; diff --git a/packages/data-provider/src/types/queries.ts b/packages/data-provider/src/types/queries.ts index f043fe88b8..8443897c1b 100644 --- a/packages/data-provider/src/types/queries.ts +++ b/packages/data-provider/src/types/queries.ts @@ -1,4 +1,5 @@ import type { InfiniteData } from '@tanstack/react-query'; +import type { RerankerTypes, SearchProviders, ScraperProviders } from '../config'; import type * as p from '../accessPermissions'; import type * as a from '../types/agents'; import type * as s from '../schemas'; @@ -152,6 +153,9 @@ export type VerifyToolAuthResponse = { authenticated: boolean; message?: string | s.AuthType; authTypes?: [string, s.AuthType][]; + searchProvider?: SearchProviders; + scraperProvider?: ScraperProviders; + rerankerType?: RerankerTypes; }; export type GetToolCallParams = { conversationId: string }; diff --git a/packages/data-provider/src/types/web.ts b/packages/data-provider/src/types/web.ts index fac46d2d89..117dce4415 100644 --- a/packages/data-provider/src/types/web.ts +++ b/packages/data-provider/src/types/web.ts @@ -12,8 +12,8 @@ export enum DATE_RANGE { PAST_YEAR = 'y', } -export type SearchProvider = 'serper' | 'searxng' | 'tavily'; -export type ScraperProvider = 'firecrawl' | 'serper' | 'tavily'; +export type SearchProvider = 'serper' | 'searxng' | 'tavily' | 'keenable'; +export type ScraperProvider = 'firecrawl' | 'serper' | 'tavily' | 'keenable'; export type RerankerType = 'infinity' | 'jina' | 'cohere' | 'none'; export interface Highlight { @@ -80,6 +80,9 @@ export interface SearchConfig { tavilyApiKey?: string; tavilySearchUrl?: string; tavilySearchOptions?: TavilyConfig['tavilySearchOptions']; + keenableApiKey?: string; + keenableApiUrl?: string; + keenableSearchOptions?: KeenableConfig['keenableSearchOptions']; } export type References = { @@ -144,6 +147,13 @@ export interface TavilyConfig { tavilyScraperOptions?: z.infer['tavilyScraperOptions']; } +export interface KeenableConfig { + keenableApiKey?: string; + keenableApiUrl?: string; + keenableSearchOptions?: z.infer['keenableSearchOptions']; + keenableScraperOptions?: z.infer['keenableScraperOptions']; +} + export interface ScraperContentResult { content: string; } diff --git a/packages/data-schemas/src/app/web.spec.ts b/packages/data-schemas/src/app/web.spec.ts index 0e0e406cd1..fd073ca0ef 100644 --- a/packages/data-schemas/src/app/web.spec.ts +++ b/packages/data-schemas/src/app/web.spec.ts @@ -1,6 +1,6 @@ import { SafeSearchTypes, SearchProviders, ScraperProviders } from 'librechat-data-provider'; import type { TCustomConfig } from 'librechat-data-provider'; -import { loadWebSearchConfig } from './web'; +import { loadWebSearchConfig, webSearchAuth, getWebSearchKeys } from './web'; describe('loadWebSearchConfig', () => { describe('firecrawlVersion', () => { @@ -60,6 +60,8 @@ describe('loadWebSearchConfig', () => { tavilyApiKey: '${TAVILY_API_KEY}', tavilySearchUrl: '${TAVILY_SEARCH_URL}', tavilyExtractUrl: '${TAVILY_EXTRACT_URL}', + keenableApiKey: '${KEENABLE_API_KEY}', + keenableApiUrl: '${KEENABLE_API_URL}', }); }); @@ -187,6 +189,8 @@ describe('loadWebSearchConfig', () => { expect(result?.firecrawlApiKey).toBe('${FIRECRAWL_API_KEY}'); expect(result?.jinaApiKey).toBe('${JINA_API_KEY}'); expect(result?.cohereApiKey).toBe('${COHERE_API_KEY}'); + expect(result?.keenableApiKey).toBe('${KEENABLE_API_KEY}'); + expect(result?.keenableApiUrl).toBe('${KEENABLE_API_URL}'); }); it('should preserve custom API keys', () => { @@ -228,3 +232,24 @@ describe('loadWebSearchConfig', () => { }); }); }); + +describe('webSearchAuth', () => { + it('registers Keenable in both the provider and the scraper category', () => { + expect(webSearchAuth.providers).toHaveProperty(SearchProviders.KEENABLE); + expect(webSearchAuth.scrapers).toHaveProperty(ScraperProviders.KEENABLE); + }); + + it('marks every Keenable auth field optional (keyless by default)', () => { + // A required field (1) would make Keenable unusable without a key, and the + // keyless carve-out in `loadWebSearchAuth` depends on there being none. + expect(Object.values(webSearchAuth.providers.keenable)).toEqual([0, 0]); + expect(Object.values(webSearchAuth.scrapers.keenable)).toEqual([0]); + }); + + it('exposes the Keenable keys once across categories', () => { + const keys = getWebSearchKeys(); + + expect(keys.filter((key) => key === 'keenableApiKey')).toHaveLength(1); + expect(keys).toContain('keenableApiUrl'); + }); +}); diff --git a/packages/data-schemas/src/app/web.ts b/packages/data-schemas/src/app/web.ts index 14a74f20e5..c30ace3ac4 100644 --- a/packages/data-schemas/src/app/web.ts +++ b/packages/data-schemas/src/app/web.ts @@ -16,6 +16,11 @@ export const webSearchAuth = { tavilyApiKey: 1 as const, tavilySearchUrl: 0 as const, }, + keenable: { + /** Optional (0) — Keenable works keyless; a key only lifts rate limits */ + keenableApiKey: 0 as const, + keenableApiUrl: 0 as const, + }, }, scrapers: { firecrawl: { @@ -31,6 +36,12 @@ export const webSearchAuth = { tavilyApiKey: 1 as const, tavilyExtractUrl: 0 as const, }, + keenable: { + /** Optional (0) — Keenable's page fetch is keyless as well; a key only + * lifts rate limits. The fetch endpoint itself is overridden with the + * `KEENABLE_FETCH_URL` env var, not through this config. */ + keenableApiKey: 0 as const, + }, }, rerankers: { jina: { @@ -68,6 +79,12 @@ export function getWebSearchKeys(): TWebSearchKeys[] { export const webSearchKeys: TWebSearchKeys[] = getWebSearchKeys(); +export const webSearchSelectionFields = { + selectedProvider: 'LIBRECHAT_WEB_SEARCH_PROVIDER', + selectedScraper: 'LIBRECHAT_WEB_SEARCH_SCRAPER', + selectedReranker: 'LIBRECHAT_WEB_SEARCH_RERANKER', +} as const; + export function loadWebSearchConfig( config: TWebSearchConfigInput | undefined, ): TCustomConfig['webSearch'] { @@ -80,6 +97,8 @@ export function loadWebSearchConfig( const tavilyApiKey = config?.tavilyApiKey ?? '${TAVILY_API_KEY}'; const tavilySearchUrl = config?.tavilySearchUrl ?? '${TAVILY_SEARCH_URL}'; const tavilyExtractUrl = config?.tavilyExtractUrl ?? '${TAVILY_EXTRACT_URL}'; + const keenableApiKey = config?.keenableApiKey ?? '${KEENABLE_API_KEY}'; + const keenableApiUrl = config?.keenableApiUrl ?? '${KEENABLE_API_URL}'; const jinaApiKey = config?.jinaApiKey ?? '${JINA_API_KEY}'; const jinaApiUrl = config?.jinaApiUrl ?? '${JINA_API_URL}'; const cohereApiKey = config?.cohereApiKey ?? '${COHERE_API_KEY}'; @@ -102,6 +121,8 @@ export function loadWebSearchConfig( tavilyApiKey, tavilySearchUrl, tavilyExtractUrl, + keenableApiKey, + keenableApiUrl, firecrawlApiKey, firecrawlApiUrl, firecrawlVersion, diff --git a/packages/data-schemas/src/types/web.ts b/packages/data-schemas/src/types/web.ts index 69018234a3..147025cb84 100644 --- a/packages/data-schemas/src/types/web.ts +++ b/packages/data-schemas/src/types/web.ts @@ -10,6 +10,8 @@ export type TWebSearchKeys = | 'tavilyApiKey' | 'tavilySearchUrl' | 'tavilyExtractUrl' + | 'keenableApiKey' + | 'keenableApiUrl' | 'jinaApiKey' | 'jinaApiUrl' | 'cohereApiKey';