From 33e42e6d5d09635bd87d4865e87c53c9fa9a3f4d Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:36:23 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=9B=EF=B8=8F=20feat:=20Configurable=20?= =?UTF-8?q?SearXNG=20Search=20Options=20(#14987)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: configurable SearXNG search options SearXNG queries were hardcoded to google,bing,duckduckgo with no way to change the engine list, the result language, or the request timeout. Most self-hosted instances get served CAPTCHAs by DuckDuckGo, so a third of every query silently returns nothing and operators have no lever to pull. Add a searxngSearchOptions block to the webSearch config that accepts engines (as a comma-separated string or a list), language, timeRange, and timeout, and thread it through to the search tool. Engines are normalized to the comma-separated form SearXNG expects, with blank entries dropped so a stray comma cannot produce an empty engines parameter. Refs #14117 * fix: normalize SearXNG engines on the runtime config path The engines transform lived only on the zod schema, but loadCustomConfig returns the raw YAML object rather than result.data, so nothing downstream ever saw the transformed value. A YAML list reached the SDK as an array and threw "options?.engines?.trim is not a function" when the search tool was built, taking web search down entirely for the exact block the example yaml documents. An untrimmed string reached SearXNG with spaces still in it. Extract the normalization into normalizeSearxngEngines and apply it in loadWebSearchConfig as well as the schema, so both the parsed and the raw path produce the same comma-separated value. Widen the loader's parameter to TWebSearchConfigInput, which models engines as the list or string an operator actually writes, and cover the raw path with tests that call the loader rather than the schema. * chore: drop unused RerankerTypes import in web config loader --- librechat.example.yaml | 23 +++++++ packages/api/src/web/web.spec.ts | 81 ++++++++++++++++++++++ packages/api/src/web/web.ts | 1 + packages/data-provider/src/config.spec.ts | 82 +++++++++++++++++++++++ packages/data-provider/src/config.ts | 40 +++++++++++ packages/data-provider/src/types/web.ts | 1 + packages/data-schemas/src/app/web.spec.ts | 53 +++++++++++++++ packages/data-schemas/src/app/web.ts | 11 ++- 8 files changed, 289 insertions(+), 3 deletions(-) diff --git a/librechat.example.yaml b/librechat.example.yaml index 7b3b8c1e89..f7d354de53 100644 --- a/librechat.example.yaml +++ b/librechat.example.yaml @@ -1061,6 +1061,29 @@ endpoints: # # includeFavicon: false # Include favicon URL for each result # # format: markdown # 'markdown' (default) or 'text' (plain text, may increase latency) # # timeout: 15000 # HTTP request timeout in milliseconds (max 120000); Tavily Extract receives seconds clamped to 1-60 +# +# SearXNG as the search provider example: +# webSearch: +# searchProvider: searxng +# searxngInstanceUrl: '${SEARXNG_INSTANCE_URL}' +# # searxngApiKey: '${SEARXNG_API_KEY}' +# searxngSearchOptions: +# # Engines your instance should query. Accepts a comma-separated string or a list. +# # Defaults to 'google,bing,duckduckgo'; DuckDuckGo serves CAPTCHAs to most +# # self-hosted instances, so override it when results come back empty. +# # +# # Names must match engines enabled on your own instance. SearXNG ignores an +# # engine it does not know instead of reporting an error, so a typo or a +# # disabled engine shows up as fewer results rather than a failure. Check the +# # enabled list at https://your-instance/config before setting this. +# engines: +# - google +# - bing +# - startpage +# - qwant +# # language: en # Result language code (default: 'all') +# # timeRange: month # 'day', 'month', or 'year' +# # timeout: 10000 # HTTP request timeout in milliseconds (1-120000, default: 10000) # Memory configuration for user memories # memory: diff --git a/packages/api/src/web/web.spec.ts b/packages/api/src/web/web.spec.ts index 64d54d2ddb..e10a7f72de 100644 --- a/packages/api/src/web/web.spec.ts +++ b/packages/api/src/web/web.spec.ts @@ -958,6 +958,87 @@ describe('web.ts', () => { } }); + it('should authenticate SearXNG as a search provider and pass search options through', async () => { + const webSearchConfig: TCustomConfig['webSearch'] = { + searxngInstanceUrl: '${SEARXNG_INSTANCE_URL}', + searxngApiKey: '${SEARXNG_API_KEY}', + firecrawlApiKey: '${FIRECRAWL_API_KEY}', + firecrawlApiUrl: '${FIRECRAWL_API_URL}', + safeSearch: SafeSearchTypes.MODERATE, + searchProvider: 'searxng' as SearchProviders, + scraperProvider: 'firecrawl' as ScraperProviders, + rerankerType: 'none' as RerankerTypes, + searxngSearchOptions: { + engines: 'google,bing,startpage,qwant', + language: 'en', + timeRange: 'month', + timeout: 15000, + }, + }; + + mockLoadAuthValues.mockImplementation(({ authFields }) => { + const result: Record = {}; + authFields.forEach((field: string) => { + if (field === 'SEARXNG_INSTANCE_URL') { + result[field] = 'https://search.example'; + } else if (field === 'SEARXNG_API_KEY') { + result[field] = 'searxng-api-key'; + } else if (field === 'FIRECRAWL_API_URL') { + result[field] = 'https://api.firecrawl.dev'; + } else { + result[field] = 'test-api-key'; + } + }); + return Promise.resolve(result); + }); + + const result = await loadWebSearchAuth({ + userId, + webSearchConfig, + loadAuthValues: mockLoadAuthValues, + }); + + expect(result.authenticated).toBe(true); + expect(result.authResult.searchProvider).toBe('searxng'); + expect(result.authResult.searxngInstanceUrl).toBe('https://search.example'); + expect(result.authResult.searxngSearchOptions).toEqual(webSearchConfig.searxngSearchOptions); + }); + + it('should leave searxngSearchOptions undefined when not configured', async () => { + const webSearchConfig: TCustomConfig['webSearch'] = { + searxngInstanceUrl: '${SEARXNG_INSTANCE_URL}', + firecrawlApiKey: '${FIRECRAWL_API_KEY}', + firecrawlApiUrl: '${FIRECRAWL_API_URL}', + safeSearch: SafeSearchTypes.MODERATE, + searchProvider: 'searxng' as SearchProviders, + scraperProvider: 'firecrawl' as ScraperProviders, + rerankerType: 'none' as RerankerTypes, + }; + + mockLoadAuthValues.mockImplementation(({ authFields }) => { + const result: Record = {}; + authFields.forEach((field: string) => { + if (field === 'SEARXNG_INSTANCE_URL') { + result[field] = 'https://search.example'; + } else if (field === 'FIRECRAWL_API_URL') { + result[field] = 'https://api.firecrawl.dev'; + } else { + result[field] = 'test-api-key'; + } + }); + return Promise.resolve(result); + }); + + const result = await loadWebSearchAuth({ + userId, + webSearchConfig, + loadAuthValues: mockLoadAuthValues, + }); + + expect(result.authenticated).toBe(true); + expect(result.authResult.searxngSearchOptions).toBeUndefined(); + }); + it('should fail authentication when Tavily search API key is missing', async () => { const webSearchConfig: TCustomConfig['webSearch'] = { tavilyApiKey: '${TAVILY_API_KEY}', diff --git a/packages/api/src/web/web.ts b/packages/api/src/web/web.ts index 6def5a887d..3933936518 100644 --- a/packages/api/src/web/web.ts +++ b/packages/api/src/web/web.ts @@ -293,6 +293,7 @@ export async function loadWebSearchAuth({ } authResult.scraperTimeout = webSearchConfig?.scraperTimeout ?? scraperOptionsTimeout ?? 7500; authResult.firecrawlOptions = webSearchConfig?.firecrawlOptions; + authResult.searxngSearchOptions = webSearchConfig?.searxngSearchOptions; authResult.tavilySearchOptions = webSearchConfig?.tavilySearchOptions; authResult.tavilyScraperOptions = webSearchConfig?.tavilyScraperOptions; diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts index 7427a551ea..52932db21e 100644 --- a/packages/data-provider/src/config.spec.ts +++ b/packages/data-provider/src/config.spec.ts @@ -665,6 +665,88 @@ describe('webSearchSchema', () => { }), ).toThrow(); }); + + it('accepts SearXNG search options', () => { + const result = webSearchSchema.parse({ + searxngSearchOptions: { + engines: 'google,bing,startpage,qwant', + language: 'en', + timeRange: 'month', + timeout: 15000, + }, + }); + + expect(result.searxngSearchOptions?.engines).toBe('google,bing,startpage,qwant'); + expect(result.searxngSearchOptions?.language).toBe('en'); + expect(result.searxngSearchOptions?.timeRange).toBe('month'); + expect(result.searxngSearchOptions?.timeout).toBe(15000); + }); + + it('normalizes a SearXNG engine list into a comma-separated string', () => { + const result = webSearchSchema.parse({ + searxngSearchOptions: { + engines: ['google', 'bing', 'startpage', 'qwant'], + }, + }); + + expect(result.searxngSearchOptions?.engines).toBe('google,bing,startpage,qwant'); + }); + + it('trims whitespace and empty entries from SearXNG engines', () => { + const result = webSearchSchema.parse({ + searxngSearchOptions: { + engines: 'google, bing , , startpage', + }, + }); + + expect(result.searxngSearchOptions?.engines).toBe('google,bing,startpage'); + }); + + it('treats a blank SearXNG engines value as unset', () => { + const result = webSearchSchema.parse({ + searxngSearchOptions: { + engines: ' , ', + }, + }); + + expect(result.searxngSearchOptions?.engines).toBeUndefined(); + }); + + it('rejects invalid SearXNG search options', () => { + expect(() => + webSearchSchema.parse({ + searxngSearchOptions: { + timeRange: 'week', + }, + }), + ).toThrow(); + + expect(() => + webSearchSchema.parse({ + searxngSearchOptions: { + timeout: 120001, + }, + }), + ).toThrow(); + + expect(() => + webSearchSchema.parse({ + searxngSearchOptions: { + engines: 42, + }, + }), + ).toThrow(); + }); + + it('rejects a zero SearXNG timeout, which axios reads as no timeout at all', () => { + expect(() => + webSearchSchema.parse({ + searxngSearchOptions: { + timeout: 0, + }, + }), + ).toThrow(); + }); }); describe('bedrockModels defaults', () => { diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 966f479663..c7661f3976 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -1844,6 +1844,22 @@ export enum SafeSearchTypes { STRICT = 2, } +/** + * Normalizes a SearXNG engine list into the comma-separated form the API expects. + * Accepts the YAML list or comma-separated string an operator may write, and is + * applied both at the schema boundary and when loading the runtime config, since + * `loadCustomConfig` returns the raw YAML object rather than the parsed result. + */ +export function normalizeSearxngEngines(engines?: string | string[]): string | undefined { + if (engines == null) { + return undefined; + } + const normalized = (Array.isArray(engines) ? engines : engines.split(',')) + .map((engine) => engine.trim()) + .filter(Boolean); + return normalized.length ? normalized.join(',') : undefined; +} + export const webSearchSchema = z.object({ allowedAddresses: allowedAddressesSchema, serperApiKey: z.string().optional().default('${SERPER_API_KEY}'), @@ -1902,6 +1918,17 @@ export const webSearchSchema = z.object({ .optional(), }) .optional(), + searxngSearchOptions: z + .object({ + engines: z + .union([z.string(), z.array(z.string())]) + .transform(normalizeSearxngEngines) + .optional(), + language: z.string().optional(), + timeRange: z.enum(['day', 'month', 'year']).optional(), + timeout: z.number().int().positive().max(120000).optional(), + }) + .optional(), tavilySearchOptions: z .object({ searchDepth: z.enum(['basic', 'advanced', 'fast', 'ultra-fast']).optional(), @@ -2210,6 +2237,19 @@ export type DeepPartial = T extends (infer U)[] export const getConfigDefaults = () => getSchemaDefaults(configSchema); export type TCustomConfig = DeepPartial>; + +/** + * Shape of the `webSearch` block as written in `librechat.yaml`, where + * `searxngSearchOptions.engines` may still be the YAML list or untrimmed string an + * operator wrote. `loadCustomConfig` returns the raw YAML object rather than the + * parsed result, so the runtime loader receives this shape, not the parsed one. + */ +export type TWebSearchConfigInput = Omit< + NonNullable, + 'searxngSearchOptions' +> & { + searxngSearchOptions?: z.input['searxngSearchOptions']; +}; export type TCustomEndpoints = z.infer; export type TProviderSchema = diff --git a/packages/data-provider/src/types/web.ts b/packages/data-provider/src/types/web.ts index 2196b3cd85..fac46d2d89 100644 --- a/packages/data-provider/src/types/web.ts +++ b/packages/data-provider/src/types/web.ts @@ -76,6 +76,7 @@ export interface SearchConfig { serperApiKey?: string; searxngInstanceUrl?: string; searxngApiKey?: string; + searxngSearchOptions?: z.infer['searxngSearchOptions']; tavilyApiKey?: string; tavilySearchUrl?: string; tavilySearchOptions?: TavilyConfig['tavilySearchOptions']; diff --git a/packages/data-schemas/src/app/web.spec.ts b/packages/data-schemas/src/app/web.spec.ts index 9a9a0596dc..0e0e406cd1 100644 --- a/packages/data-schemas/src/app/web.spec.ts +++ b/packages/data-schemas/src/app/web.spec.ts @@ -93,6 +93,59 @@ describe('loadWebSearchConfig', () => { expect(result?.searchProvider).toBe('serper'); expect(result?.serperApiKey).toBe('test-key'); }); + + it('should preserve searxngSearchOptions', () => { + const config: TCustomConfig['webSearch'] = { + searchProvider: SearchProviders.SEARXNG, + searxngSearchOptions: { + engines: 'google,bing,startpage,qwant', + language: 'en', + }, + }; + + const result = loadWebSearchConfig(config); + + expect(result?.searxngSearchOptions).toEqual({ + engines: 'google,bing,startpage,qwant', + language: 'en', + }); + }); + + it('should normalize a YAML engine list, which reaches this loader unparsed', () => { + const result = loadWebSearchConfig({ + searchProvider: SearchProviders.SEARXNG, + searxngSearchOptions: { + engines: ['google', 'bing', 'startpage'], + language: 'en', + }, + }); + + expect(result?.searxngSearchOptions?.engines).toBe('google,bing,startpage'); + }); + + it('should trim whitespace and empty entries from a raw engines string', () => { + const result = loadWebSearchConfig({ + searchProvider: SearchProviders.SEARXNG, + searxngSearchOptions: { engines: 'google, bing , , startpage' }, + }); + + expect(result?.searxngSearchOptions?.engines).toBe('google,bing,startpage'); + }); + + it('should treat an all-blank engines value as unset', () => { + const result = loadWebSearchConfig({ + searchProvider: SearchProviders.SEARXNG, + searxngSearchOptions: { engines: ' , ' }, + }); + + expect(result?.searxngSearchOptions?.engines).toBeUndefined(); + }); + + it('should leave searxngSearchOptions undefined when the block is absent', () => { + const result = loadWebSearchConfig({ searchProvider: SearchProviders.SEARXNG }); + + expect(result?.searxngSearchOptions).toBeUndefined(); + }); }); describe('safeSearch', () => { diff --git a/packages/data-schemas/src/app/web.ts b/packages/data-schemas/src/app/web.ts index 989931db87..14a74f20e5 100644 --- a/packages/data-schemas/src/app/web.ts +++ b/packages/data-schemas/src/app/web.ts @@ -1,5 +1,5 @@ -import { RerankerTypes, SafeSearchTypes } from 'librechat-data-provider'; -import type { TCustomConfig } from 'librechat-data-provider'; +import { SafeSearchTypes, normalizeSearxngEngines } from 'librechat-data-provider'; +import type { TCustomConfig, TWebSearchConfigInput } from 'librechat-data-provider'; import type { TWebSearchKeys, TWebSearchCategories } from '~/types/web'; export const webSearchAuth = { @@ -69,7 +69,7 @@ export function getWebSearchKeys(): TWebSearchKeys[] { export const webSearchKeys: TWebSearchKeys[] = getWebSearchKeys(); export function loadWebSearchConfig( - config: TCustomConfig['webSearch'], + config: TWebSearchConfigInput | undefined, ): TCustomConfig['webSearch'] { const serperApiKey = config?.serperApiKey ?? '${SERPER_API_KEY}'; const searxngInstanceUrl = config?.searxngInstanceUrl ?? '${SEARXNG_INSTANCE_URL}'; @@ -85,10 +85,15 @@ export function loadWebSearchConfig( const cohereApiKey = config?.cohereApiKey ?? '${COHERE_API_KEY}'; const safeSearch = config?.safeSearch ?? SafeSearchTypes.MODERATE; const rerankerType = config?.rerankerType; + const searxngSearchOptions = config?.searxngSearchOptions && { + ...config.searxngSearchOptions, + engines: normalizeSearxngEngines(config.searxngSearchOptions.engines), + }; return { ...config, // Preserve provider-specific option blocks such as firecrawlOptions and tavilySearchOptions. safeSearch, + searxngSearchOptions, jinaApiKey, jinaApiUrl, cohereApiKey,