diff --git a/librechat.example.yaml b/librechat.example.yaml index 7b3b8c1e89..f7d354de53 100644 --- a/librechat.example.yaml +++ b/librechat.example.yaml @@ -1061,6 +1061,29 @@ endpoints: # # includeFavicon: false # Include favicon URL for each result # # format: markdown # 'markdown' (default) or 'text' (plain text, may increase latency) # # timeout: 15000 # HTTP request timeout in milliseconds (max 120000); Tavily Extract receives seconds clamped to 1-60 +# +# SearXNG as the search provider example: +# webSearch: +# searchProvider: searxng +# searxngInstanceUrl: '${SEARXNG_INSTANCE_URL}' +# # searxngApiKey: '${SEARXNG_API_KEY}' +# searxngSearchOptions: +# # Engines your instance should query. Accepts a comma-separated string or a list. +# # Defaults to 'google,bing,duckduckgo'; DuckDuckGo serves CAPTCHAs to most +# # self-hosted instances, so override it when results come back empty. +# # +# # Names must match engines enabled on your own instance. SearXNG ignores an +# # engine it does not know instead of reporting an error, so a typo or a +# # disabled engine shows up as fewer results rather than a failure. Check the +# # enabled list at https://your-instance/config before setting this. +# engines: +# - google +# - bing +# - startpage +# - qwant +# # language: en # Result language code (default: 'all') +# # timeRange: month # 'day', 'month', or 'year' +# # timeout: 10000 # HTTP request timeout in milliseconds (1-120000, default: 10000) # Memory configuration for user memories # memory: diff --git a/packages/api/src/web/web.spec.ts b/packages/api/src/web/web.spec.ts index 64d54d2ddb..e10a7f72de 100644 --- a/packages/api/src/web/web.spec.ts +++ b/packages/api/src/web/web.spec.ts @@ -958,6 +958,87 @@ describe('web.ts', () => { } }); + it('should authenticate SearXNG as a search provider and pass search options through', async () => { + const webSearchConfig: TCustomConfig['webSearch'] = { + searxngInstanceUrl: '${SEARXNG_INSTANCE_URL}', + searxngApiKey: '${SEARXNG_API_KEY}', + firecrawlApiKey: '${FIRECRAWL_API_KEY}', + firecrawlApiUrl: '${FIRECRAWL_API_URL}', + safeSearch: SafeSearchTypes.MODERATE, + searchProvider: 'searxng' as SearchProviders, + scraperProvider: 'firecrawl' as ScraperProviders, + rerankerType: 'none' as RerankerTypes, + searxngSearchOptions: { + engines: 'google,bing,startpage,qwant', + language: 'en', + timeRange: 'month', + timeout: 15000, + }, + }; + + mockLoadAuthValues.mockImplementation(({ authFields }) => { + const result: Record = {}; + authFields.forEach((field: string) => { + if (field === 'SEARXNG_INSTANCE_URL') { + result[field] = 'https://search.example'; + } else if (field === 'SEARXNG_API_KEY') { + result[field] = 'searxng-api-key'; + } else if (field === 'FIRECRAWL_API_URL') { + result[field] = 'https://api.firecrawl.dev'; + } else { + result[field] = 'test-api-key'; + } + }); + return Promise.resolve(result); + }); + + const result = await loadWebSearchAuth({ + userId, + webSearchConfig, + loadAuthValues: mockLoadAuthValues, + }); + + expect(result.authenticated).toBe(true); + expect(result.authResult.searchProvider).toBe('searxng'); + expect(result.authResult.searxngInstanceUrl).toBe('https://search.example'); + expect(result.authResult.searxngSearchOptions).toEqual(webSearchConfig.searxngSearchOptions); + }); + + it('should leave searxngSearchOptions undefined when not configured', async () => { + const webSearchConfig: TCustomConfig['webSearch'] = { + searxngInstanceUrl: '${SEARXNG_INSTANCE_URL}', + firecrawlApiKey: '${FIRECRAWL_API_KEY}', + firecrawlApiUrl: '${FIRECRAWL_API_URL}', + safeSearch: SafeSearchTypes.MODERATE, + searchProvider: 'searxng' as SearchProviders, + scraperProvider: 'firecrawl' as ScraperProviders, + rerankerType: 'none' as RerankerTypes, + }; + + mockLoadAuthValues.mockImplementation(({ authFields }) => { + const result: Record = {}; + authFields.forEach((field: string) => { + if (field === 'SEARXNG_INSTANCE_URL') { + result[field] = 'https://search.example'; + } else if (field === 'FIRECRAWL_API_URL') { + result[field] = 'https://api.firecrawl.dev'; + } else { + result[field] = 'test-api-key'; + } + }); + return Promise.resolve(result); + }); + + const result = await loadWebSearchAuth({ + userId, + webSearchConfig, + loadAuthValues: mockLoadAuthValues, + }); + + expect(result.authenticated).toBe(true); + expect(result.authResult.searxngSearchOptions).toBeUndefined(); + }); + it('should fail authentication when Tavily search API key is missing', async () => { const webSearchConfig: TCustomConfig['webSearch'] = { tavilyApiKey: '${TAVILY_API_KEY}', diff --git a/packages/api/src/web/web.ts b/packages/api/src/web/web.ts index 6def5a887d..3933936518 100644 --- a/packages/api/src/web/web.ts +++ b/packages/api/src/web/web.ts @@ -293,6 +293,7 @@ export async function loadWebSearchAuth({ } authResult.scraperTimeout = webSearchConfig?.scraperTimeout ?? scraperOptionsTimeout ?? 7500; authResult.firecrawlOptions = webSearchConfig?.firecrawlOptions; + authResult.searxngSearchOptions = webSearchConfig?.searxngSearchOptions; authResult.tavilySearchOptions = webSearchConfig?.tavilySearchOptions; authResult.tavilyScraperOptions = webSearchConfig?.tavilyScraperOptions; diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts index 7427a551ea..52932db21e 100644 --- a/packages/data-provider/src/config.spec.ts +++ b/packages/data-provider/src/config.spec.ts @@ -665,6 +665,88 @@ describe('webSearchSchema', () => { }), ).toThrow(); }); + + it('accepts SearXNG search options', () => { + const result = webSearchSchema.parse({ + searxngSearchOptions: { + engines: 'google,bing,startpage,qwant', + language: 'en', + timeRange: 'month', + timeout: 15000, + }, + }); + + expect(result.searxngSearchOptions?.engines).toBe('google,bing,startpage,qwant'); + expect(result.searxngSearchOptions?.language).toBe('en'); + expect(result.searxngSearchOptions?.timeRange).toBe('month'); + expect(result.searxngSearchOptions?.timeout).toBe(15000); + }); + + it('normalizes a SearXNG engine list into a comma-separated string', () => { + const result = webSearchSchema.parse({ + searxngSearchOptions: { + engines: ['google', 'bing', 'startpage', 'qwant'], + }, + }); + + expect(result.searxngSearchOptions?.engines).toBe('google,bing,startpage,qwant'); + }); + + it('trims whitespace and empty entries from SearXNG engines', () => { + const result = webSearchSchema.parse({ + searxngSearchOptions: { + engines: 'google, bing , , startpage', + }, + }); + + expect(result.searxngSearchOptions?.engines).toBe('google,bing,startpage'); + }); + + it('treats a blank SearXNG engines value as unset', () => { + const result = webSearchSchema.parse({ + searxngSearchOptions: { + engines: ' , ', + }, + }); + + expect(result.searxngSearchOptions?.engines).toBeUndefined(); + }); + + it('rejects invalid SearXNG search options', () => { + expect(() => + webSearchSchema.parse({ + searxngSearchOptions: { + timeRange: 'week', + }, + }), + ).toThrow(); + + expect(() => + webSearchSchema.parse({ + searxngSearchOptions: { + timeout: 120001, + }, + }), + ).toThrow(); + + expect(() => + webSearchSchema.parse({ + searxngSearchOptions: { + engines: 42, + }, + }), + ).toThrow(); + }); + + it('rejects a zero SearXNG timeout, which axios reads as no timeout at all', () => { + expect(() => + webSearchSchema.parse({ + searxngSearchOptions: { + timeout: 0, + }, + }), + ).toThrow(); + }); }); describe('bedrockModels defaults', () => { diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 966f479663..c7661f3976 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -1844,6 +1844,22 @@ export enum SafeSearchTypes { STRICT = 2, } +/** + * Normalizes a SearXNG engine list into the comma-separated form the API expects. + * Accepts the YAML list or comma-separated string an operator may write, and is + * applied both at the schema boundary and when loading the runtime config, since + * `loadCustomConfig` returns the raw YAML object rather than the parsed result. + */ +export function normalizeSearxngEngines(engines?: string | string[]): string | undefined { + if (engines == null) { + return undefined; + } + const normalized = (Array.isArray(engines) ? engines : engines.split(',')) + .map((engine) => engine.trim()) + .filter(Boolean); + return normalized.length ? normalized.join(',') : undefined; +} + export const webSearchSchema = z.object({ allowedAddresses: allowedAddressesSchema, serperApiKey: z.string().optional().default('${SERPER_API_KEY}'), @@ -1902,6 +1918,17 @@ export const webSearchSchema = z.object({ .optional(), }) .optional(), + searxngSearchOptions: z + .object({ + engines: z + .union([z.string(), z.array(z.string())]) + .transform(normalizeSearxngEngines) + .optional(), + language: z.string().optional(), + timeRange: z.enum(['day', 'month', 'year']).optional(), + timeout: z.number().int().positive().max(120000).optional(), + }) + .optional(), tavilySearchOptions: z .object({ searchDepth: z.enum(['basic', 'advanced', 'fast', 'ultra-fast']).optional(), @@ -2210,6 +2237,19 @@ export type DeepPartial = T extends (infer U)[] export const getConfigDefaults = () => getSchemaDefaults(configSchema); export type TCustomConfig = DeepPartial>; + +/** + * Shape of the `webSearch` block as written in `librechat.yaml`, where + * `searxngSearchOptions.engines` may still be the YAML list or untrimmed string an + * operator wrote. `loadCustomConfig` returns the raw YAML object rather than the + * parsed result, so the runtime loader receives this shape, not the parsed one. + */ +export type TWebSearchConfigInput = Omit< + NonNullable, + 'searxngSearchOptions' +> & { + searxngSearchOptions?: z.input['searxngSearchOptions']; +}; export type TCustomEndpoints = z.infer; export type TProviderSchema = diff --git a/packages/data-provider/src/types/web.ts b/packages/data-provider/src/types/web.ts index 2196b3cd85..fac46d2d89 100644 --- a/packages/data-provider/src/types/web.ts +++ b/packages/data-provider/src/types/web.ts @@ -76,6 +76,7 @@ export interface SearchConfig { serperApiKey?: string; searxngInstanceUrl?: string; searxngApiKey?: string; + searxngSearchOptions?: z.infer['searxngSearchOptions']; tavilyApiKey?: string; tavilySearchUrl?: string; tavilySearchOptions?: TavilyConfig['tavilySearchOptions']; diff --git a/packages/data-schemas/src/app/web.spec.ts b/packages/data-schemas/src/app/web.spec.ts index 9a9a0596dc..0e0e406cd1 100644 --- a/packages/data-schemas/src/app/web.spec.ts +++ b/packages/data-schemas/src/app/web.spec.ts @@ -93,6 +93,59 @@ describe('loadWebSearchConfig', () => { expect(result?.searchProvider).toBe('serper'); expect(result?.serperApiKey).toBe('test-key'); }); + + it('should preserve searxngSearchOptions', () => { + const config: TCustomConfig['webSearch'] = { + searchProvider: SearchProviders.SEARXNG, + searxngSearchOptions: { + engines: 'google,bing,startpage,qwant', + language: 'en', + }, + }; + + const result = loadWebSearchConfig(config); + + expect(result?.searxngSearchOptions).toEqual({ + engines: 'google,bing,startpage,qwant', + language: 'en', + }); + }); + + it('should normalize a YAML engine list, which reaches this loader unparsed', () => { + const result = loadWebSearchConfig({ + searchProvider: SearchProviders.SEARXNG, + searxngSearchOptions: { + engines: ['google', 'bing', 'startpage'], + language: 'en', + }, + }); + + expect(result?.searxngSearchOptions?.engines).toBe('google,bing,startpage'); + }); + + it('should trim whitespace and empty entries from a raw engines string', () => { + const result = loadWebSearchConfig({ + searchProvider: SearchProviders.SEARXNG, + searxngSearchOptions: { engines: 'google, bing , , startpage' }, + }); + + expect(result?.searxngSearchOptions?.engines).toBe('google,bing,startpage'); + }); + + it('should treat an all-blank engines value as unset', () => { + const result = loadWebSearchConfig({ + searchProvider: SearchProviders.SEARXNG, + searxngSearchOptions: { engines: ' , ' }, + }); + + expect(result?.searxngSearchOptions?.engines).toBeUndefined(); + }); + + it('should leave searxngSearchOptions undefined when the block is absent', () => { + const result = loadWebSearchConfig({ searchProvider: SearchProviders.SEARXNG }); + + expect(result?.searxngSearchOptions).toBeUndefined(); + }); }); describe('safeSearch', () => { diff --git a/packages/data-schemas/src/app/web.ts b/packages/data-schemas/src/app/web.ts index 989931db87..14a74f20e5 100644 --- a/packages/data-schemas/src/app/web.ts +++ b/packages/data-schemas/src/app/web.ts @@ -1,5 +1,5 @@ -import { RerankerTypes, SafeSearchTypes } from 'librechat-data-provider'; -import type { TCustomConfig } from 'librechat-data-provider'; +import { SafeSearchTypes, normalizeSearxngEngines } from 'librechat-data-provider'; +import type { TCustomConfig, TWebSearchConfigInput } from 'librechat-data-provider'; import type { TWebSearchKeys, TWebSearchCategories } from '~/types/web'; export const webSearchAuth = { @@ -69,7 +69,7 @@ export function getWebSearchKeys(): TWebSearchKeys[] { export const webSearchKeys: TWebSearchKeys[] = getWebSearchKeys(); export function loadWebSearchConfig( - config: TCustomConfig['webSearch'], + config: TWebSearchConfigInput | undefined, ): TCustomConfig['webSearch'] { const serperApiKey = config?.serperApiKey ?? '${SERPER_API_KEY}'; const searxngInstanceUrl = config?.searxngInstanceUrl ?? '${SEARXNG_INSTANCE_URL}'; @@ -85,10 +85,15 @@ export function loadWebSearchConfig( const cohereApiKey = config?.cohereApiKey ?? '${COHERE_API_KEY}'; const safeSearch = config?.safeSearch ?? SafeSearchTypes.MODERATE; const rerankerType = config?.rerankerType; + const searxngSearchOptions = config?.searxngSearchOptions && { + ...config.searxngSearchOptions, + engines: normalizeSearxngEngines(config.searxngSearchOptions.engines), + }; return { ...config, // Preserve provider-specific option blocks such as firecrawlOptions and tavilySearchOptions. safeSearch, + searxngSearchOptions, jinaApiKey, jinaApiUrl, cohereApiKey,