import { webSearchAuth, webSearchKeys, webSearchSelectionFields } from '@librechat/data-schemas'; import { AuthType, RerankerTypes, SafeSearchTypes, SearchCategories, SearchProviders, ScraperProviders, extractVariableName, } from 'librechat-data-provider'; import type { TWebSearchKeys, TWebSearchCategories } from '@librechat/data-schemas'; import type { TCustomConfig, TWebSearchConfig } from 'librechat-data-provider'; import { isSSRFTarget, resolveHostnameSSRF, getEffectivePort } from '../auth'; /** * User-provided URL keys and their paired API keys, checked after SSRF preflight. */ const USER_PROVIDED_URL_KEYS = new Map([ ['searxngInstanceUrl', 'searxngApiKey'], ['firecrawlApiUrl', 'firecrawlApiKey'], ['jinaApiUrl', 'jinaApiKey'], ]); /** * URL keys that require explicit admin opt-in before user-provided values may pass through. */ const USER_PROVIDED_OPT_IN_URL_KEYS = new Set([ 'tavilySearchUrl', 'tavilyExtractUrl', ]); const SEARCH_PROVIDER_VALUES = new Set(Object.values(SearchProviders)); const SCRAPER_PROVIDER_VALUES = new Set(Object.values(ScraperProviders)); const RERANKER_VALUES = new Set(Object.values(RerankerTypes)); const DEFAULT_JINA_API_URL = 'https://api.jina.ai/v1/rerank'; function isUserProvidedEnabled(field: string): boolean { return process.env[field] === AuthType.USER_PROVIDED; } /** * Returns true if the URL should be blocked for SSRF risk. * Fail-closed: unparseable URLs and non-HTTP(S) schemes return true. * `allowedAddresses` keeps this preflight consistent with the connect-time agent * so an admin-permitted private endpoint is not stripped before the agent runs. */ async function isSSRFUrl(url: string, allowedAddresses?: string[] | null): Promise { let parsed: URL; try { parsed = new URL(url); } catch { return true; } if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { return true; } const port = getEffectivePort(parsed.protocol, parsed.port); if (isSSRFTarget(parsed.hostname, allowedAddresses, port)) { return true; } return resolveHostnameSSRF(parsed.hostname, allowedAddresses, port); } export function extractWebSearchEnvVars({ keys, config, }: { keys: TWebSearchKeys[]; config: TCustomConfig['webSearch'] | undefined; }): string[] { if (!config) { return []; } const authFields: string[] = []; const relevantKeys = keys.filter((k) => k in config); for (const key of relevantKeys) { const value = config[key]; if (typeof value === 'string') { const varName = extractVariableName(value); if (varName) { authFields.push(varName); } } } return authFields; } function mapWebSearchSelection(key: string, value: string): [string, string] | undefined { if (key === 'selectedProvider' && SEARCH_PROVIDER_VALUES.has(value)) { return [webSearchSelectionFields.selectedProvider, value]; } if (key === 'selectedScraper' && SCRAPER_PROVIDER_VALUES.has(value)) { return [webSearchSelectionFields.selectedScraper, value]; } if (key === 'selectedReranker' && RERANKER_VALUES.has(value)) { return [webSearchSelectionFields.selectedReranker, value]; } return undefined; } export function getWebSearchInstallEntries({ auth, config, }: { auth: Partial>; config: TCustomConfig['webSearch']; }): [string, string][] { const credentialEntries: [string, string][] = []; const selectionEntries: [string, string][] = []; for (const [key, value] of Object.entries(auth)) { if (typeof value !== 'string') { continue; } const selection = mapWebSearchSelection(key, value); if (selection) { selectionEntries.push(selection); continue; } const [authField] = extractWebSearchEnvVars({ keys: [key as TWebSearchKeys], config, }); if (authField) { credentialEntries.push([authField, value]); } } return [...credentialEntries, ...selectionEntries]; } export function getWebSearchUninstallFields(config: TCustomConfig['webSearch']): string[] { return [ ...extractWebSearchEnvVars({ keys: webSearchKeys, config }), ...Object.values(webSearchSelectionFields), ]; } /** * Type for web search authentication result */ export interface WebSearchAuthResult { /** Whether all required categories have at least one authenticated service */ authenticated: boolean; /** Authentication type (user_provided or system_defined) by category */ authTypes: [TWebSearchCategories, AuthType][]; /** Original authentication values mapped to their respective keys */ authResult: Partial; } interface KeenableAuthResolution { isUserProvided: boolean; hasSystemApiKey: boolean; rejectedUserApiUrl: boolean; lookupFailed: boolean; } /** * Loads and verifies web search authentication values * @param params - Authentication parameters * @returns Authentication result */ export async function loadWebSearchAuth({ userId, webSearchConfig, loadAuthValues, throwError = true, }: { userId: string; webSearchConfig: TCustomConfig['webSearch']; loadAuthValues: (params: { userId: string; authFields: string[]; optional?: Set; throwError?: boolean; failOnOptionalError?: boolean; }) => Promise>; throwError?: boolean; }): Promise { let authenticated = true; const authResult: Partial = {}; /** * Keenable is keyless by default: both its search and its page fetch work * against public endpoints with no key, so neither category needs a secret to * authenticate. This resolves the optional key/URL overrides once (a key only * lifts rate limits) and reports whether any of them came from the user. */ const keenableAuth = new Map>(); function resolveKeenableAuth(includeApiUrl: boolean): Promise { const cached = keenableAuth.get(includeApiUrl); if (cached) { return cached; } const resolution = (async () => { let keenableUserProvided = false; let hasSystemApiKey = false; let rejectedUserApiUrl = false; const keenableKeys: TWebSearchKeys[] = includeApiUrl ? ['keenableApiKey', 'keenableApiUrl'] : ['keenableApiKey']; const authEntries: Array<{ key: TWebSearchKeys; field: string }> = []; for (const originalKey of keenableKeys) { const [field] = extractWebSearchEnvVars({ keys: [originalKey], config: webSearchConfig, }); if (field) { authEntries.push({ key: originalKey, field }); } } let authValues: Record = {}; try { const authFields = authEntries.map(({ field }) => field); authValues = await loadAuthValues({ userId, authFields, optional: new Set(authFields), throwError: true, failOnOptionalError: true, }); } catch { return { isUserProvided: false, hasSystemApiKey: false, rejectedUserApiUrl: false, lookupFailed: true, }; } const resolvedEntries: Array<{ key: TWebSearchKeys; value: string; isFieldUserProvided: boolean; }> = []; for (const { key: originalKey, field } of authEntries) { const value = authValues[field]; const envValue = process.env[field]; const normalizedEnvValue = envValue?.trim(); const isFieldUserProvided = normalizedEnvValue == null || normalizedEnvValue === '' || normalizedEnvValue === AuthType.USER_PROVIDED; if (isFieldUserProvided) { // The category stays editable even before the user saves a value. // Otherwise a system key would hide a separately user-provided URL. keenableUserProvided = true; } if (!value) { continue; } if ( originalKey === 'keenableApiUrl' && isFieldUserProvided && (await isSSRFUrl(value, webSearchConfig?.allowedAddresses)) ) { rejectedUserApiUrl = true; continue; } resolvedEntries.push({ key: originalKey, value, isFieldUserProvided }); } if (rejectedUserApiUrl) { return { isUserProvided: true, hasSystemApiKey: false, rejectedUserApiUrl: true, lookupFailed: false, }; } const hasUserProvidedApiUrl = resolvedEntries.some( ({ key, isFieldUserProvided }) => key === 'keenableApiUrl' && isFieldUserProvided, ); for (const { key: originalKey, value, isFieldUserProvided } of resolvedEntries) { // Never forward an administrator's secret to an endpoint controlled by // the user. Keenable remains functional without the key at that URL. if (originalKey === 'keenableApiKey' && !isFieldUserProvided && hasUserProvidedApiUrl) { continue; } authResult[originalKey] = value; if (originalKey === 'keenableApiKey' && !isFieldUserProvided) { hasSystemApiKey = true; } } return { isUserProvided: keenableUserProvided, hasSystemApiKey, rejectedUserApiUrl: false, lookupFailed: false, }; })(); keenableAuth.set(includeApiUrl, resolution); return resolution; } let userSelections: | Promise<{ searchProvider?: SearchProviders; scraperProvider?: ScraperProviders; rerankerType?: RerankerTypes; lookupFailed?: boolean; }> | undefined; function resolveUserSelections(): Promise<{ searchProvider?: SearchProviders; scraperProvider?: ScraperProviders; rerankerType?: RerankerTypes; lookupFailed?: boolean; }> { userSelections ??= (async () => { const fields: string[] = []; if (!webSearchConfig?.searchProvider) { fields.push(webSearchSelectionFields.selectedProvider); } if (!webSearchConfig?.scraperProvider) { fields.push(webSearchSelectionFields.selectedScraper); } if (!webSearchConfig?.rerankerType) { fields.push(webSearchSelectionFields.selectedReranker); } if (fields.length === 0) { return {}; } let values: Record; try { values = await loadAuthValues({ userId, authFields: fields, optional: new Set(fields), throwError: true, failOnOptionalError: true, }); } catch { return { lookupFailed: true }; } const searchProvider = values[webSearchSelectionFields.selectedProvider]; const scraperProvider = values[webSearchSelectionFields.selectedScraper]; const rerankerType = values[webSearchSelectionFields.selectedReranker]; return { searchProvider: searchProvider != null && SEARCH_PROVIDER_VALUES.has(searchProvider) ? (searchProvider as SearchProviders) : undefined, scraperProvider: scraperProvider != null && SCRAPER_PROVIDER_VALUES.has(scraperProvider) ? (scraperProvider as ScraperProviders) : undefined, rerankerType: rerankerType != null && RERANKER_VALUES.has(rerankerType) ? (rerankerType as RerankerTypes) : undefined, }; })(); return userSelections; } /** Type-safe iterator for the category-service combinations */ async function checkAuth( category: C, ): Promise<[boolean, boolean]> { type ServiceType = keyof (typeof webSearchAuth)[C]; let isUserProvided = false; // Check if a specific service is specified in the config let specificService: ServiceType | undefined; if (category === SearchCategories.PROVIDERS && webSearchConfig?.searchProvider) { specificService = webSearchConfig.searchProvider as unknown as ServiceType; } else if (category === SearchCategories.SCRAPERS && webSearchConfig?.scraperProvider) { specificService = webSearchConfig.scraperProvider as unknown as ServiceType; } else if (category === SearchCategories.RERANKERS && webSearchConfig?.rerankerType) { specificService = webSearchConfig.rerankerType as unknown as ServiceType; } if (!specificService) { const selections = await resolveUserSelections(); if (selections.lookupFailed) { return [false, true]; } if (category === SearchCategories.PROVIDERS && selections.searchProvider) { specificService = selections.searchProvider as unknown as ServiceType; authResult.searchProvider = selections.searchProvider; isUserProvided = true; } else if (category === SearchCategories.SCRAPERS && selections.scraperProvider) { specificService = selections.scraperProvider as unknown as ServiceType; authResult.scraperProvider = selections.scraperProvider; isUserProvided = true; } else if (category === SearchCategories.RERANKERS && selections.rerankerType) { specificService = selections.rerankerType as unknown as ServiceType; authResult.rerankerType = selections.rerankerType; isUserProvided = true; } } if (category === SearchCategories.RERANKERS && specificService === 'none') { authResult.rerankerType = specificService as RerankerTypes; return [true, isUserProvided]; } // Special case: Keenable is keyless by default. The public endpoints need no // key, so a pinned Keenable authenticates even when nothing is configured — // as a search provider and as a scraper alike. if (category === SearchCategories.PROVIDERS && specificService === SearchProviders.KEENABLE) { const keenable = await resolveKeenableAuth(true); authResult.searchProvider = SearchProviders.KEENABLE; if (keenable.lookupFailed || keenable.rejectedUserApiUrl) { return [false, true]; } return [true, isUserProvided || keenable.isUserProvided || !keenable.hasSystemApiKey]; } if (category === SearchCategories.SCRAPERS && specificService === ScraperProviders.KEENABLE) { const searchUsesKeenable = authResult.searchProvider === SearchProviders.KEENABLE || webSearchConfig?.searchProvider === SearchProviders.KEENABLE; const keenable = await resolveKeenableAuth(searchUsesKeenable); authResult.scraperProvider = ScraperProviders.KEENABLE; if (keenable.lookupFailed || keenable.rejectedUserApiUrl) { return [false, true]; } return [true, isUserProvided || keenable.isUserProvided || !keenable.hasSystemApiKey]; } // If a specific service is specified, only check that one const services = specificService ? [specificService] : (Object.keys(webSearchAuth[category]) as ServiceType[]); for (const service of services) { // Skip if the service doesn't exist in the webSearchAuth config if (!webSearchAuth[category][service]) { continue; } const serviceConfig = webSearchAuth[category][service]; // Split keys into required and optional const requiredKeys: TWebSearchKeys[] = []; const optionalKeys: TWebSearchKeys[] = []; for (const key in serviceConfig) { const typedKey = key as TWebSearchKeys; if (serviceConfig[typedKey as keyof typeof serviceConfig] === 1) { requiredKeys.push(typedKey); } else if (serviceConfig[typedKey as keyof typeof serviceConfig] === 0) { optionalKeys.push(typedKey); } } if (requiredKeys.length === 0) continue; const requiredAuthFields = extractWebSearchEnvVars({ keys: requiredKeys, config: webSearchConfig, }); const optionalAuthFields = extractWebSearchEnvVars({ keys: optionalKeys, config: webSearchConfig, }); if (requiredAuthFields.length !== requiredKeys.length) continue; const allKeys = [...requiredKeys, ...optionalKeys]; const allAuthFields = [...requiredAuthFields, ...optionalAuthFields]; const optionalSet = new Set(optionalAuthFields); try { const authValues = await loadAuthValues({ userId, authFields: allAuthFields, optional: optionalSet, throwError, }); const jinaApiKeyIndex = allKeys.indexOf('jinaApiKey'); const jinaApiKeyField = allAuthFields[jinaApiKeyIndex]; const jinaApiKeyValue = jinaApiKeyField ? authValues[jinaApiKeyField] : undefined; let allFieldsAuthenticated = true; for (let j = 0; j < allAuthFields.length; j++) { const field = allAuthFields[j]; const value = authValues[field]; const originalKey = allKeys[j]; if (!optionalSet.has(field) && !value) { allFieldsAuthenticated = false; break; } const isFieldUserProvided = value != null && process.env[field] !== value; const isUserProvidedUrlKey = originalKey != null && USER_PROVIDED_URL_KEYS.has(originalKey); const isUserProvidedOptInUrlKey = originalKey != null && USER_PROVIDED_OPT_IN_URL_KEYS.has(originalKey); const isUserProvidedUrlEnabled = isUserProvidedUrlKey || (isUserProvidedOptInUrlKey && isUserProvidedEnabled(field)); let contributed = false; if (isUserProvidedOptInUrlKey && isFieldUserProvided && !isUserProvidedUrlEnabled) { if (!optionalSet.has(field)) { allFieldsAuthenticated = false; break; } continue; } if ( isUserProvidedUrlEnabled && isFieldUserProvided && (await isSSRFUrl(value, webSearchConfig?.allowedAddresses)) ) { if (!optionalSet.has(field)) { allFieldsAuthenticated = false; break; } continue; } if (isUserProvidedUrlKey && isFieldUserProvided) { const apiKey = USER_PROVIDED_URL_KEYS.get(originalKey); const apiKeyField = apiKey ? allAuthFields[allKeys.indexOf(apiKey)] : undefined; const apiKeyValue = apiKeyField ? authValues[apiKeyField] : undefined; if (apiKeyField && apiKeyValue && process.env[apiKeyField] === apiKeyValue) { if (!optionalSet.has(field)) { allFieldsAuthenticated = false; break; } continue; } } if (originalKey) { authResult[originalKey] = value; contributed = true; } if (!isUserProvided && isFieldUserProvided && contributed) { isUserProvided = true; } } if (jinaApiKeyValue && authResult.jinaApiUrl == null) { authResult.jinaApiUrl = DEFAULT_JINA_API_URL; } if (!allFieldsAuthenticated) { continue; } if (category === SearchCategories.PROVIDERS) { authResult.searchProvider = service as SearchProviders; } else if (category === SearchCategories.SCRAPERS) { authResult.scraperProvider = service as ScraperProviders; } else if (category === SearchCategories.RERANKERS) { authResult.rerankerType = service as RerankerTypes; } return [true, isUserProvided]; } catch { continue; } } if ( category === SearchCategories.RERANKERS && !webSearchConfig?.rerankerType && !specificService ) { authResult.rerankerType = 'none' as RerankerTypes; return [true, false]; } /** * Keyless fallback, reached only when no keyed service in the category * authenticated. The loop above skips Keenable whenever it isn't pinned, * because none of its auth fields are required (`requiredKeys.length === 0`), * so a legacy Keenable credential saved before provider selections were * persisted would otherwise leave the category unauthenticated. Gating the * compatibility path on an actual Keenable value preserves those installs * without making Keenable the implicit default for new users. */ if (category === SearchCategories.PROVIDERS && !specificService) { const keenable = await resolveKeenableAuth(true); if (keenable.lookupFailed || keenable.rejectedUserApiUrl) { return [false, true]; } if (authResult.keenableApiKey || authResult.keenableApiUrl) { authResult.searchProvider = SearchProviders.KEENABLE; return [true, keenable.isUserProvided]; } } /** * Same for the scraper. Reached only when no keyed scraper authenticated, so * the alternative is web search staying disabled entirely. Two triggers: * Keenable being the resolved search provider (providers are checked first), * which is what completes a fully keyless stack, or a Keenable value actually * being present, which is how the API-key dialog expresses "scrape with * Keenable" for a deployment whose search runs on someone else. With neither, * it stays unauthenticated rather than silently scraping for a provider that * was never told to use Keenable. */ if (category === SearchCategories.SCRAPERS && !specificService) { const searchUsesKeenable = authResult.searchProvider === SearchProviders.KEENABLE; const keenable = await resolveKeenableAuth(searchUsesKeenable); if (keenable.lookupFailed || keenable.rejectedUserApiUrl) { return [false, true]; } if ( authResult.searchProvider === SearchProviders.KEENABLE || authResult.keenableApiKey || authResult.keenableApiUrl ) { authResult.scraperProvider = ScraperProviders.KEENABLE; return [true, keenable.isUserProvided]; } } return [false, isUserProvided]; } const categories = [ SearchCategories.PROVIDERS, SearchCategories.SCRAPERS, SearchCategories.RERANKERS, ] as const; const authTypes: [TWebSearchCategories, AuthType][] = []; for (const category of categories) { const [isCategoryAuthenticated, isUserProvided] = await checkAuth(category); if (!isCategoryAuthenticated) { authenticated = false; authTypes.push([category, AuthType.USER_PROVIDED]); continue; } authTypes.push([category, isUserProvided ? AuthType.USER_PROVIDED : AuthType.SYSTEM_DEFINED]); } const scraperProvider = authResult.scraperProvider ?? webSearchConfig?.scraperProvider ?? ScraperProviders.FIRECRAWL; let scraperOptionsTimeout: number | undefined; if (scraperProvider === ScraperProviders.TAVILY) { scraperOptionsTimeout = webSearchConfig?.tavilyScraperOptions?.timeout; } else if (scraperProvider === ScraperProviders.FIRECRAWL) { scraperOptionsTimeout = webSearchConfig?.firecrawlOptions?.timeout; } else if (scraperProvider === ScraperProviders.KEENABLE) { scraperOptionsTimeout = webSearchConfig?.keenableScraperOptions?.timeout; } const searchProvider = authResult.searchProvider ?? webSearchConfig?.searchProvider; if (searchProvider !== SearchProviders.TAVILY) { authResult.safeSearch = webSearchConfig?.safeSearch ?? SafeSearchTypes.MODERATE; } authResult.scraperTimeout = webSearchConfig?.scraperTimeout ?? scraperOptionsTimeout ?? 7500; authResult.firecrawlOptions = webSearchConfig?.firecrawlOptions; authResult.searxngSearchOptions = webSearchConfig?.searxngSearchOptions; authResult.tavilySearchOptions = webSearchConfig?.tavilySearchOptions; authResult.tavilyScraperOptions = webSearchConfig?.tavilyScraperOptions; authResult.keenableSearchOptions = webSearchConfig?.keenableSearchOptions; authResult.keenableScraperOptions = webSearchConfig?.keenableScraperOptions; return { authTypes, authResult, authenticated, }; }