mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-10-20 17:13:44 +00:00
🔍 feat: add Tavily as Search and Scraper Provider (#12581)
* feat: add Tavily integration as search provider and scraper provider * chore:update tavily web search parameters * chore:tavily paramer update * chore:update data-schemas test for tavily * fix: allow Tavily string option modes * fix: align Tavily config options * fix: scope Tavily scraper timeout * fix: use resolved scraper provider timeout * fix: widen Tavily search provider types * fix: harden Tavily web search config * fix: cap Tavily option timeouts --------- Co-authored-by: Danny Avila <danny@librechat.ai>
This commit is contained in:
Yashwanth Alapati
GitHub
parent
d6d70eeb26
commit
3da1d8c961
13 changed files with 495 additions and 5 deletions
|
|
@ -730,6 +730,130 @@ describe('web.ts', () => {
|
|||
expect(providerCalls.length).toBe(1);
|
||||
});
|
||||
|
||||
it('should authenticate Tavily as a search provider and pass options through', async () => {
|
||||
const webSearchConfig: TCustomConfig['webSearch'] = {
|
||||
tavilyApiKey: '${TAVILY_API_KEY}',
|
||||
tavilySearchUrl: '${TAVILY_SEARCH_URL}',
|
||||
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
|
||||
firecrawlApiUrl: '${FIRECRAWL_API_URL}',
|
||||
safeSearch: SafeSearchTypes.MODERATE,
|
||||
searchProvider: 'tavily' as SearchProviders,
|
||||
scraperProvider: 'firecrawl' as ScraperProviders,
|
||||
rerankerType: 'none' as RerankerTypes,
|
||||
tavilySearchOptions: {
|
||||
searchDepth: 'advanced',
|
||||
maxResults: 5,
|
||||
includeRawContent: 'markdown',
|
||||
},
|
||||
};
|
||||
|
||||
mockLoadAuthValues.mockImplementation(({ authFields }) => {
|
||||
const result: Record<string, string> = {};
|
||||
authFields.forEach((field: string) => {
|
||||
if (field === 'TAVILY_API_KEY') {
|
||||
result[field] = 'tavily-api-key';
|
||||
} else if (field === 'TAVILY_SEARCH_URL') {
|
||||
result[field] = 'https://api.tavily.com/search';
|
||||
} else if (field === 'FIRECRAWL_API_URL') {
|
||||
result[field] = 'https://api.firecrawl.dev';
|
||||
} else {
|
||||
result[field] = 'test-api-key';
|
||||
}
|
||||
});
|
||||
return Promise.resolve(result);
|
||||
});
|
||||
|
||||
const result = await loadWebSearchAuth({
|
||||
userId,
|
||||
webSearchConfig,
|
||||
loadAuthValues: mockLoadAuthValues,
|
||||
});
|
||||
|
||||
expect(result.authenticated).toBe(true);
|
||||
expect(result.authResult.searchProvider).toBe('tavily');
|
||||
expect(result.authResult.tavilyApiKey).toBe('tavily-api-key');
|
||||
expect(result.authResult.tavilySearchUrl).toBe('https://api.tavily.com/search');
|
||||
expect(result.authResult.tavilySearchOptions).toEqual(webSearchConfig.tavilySearchOptions);
|
||||
});
|
||||
|
||||
it('should fail authentication when Tavily search API key is missing', async () => {
|
||||
const webSearchConfig: TCustomConfig['webSearch'] = {
|
||||
tavilyApiKey: '${TAVILY_API_KEY}',
|
||||
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
|
||||
firecrawlApiUrl: '${FIRECRAWL_API_URL}',
|
||||
safeSearch: SafeSearchTypes.MODERATE,
|
||||
searchProvider: 'tavily' as SearchProviders,
|
||||
scraperProvider: 'firecrawl' as ScraperProviders,
|
||||
rerankerType: 'none' as RerankerTypes,
|
||||
};
|
||||
|
||||
mockLoadAuthValues.mockImplementation(({ authFields }) => {
|
||||
const result: Record<string, string> = {};
|
||||
authFields.forEach((field: string) => {
|
||||
if (field !== 'TAVILY_API_KEY') {
|
||||
result[field] =
|
||||
field === 'FIRECRAWL_API_URL' ? 'https://api.firecrawl.dev' : 'test-api-key';
|
||||
}
|
||||
});
|
||||
return Promise.resolve(result);
|
||||
});
|
||||
|
||||
const result = await loadWebSearchAuth({
|
||||
userId,
|
||||
webSearchConfig,
|
||||
loadAuthValues: mockLoadAuthValues,
|
||||
});
|
||||
|
||||
expect(result.authenticated).toBe(false);
|
||||
const providersAuthType = result.authTypes.find(
|
||||
([category]) => category === 'providers',
|
||||
)?.[1];
|
||||
expect(providersAuthType).toBe(AuthType.USER_PROVIDED);
|
||||
});
|
||||
|
||||
it('should authenticate Tavily as both search provider and scraper with a shared key', async () => {
|
||||
const webSearchConfig: TCustomConfig['webSearch'] = {
|
||||
tavilyApiKey: '${TAVILY_API_KEY}',
|
||||
tavilySearchUrl: '${TAVILY_SEARCH_URL}',
|
||||
tavilyExtractUrl: '${TAVILY_EXTRACT_URL}',
|
||||
safeSearch: SafeSearchTypes.MODERATE,
|
||||
searchProvider: 'tavily' as SearchProviders,
|
||||
scraperProvider: 'tavily' as ScraperProviders,
|
||||
rerankerType: 'none' as RerankerTypes,
|
||||
tavilyScraperOptions: {
|
||||
extractDepth: 'advanced',
|
||||
timeout: 20000,
|
||||
},
|
||||
};
|
||||
|
||||
mockLoadAuthValues.mockImplementation(({ authFields }) => {
|
||||
const result: Record<string, string> = {};
|
||||
authFields.forEach((field: string) => {
|
||||
if (field === 'TAVILY_API_KEY') {
|
||||
result[field] = 'tavily-api-key';
|
||||
} else if (field === 'TAVILY_SEARCH_URL') {
|
||||
result[field] = 'https://api.tavily.com/search';
|
||||
} else if (field === 'TAVILY_EXTRACT_URL') {
|
||||
result[field] = 'https://api.tavily.com/extract';
|
||||
}
|
||||
});
|
||||
return Promise.resolve(result);
|
||||
});
|
||||
|
||||
const result = await loadWebSearchAuth({
|
||||
userId,
|
||||
webSearchConfig,
|
||||
loadAuthValues: mockLoadAuthValues,
|
||||
});
|
||||
|
||||
expect(result.authenticated).toBe(true);
|
||||
expect(result.authResult.searchProvider).toBe('tavily');
|
||||
expect(result.authResult.scraperProvider).toBe('tavily');
|
||||
expect(result.authResult.tavilyApiKey).toBe('tavily-api-key');
|
||||
expect(result.authResult.tavilyScraperOptions).toEqual(webSearchConfig.tavilyScraperOptions);
|
||||
expect(result.authResult.scraperTimeout).toBe(20000);
|
||||
});
|
||||
|
||||
it('should only check the specified scraperProvider', async () => {
|
||||
// Initialize a webSearchConfig with a specific scraperProvider
|
||||
const webSearchConfig: TCustomConfig['webSearch'] = {
|
||||
|
|
@ -1165,6 +1289,76 @@ describe('web.ts', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('should not use tavilyScraperOptions.timeout for firecrawl scraper', async () => {
|
||||
const webSearchConfig = {
|
||||
serperApiKey: '${SERPER_API_KEY}',
|
||||
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
|
||||
firecrawlApiUrl: '${FIRECRAWL_API_URL}',
|
||||
tavilyApiKey: '${TAVILY_API_KEY}',
|
||||
jinaApiKey: '${JINA_API_KEY}',
|
||||
jinaApiUrl: '${JINA_API_URL}',
|
||||
safeSearch: SafeSearchTypes.MODERATE,
|
||||
scraperProvider: 'firecrawl' as ScraperProviders,
|
||||
tavilyScraperOptions: {
|
||||
timeout: 22000,
|
||||
},
|
||||
} as TCustomConfig['webSearch'];
|
||||
|
||||
mockLoadAuthValues.mockImplementation(({ authFields }) => {
|
||||
const result: Record<string, string> = {};
|
||||
authFields.forEach((field: string) => {
|
||||
result[field] =
|
||||
field === 'FIRECRAWL_API_URL' ? 'https://api.firecrawl.dev' : 'test-api-key';
|
||||
});
|
||||
return Promise.resolve(result);
|
||||
});
|
||||
|
||||
const result = await loadWebSearchAuth({
|
||||
userId,
|
||||
webSearchConfig,
|
||||
loadAuthValues: mockLoadAuthValues,
|
||||
});
|
||||
|
||||
expect(result.authenticated).toBe(true);
|
||||
expect(result.authResult.scraperTimeout).toBe(7500);
|
||||
});
|
||||
|
||||
it('should use tavilyScraperOptions.timeout for tavily scraper', async () => {
|
||||
const webSearchConfig = {
|
||||
serperApiKey: '${SERPER_API_KEY}',
|
||||
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
|
||||
firecrawlApiUrl: '${FIRECRAWL_API_URL}',
|
||||
tavilyApiKey: '${TAVILY_API_KEY}',
|
||||
jinaApiKey: '${JINA_API_KEY}',
|
||||
jinaApiUrl: '${JINA_API_URL}',
|
||||
safeSearch: SafeSearchTypes.MODERATE,
|
||||
scraperProvider: 'tavily' as ScraperProviders,
|
||||
firecrawlOptions: {
|
||||
timeout: 12000,
|
||||
},
|
||||
tavilyScraperOptions: {
|
||||
timeout: 22000,
|
||||
},
|
||||
} as TCustomConfig['webSearch'];
|
||||
|
||||
mockLoadAuthValues.mockImplementation(({ authFields }) => {
|
||||
const result: Record<string, string> = {};
|
||||
authFields.forEach((field: string) => {
|
||||
result[field] =
|
||||
field === 'FIRECRAWL_API_URL' ? 'https://api.firecrawl.dev' : 'test-api-key';
|
||||
});
|
||||
return Promise.resolve(result);
|
||||
});
|
||||
|
||||
const result = await loadWebSearchAuth({
|
||||
userId,
|
||||
webSearchConfig,
|
||||
loadAuthValues: mockLoadAuthValues,
|
||||
});
|
||||
|
||||
expect(result.authResult.scraperTimeout).toBe(22000);
|
||||
});
|
||||
|
||||
it('should handle firecrawlOptions.formats when only formats is provided', async () => {
|
||||
// Initialize a webSearchConfig with only firecrawlOptions.formats
|
||||
const webSearchConfig = {
|
||||
|
|
@ -1359,6 +1553,81 @@ describe('web.ts', () => {
|
|||
expect(scrapersAuth).toBe(AuthType.USER_PROVIDED);
|
||||
});
|
||||
|
||||
it('should block user-provided tavilySearchUrl targeting localhost', async () => {
|
||||
mockIsSSRFTarget.mockImplementation((hostname: string) => hostname === 'localhost');
|
||||
|
||||
const webSearchConfig: TCustomConfig['webSearch'] = {
|
||||
tavilyApiKey: '${TAVILY_API_KEY}',
|
||||
tavilySearchUrl: '${TAVILY_SEARCH_URL}',
|
||||
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
|
||||
safeSearch: SafeSearchTypes.MODERATE,
|
||||
searchProvider: 'tavily' as SearchProviders,
|
||||
rerankerType: 'none' as RerankerTypes,
|
||||
};
|
||||
|
||||
mockLoadAuthValues.mockImplementation(({ authFields }) => {
|
||||
const result: Record<string, string> = {};
|
||||
authFields.forEach((field: string) => {
|
||||
if (field === 'TAVILY_SEARCH_URL') {
|
||||
result[field] = 'http://localhost:8080/search';
|
||||
} else {
|
||||
result[field] = 'test-api-key';
|
||||
}
|
||||
});
|
||||
return Promise.resolve(result);
|
||||
});
|
||||
|
||||
const result = await loadWebSearchAuth({
|
||||
userId,
|
||||
webSearchConfig,
|
||||
loadAuthValues: mockLoadAuthValues,
|
||||
});
|
||||
|
||||
expect(result.authResult.tavilySearchUrl).toBeUndefined();
|
||||
expect(result.authResult.searchProvider).toBe('tavily');
|
||||
expect(result.authenticated).toBe(true);
|
||||
expect(mockIsSSRFTarget).toHaveBeenCalledWith('localhost');
|
||||
});
|
||||
|
||||
it('should block user-provided tavilyExtractUrl resolving to private IP', async () => {
|
||||
mockResolveHostnameSSRF.mockImplementation((hostname: string) =>
|
||||
Promise.resolve(hostname === 'extract.internal-service.com'),
|
||||
);
|
||||
|
||||
const webSearchConfig: TCustomConfig['webSearch'] = {
|
||||
serperApiKey: '${SERPER_API_KEY}',
|
||||
tavilyApiKey: '${TAVILY_API_KEY}',
|
||||
tavilyExtractUrl: '${TAVILY_EXTRACT_URL}',
|
||||
safeSearch: SafeSearchTypes.MODERATE,
|
||||
scraperProvider: 'tavily' as ScraperProviders,
|
||||
rerankerType: 'none' as RerankerTypes,
|
||||
};
|
||||
|
||||
mockLoadAuthValues.mockImplementation(({ authFields }) => {
|
||||
const result: Record<string, string> = {};
|
||||
authFields.forEach((field: string) => {
|
||||
if (field === 'TAVILY_EXTRACT_URL') {
|
||||
result[field] = 'https://extract.internal-service.com/extract';
|
||||
} else {
|
||||
result[field] = 'test-api-key';
|
||||
}
|
||||
});
|
||||
return Promise.resolve(result);
|
||||
});
|
||||
|
||||
const result = await loadWebSearchAuth({
|
||||
userId,
|
||||
webSearchConfig,
|
||||
loadAuthValues: mockLoadAuthValues,
|
||||
});
|
||||
|
||||
expect(result.authResult.tavilyExtractUrl).toBeUndefined();
|
||||
expect(result.authResult.scraperProvider).toBe('tavily');
|
||||
expect(result.authenticated).toBe(true);
|
||||
const scrapersAuth = result.authTypes.find(([c]) => c === 'scrapers')?.[1];
|
||||
expect(scrapersAuth).toBe(AuthType.USER_PROVIDED);
|
||||
});
|
||||
|
||||
it('should block user-provided searxngInstanceUrl targeting metadata endpoint', async () => {
|
||||
mockIsSSRFTarget.mockImplementation((hostname: string) => hostname === '169.254.169.254');
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,8 @@ const WEB_SEARCH_URL_KEYS = new Set<TWebSearchKeys>([
|
|||
'searxngInstanceUrl',
|
||||
'firecrawlApiUrl',
|
||||
'jinaApiUrl',
|
||||
'tavilySearchUrl',
|
||||
'tavilyExtractUrl',
|
||||
]);
|
||||
|
||||
/**
|
||||
|
|
@ -245,10 +247,20 @@ export async function loadWebSearchAuth({
|
|||
authTypes.push([category, isUserProvided ? AuthType.USER_PROVIDED : AuthType.SYSTEM_DEFINED]);
|
||||
}
|
||||
|
||||
const scraperProvider =
|
||||
authResult.scraperProvider ?? webSearchConfig?.scraperProvider ?? 'firecrawl';
|
||||
let scraperOptionsTimeout: number | undefined;
|
||||
if (scraperProvider === 'tavily') {
|
||||
scraperOptionsTimeout = webSearchConfig?.tavilyScraperOptions?.timeout;
|
||||
} else if (scraperProvider === 'firecrawl') {
|
||||
scraperOptionsTimeout = webSearchConfig?.firecrawlOptions?.timeout;
|
||||
}
|
||||
|
||||
authResult.safeSearch = webSearchConfig?.safeSearch ?? SafeSearchTypes.MODERATE;
|
||||
authResult.scraperTimeout =
|
||||
webSearchConfig?.scraperTimeout ?? webSearchConfig?.firecrawlOptions?.timeout ?? 7500;
|
||||
authResult.scraperTimeout = webSearchConfig?.scraperTimeout ?? scraperOptionsTimeout ?? 7500;
|
||||
authResult.firecrawlOptions = webSearchConfig?.firecrawlOptions;
|
||||
authResult.tavilySearchOptions = webSearchConfig?.tavilySearchOptions;
|
||||
authResult.tavilyScraperOptions = webSearchConfig?.tavilyScraperOptions;
|
||||
|
||||
return {
|
||||
authTypes,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue