Coverage for open_webui/retrieval/web/ydc.py: 23%

1import logging 

2from typing import List, Optional 

3 

4import requests 

5from open_webui.retrieval.web.main import SearchResult, get_filtered_results 

6 

7log = logging.getLogger(__name__) 

8 

9 

10def search_youcom( 

11 api_key: str, 

12 query: str, 

13 count: int, 

14 filter_list: Optional[List[str]] = None, 

15 language: str = 'EN', 

16) -> List[SearchResult]: 

17 """Search using You.com's YDC Index API and return the results as a list of SearchResult objects. 

18 

19 Args: 

20 api_key (str): A You.com API key 

21 query (str): The query to search for 

22 count (int): Maximum number of results to return 

23 filter_list (list[str], optional): Domain filter list 

24 language (str): Language code for search results (default: "EN") 

25 """ 

26 url = 'https://ydc-index.io/v1/search' 

27 headers = { 

28 'Accept': 'application/json', 

29 'X-API-KEY': api_key, 

30 } 

31 params = { 

32 'query': query, 

33 'count': count, 

34 'language': language, 

35 } 

36 

37 response = requests.get(url, headers=headers, params=params) 

38 response.raise_for_status() 

39 

40 json_response = response.json() 

41 results = json_response.get('results', {}).get('web', []) 

42 

43 if filter_list: 

44 results = get_filtered_results(results, filter_list) 

45 

46 return [ 

47 SearchResult( 

48 link=result['url'], 

49 title=result.get('title'), 

50 snippet=_build_snippet(result), 

51 ) 

52 for result in results[:count] 

53 ] 

54 

55 

56def _build_snippet(result: dict) -> str: 

57 """Combine the description and snippets list into a single string. 

58 

59 The You.com API returns a short ``description`` plus a ``snippets`` 

60 list with richer passages. Merging them gives downstream retrieval 

61 (embedding, BM25, bypass-loader context) the most content to work with. 

62 """ 

63 parts: list[str] = [] 

64 

65 description = result.get('description') 

66 if description: 

67 parts.append(description) 

68 

69 snippets = result.get('snippets') 

70 if snippets and isinstance(snippets, list): 

71 parts.extend(snippets) 

72 

73 return '\n\n'.join(parts)