Coverage for open_webui/retrieval/web/searxng.py: 34%

1from __future__ import annotations 

2 

3import logging 

4import ssl 

5from functools import lru_cache 

6 

7from open_webui.env import AIOHTTP_CLIENT_SESSION_SSL, SEARXNG_CLIENT_CERT_FILE, SEARXNG_CLIENT_KEY_FILE 

8from open_webui.retrieval.web.main import SearchResult, get_filtered_results 

9from open_webui.utils.session_pool import get_session 

10 

11log = logging.getLogger(__name__) 

12 

13# SearXNG request headers — identifies the bot to instance operators. 

14_SEARXNG_HEADERS = { 

15 # LICENSE covers this Open WebUI user-agent identifier. 

16 # Do not alter, remove, obscure, or replace it except as LICENSE permits: 

17 # https://docs.openwebui.com/license. 

18 'User-Agent': 'Open WebUI (https://github.com/open-webui/open-webui) RAG Bot', 

19 'Accept': 'text/html', 

20 'Accept-Encoding': 'gzip, deflate', 

21 'Accept-Language': 'en-US,en;q=0.5', 

22 'Connection': 'keep-alive', 

23} 

24 

25 

26@lru_cache 

27def _get_ssl_context() -> bool | ssl.SSLContext: 

28 if not SEARXNG_CLIENT_CERT_FILE: 

29 return AIOHTTP_CLIENT_SESSION_SSL 

30 

31 ssl_context = ssl.create_default_context() 

32 ssl_context.load_cert_chain( 

33 certfile=SEARXNG_CLIENT_CERT_FILE, 

34 keyfile=SEARXNG_CLIENT_KEY_FILE or None, 

35 ) 

36 return ssl_context 

37 

38 

39async def search_searxng( 

40 query_url: str, 

41 query: str, 

42 count: int, 

43 filter_list: list[str] | None = None, 

44 **kwargs, 

45) -> list[SearchResult]: 

46 """Query a SearXNG instance and return results sorted by relevance score. 

47 

48 Optional keyword arguments (language, safesearch, time_range, categories) 

49 are forwarded directly as SearXNG query parameters. 

50 """ 

51 # Normalise legacy ``<query>``-style URLs by stripping any query string. 

52 if '<query>' in query_url: 

53 query_url = query_url.split('?')[0] 

54 

55 params = { 

56 'q': query, 

57 'format': 'json', 

58 'pageno': 1, 

59 'safesearch': kwargs.get('safesearch', '1'), 

60 'language': kwargs.get('language', 'all').strip().rstrip(','), 

61 'time_range': kwargs.get('time_range', ''), 

62 'categories': ''.join(kwargs.get('categories', [])), 

63 'theme': 'simple', 

64 'image_proxy': 0, 

65 } 

66 

67 log.debug('searching %s', query_url) 

68 

69 session = await get_session() 

70 async with session.get( 

71 query_url, 

72 headers=_SEARXNG_HEADERS, 

73 params=params, 

74 ssl=_get_ssl_context(), 

75 ) as response: 

76 response.raise_for_status() 

77 payload = await response.json() 

78 

79 results = sorted(payload.get('results', []), key=lambda x: x.get('score', 0), reverse=True) 

80 if filter_list: 

81 results = get_filtered_results(results, filter_list) 

82 

83 return [ 

84 SearchResult( 

85 link=item.get('url', ''), 

86 title=item.get('title'), 

87 snippet=item.get('content'), 

88 ) 

89 for item in results[:count] 

90 ]