Coverage for open_webui/retrieval/web/main.py: 23%

1from __future__ import annotations 

2 

3from urllib.parse import urlparse 

4 

5import validators 

6from open_webui.retrieval.web.utils import resolve_hostname 

7from open_webui.utils.misc import as_network, get_allow_block_lists, is_host_allowed 

8from pydantic import BaseModel 

9 

10 

11def get_filtered_results(results, filter_list): 

12 if not filter_list: 

13 return results 

14 

15 allow_list, block_list = get_allow_block_lists(filter_list) 

16 # Only worth a lookup when an entry names an address, since a hostname entry matches by name. 

17 resolve_ips = any(as_network(entry) is not None for entry in allow_list + block_list) 

18 

19 filtered_results = [] 

20 

21 for result in results: 

22 url = result.get('url') or result.get('link', '') or result.get('href', '') 

23 if not validators.url(url): 

24 continue 

25 

26 domain = urlparse(url).hostname 

27 if not domain: 

28 continue 

29 

30 hostnames = [domain] 

31 

32 if resolve_ips: 

33 try: 

34 ipv4_addresses, ipv6_addresses = resolve_hostname(domain) 

35 hostnames.extend(ipv4_addresses) 

36 hostnames.extend(ipv6_addresses) 

37 except Exception: 

38 pass 

39 

40 if is_host_allowed(hostnames, filter_list): 

41 filtered_results.append(result) 

42 continue 

43 

44 return filtered_results 

45 

46 

47class SearchResult(BaseModel): 

48 link: str 

49 title: str | None 

50 snippet: str | None