Coverage for open_webui/retrieval/loaders/microsoft_web_iq.py: 19%

1import logging 

2import time 

3from collections.abc import Iterator 

4from typing import Any 

5from urllib.parse import urlparse 

6 

7import requests 

8from langchain_core.document_loaders import BaseLoader 

9from langchain_core.documents import Document 

10 

11log = logging.getLogger(__name__) 

12 

13MICROSOFT_BROWSE_RETRY_STATUS_CODES = {202, 429, 500, 502, 503, 504} 

14MICROSOFT_BROWSE_MAX_RETRIES = 2 

15 

16 

17class MicrosoftWebIQLoader(BaseLoader): 

18 def __init__( 

19 self, 

20 urls: str | list[str], 

21 api_base_url: str, 

22 api_key: str, 

23 language: str = 'en', 

24 verify_ssl: bool = True, 

25 timeout: Any = None, 

26 continue_on_failure: bool = True, 

27 ) -> None: 

28 self.urls = urls if isinstance(urls, list) else [urls] 

29 self.api_base_url = api_base_url.rstrip('/') 

30 self.api_key = api_key 

31 self.language = language 

32 self.verify_ssl = verify_ssl 

33 self.timeout = timeout 

34 self.continue_on_failure = continue_on_failure 

35 

36 def lazy_load(self) -> Iterator[Document]: 

37 for url in self.urls: 

38 try: 

39 doc = self._browse_url(url) 

40 if doc is not None: 

41 yield doc 

42 except Exception as e: 

43 if self.continue_on_failure: 

44 log.warning(f'Error browsing {url} with Microsoft Web IQ: {e}') 

45 else: 

46 raise e 

47 

48 def _browse_url(self, url: str) -> Document | None: 

49 headers = { 

50 'host': urlparse(self.api_base_url).netloc or 'api.microsoft.ai', 

51 'x-apikey': self.api_key, 

52 'content-type': 'application/json', 

53 } 

54 payload = { 

55 'url': url, 

56 'contentFormat': 'markdown', 

57 'liveCrawl': 'fallback', 

58 'renderDynamicPages': True, 

59 'language': self.language, 

60 } 

61 try: 

62 request_timeout = float(self.timeout) 

63 except (TypeError, ValueError): 

64 request_timeout = 60 

65 request_timeout = request_timeout if request_timeout > 0 else 60 

66 

67 data: dict[str, Any] = {} 

68 for attempt in range(MICROSOFT_BROWSE_MAX_RETRIES + 1): 

69 response = requests.post( 

70 f'{self.api_base_url}/browse', 

71 json=payload, 

72 headers=headers, 

73 timeout=request_timeout, 

74 verify=self.verify_ssl, 

75 ) 

76 

77 if response.status_code in MICROSOFT_BROWSE_RETRY_STATUS_CODES and attempt < MICROSOFT_BROWSE_MAX_RETRIES: 

78 try: 

79 body = response.json() 

80 except Exception: 

81 body = {} 

82 retry_after = body.get('retryAfter') if isinstance(body, dict) else None 

83 retry_after = retry_after or response.headers.get('Retry-After') 

84 try: 

85 delay = min(10.0, max(0.0, float(str(retry_after).rstrip('s')))) 

86 except (TypeError, ValueError): 

87 delay = min(8.0, float(2**attempt)) 

88 log.warning( 

89 'Microsoft Browse %s returned HTTP %s; retrying in %.1fs', 

90 url, 

91 response.status_code, 

92 delay, 

93 ) 

94 time.sleep(delay) 

95 continue 

96 

97 response.raise_for_status() 

98 data = response.json() 

99 break 

100 

101 content = data.get('content') or '' 

102 if not isinstance(content, str) or not content.strip(): 

103 return None 

104 

105 metadata = {'source': data.get('url') or url} 

106 if data.get('title'): 

107 metadata['title'] = data['title'] 

108 

109 return Document(page_content=content, metadata=metadata)