Coverage for opt/mealie/lib/python3.12/site-packages/mealie/services/scraper/user_agents_manager.py: 86%

1from __future__ import annotations 1 ctx1b

2 

3import os 1 ctx1b

4import random 1 ctx1b

5 

6_USER_AGENTS_MANAGER: UserAgentsManager | None = None 1 ctx1b

7 

8 

9def get_user_agents_manager() -> UserAgentsManager: 1 ctx1b

10 global _USER_AGENTS_MANAGER 

11 

12 if not _USER_AGENTS_MANAGER: (empty)

13 _USER_AGENTS_MANAGER = UserAgentsManager() (empty)

14 

15 return _USER_AGENTS_MANAGER (empty)

16 

17 

18class UserAgentsManager: 1 ctx1b

19 def __init__(self) -> None: 1 ctx1b

20 self._user_agents: list[str] | None = None (empty)

21 self._user_agents_text_path = os.path.join(os.path.dirname(__file__), "user-agents.txt") (empty)

22 

23 def get_scrape_headers(self, user_agent: str | None = None) -> dict[str, str]: 1 ctx1b

24 # From: https://scrapeops.io/web-scraping-playbook/403-forbidden-error-web-scraping/#optimize-request-headers 

25 if user_agent is None: 25 ↛ 26line 25 didn't jump to line 26 because the condition on line 25 was never true(empty)

26 user_agent = random.choice(self.user_agents) 

27 

28 return { (empty)

29 "User-Agent": user_agent, 

30 "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", 

31 "Accept-Language": "en-US,en;q=0.5", 

32 "Accept-Encoding": "gzip, deflate", 

33 "Connection": "keep-alive", 

34 "Upgrade-Insecure-Requests": "1", 

35 "Sec-Fetch-Dest": "document", 

36 "Sec-Fetch-Mode": "navigate", 

37 "Sec-Fetch-Site": "none", 

38 "Sec-Fetch-User": "?1", 

39 "Cache-Control": "max-age=0", 

40 } 

41 

42 @property 1 ctx1b

43 def user_agents(self) -> list[str]: 1 ctx1b

44 if not self._user_agents: (empty)

45 self._user_agents = self._fetch_user_agents() (empty)

46 

47 return self._user_agents (empty)

48 

49 def _fetch_user_agents(self) -> list[str]: 1 ctx1b

50 user_agents: list[str] = [] (empty)

51 

52 try: (empty)

53 from recipe_scrapers._abstract import HEADERS (empty)

54 

55 user_agents.append(HEADERS["User-Agent"]) (empty)

56 except (ImportError, KeyError): 

57 user_agents.append("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/128.0") 

58 

59 with open(self._user_agents_text_path) as f: (empty)

60 for line in f: (empty)

61 if not line: 61 ↛ 62line 61 didn't jump to line 62 because the condition on line 61 was never true(empty)

62 continue 

63 user_agents.append(line.strip()) (empty)

64 

65 return user_agents (empty)