Coverage for opt/mealie/lib/python3.12/site-packages/mealie/services/scraper/recipe_bulk_scraper.py: 73%

1import asyncio 1 ctx1b

2 

3from pydantic import UUID4 1 ctx1b

4 

5from mealie.lang.providers import Translator 1 ctx1b

6from mealie.repos.repository_factory import AllRepositories 1 ctx1b

7from mealie.schema.recipe.recipe import CreateRecipeByUrlBulk, Recipe 1 ctx1b

8from mealie.schema.reports.reports import ( 1 ctx1b

9 ReportCategory, 

10 ReportCreate, 

11 ReportEntryCreate, 

12 ReportEntryOut, 

13 ReportSummaryStatus, 

14) 

15from mealie.schema.user.user import GroupInDB 1 ctx1b

16from mealie.services._base_service import BaseService 1 ctx1b

17from mealie.services.recipe.recipe_service import RecipeService 1 ctx1b

18from mealie.services.scraper.scraper import create_from_html 1 ctx1b

19 

20 

21class RecipeBulkScraperService(BaseService): 1 ctx1b

22 report_entries: list[ReportEntryCreate] 1 ctx1b

23 

24 def __init__( 1 ctx1b

25 self, service: RecipeService, repos: AllRepositories, group: GroupInDB, translator: Translator 

26 ) -> None: 

27 self.service = service (empty)

28 self.repos = repos (empty)

29 self.group = group (empty)

30 self.report_entries = [] (empty)

31 self.translator = translator (empty)

32 

33 super().__init__() (empty)

34 

35 def get_report_id(self) -> UUID4: 1 ctx1b

36 import_report = ReportCreate( (empty)

37 name="Bulk Import", 

38 category=ReportCategory.bulk_import, 

39 status=ReportSummaryStatus.in_progress, 

40 group_id=self.group.id, 

41 ) 

42 

43 self.report = self.repos.group_reports.create(import_report) (empty)

44 return self.report.id (empty)

45 

46 def _add_error_entry(self, message: str, exception: str = "") -> None: 1 ctx1b

47 self.report_entries.append( (empty)

48 ReportEntryCreate( 

49 report_id=self.report.id, 

50 success=False, 

51 message=message, 

52 exception=exception, 

53 ) 

54 ) 

55 

56 def _save_all_entries(self) -> None: 1 ctx1b

57 is_success = True (empty)

58 is_failure = True (empty)

59 

60 new_entries: list[ReportEntryOut] = [] (empty)

61 for entry in self.report_entries: (empty)

62 if is_failure and entry.success: 62 ↛ 63line 62 didn't jump to line 63 because the condition on line 62 was never true(empty)

63 is_failure = False 

64 

65 if is_success and not entry.success: 65 ↛ 68line 65 didn't jump to line 68 because the condition on line 65 was always true(empty)

66 is_success = False (empty)

67 

68 new_entries.append(self.repos.group_report_entries.create(entry)) (empty)

69 

70 if is_success: (empty)

71 self.report.status = ReportSummaryStatus.success (empty)

72 

73 if is_failure: 73 ↛ 76line 73 didn't jump to line 76 because the condition on line 73 was always true(empty)

74 self.report.status = ReportSummaryStatus.failure (empty)

75 

76 if not is_success and not is_failure: 76 ↛ 77line 76 didn't jump to line 77 because the condition on line 76 was never true(empty)

77 self.report.status = ReportSummaryStatus.partial 

78 

79 self.report.entries = new_entries (empty)

80 self.repos.group_reports.update(self.report.id, self.report) (empty)

81 

82 async def scrape(self, urls: CreateRecipeByUrlBulk) -> None: 1 ctx1b

83 sem = asyncio.Semaphore(3) (empty)

84 

85 async def _do(url: str) -> Recipe | None: (empty)

86 async with sem: (empty)

87 try: (empty)

88 recipe, _ = await create_from_html(url, self.translator) (empty)

89 return recipe 

90 except Exception as e: (empty)

91 self.service.logger.error(f"failed to scrape url during bulk url import {url}") (empty)

92 self.service.logger.exception(e) (empty)

93 self._add_error_entry(f"failed to scrape url {url}", str(e)) (empty)

94 return None (empty)

95 

96 if self.report is None: 96 ↛ 97line 96 didn't jump to line 97 because the condition on line 96 was never true(empty)

97 self.get_report_id() 

98 tasks = [_do(b.url) for b in urls.imports] (empty)

99 results = await asyncio.gather(*tasks, return_exceptions=True) (empty)

100 for b, recipe in zip(urls.imports, results, strict=True): (empty)

101 if not recipe or isinstance(recipe, BaseException): 101 ↛ 104line 101 didn't jump to line 104 because the condition on line 101 was always true(empty)

102 continue (empty)

103 

104 if b.tags: 

105 recipe.tags = b.tags 

106 

107 if b.categories: 

108 recipe.recipe_category = b.categories 

109 

110 try: 

111 self.service.create_one(recipe) 

112 except Exception as e: 

113 self.service.logger.error(f"Failed to save recipe to database during bulk url import {b.url}") 

114 self.service.logger.exception(e) 

115 self._add_error_entry(f"Failed to save recipe to database during bulk url import {b.url}", str(e)) 

116 continue 

117 

118 self.report_entries.append( 

119 ReportEntryCreate( 

120 report_id=self.report.id, 

121 success=True, 

122 message=f"Successfully imported recipe {recipe.name}", 

123 exception="", 

124 ) 

125 ) 

126 

127 self._save_all_entries() (empty)