Coverage for app/venv/lib/python3.14/site-packages/weblate/addons/tasks.py: 35%
123 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-10-07 07:15 +0000
1# Copyright © Michal Čihař <michal@weblate.org>
2#
3# SPDX-License-Identifier: GPL-3.0-or-later
5from __future__ import annotations
7import os
8from datetime import timedelta
10from celery.schedules import crontab
11from django.conf import settings
12from django.db import transaction
13from django.db.models import Count, F, Q
14from django.http import HttpRequest
15from django.utils import timezone
16from django.utils.timezone import now
17from lxml import html
19from weblate.addons.events import AddonEvent
20from weblate.addons.models import Addon, AddonActivityLog, handle_addon_event
21from weblate.lang.models import Language
22from weblate.trans.exceptions import FileParseError
23from weblate.trans.models import Change, Component, Project
24from weblate.utils.celery import app
25from weblate.utils.hash import calculate_checksum
26from weblate.utils.lock import WeblateLockTimeoutError
27from weblate.utils.requests import request
29IGNORED_TAGS = {"script", "style"}
32@app.task(trail=False)
33def cdn_parse_html(addon_id: int, component_id: int) -> None:
34 try:
35 addon = Addon.objects.get(pk=addon_id)
36 except Addon.DoesNotExist:
37 return
39 component = Component.objects.get(pk=component_id)
40 source_translation = component.source_translation
41 source_units = set(source_translation.unit_set.values_list("source", flat=True))
42 units = []
43 errors = []
45 for filename in addon.configuration["files"].splitlines():
46 filename = filename.strip()
47 try:
48 if filename.startswith(("http://", "https://")):
49 with request("get", filename) as handle:
50 content = handle.text
51 else:
52 with open(os.path.join(component.full_path, filename)) as handle:
53 content = handle.read()
54 except OSError as error:
55 errors.append({"filename": filename, "error": str(error)})
56 continue
58 document = html.fromstring(content)
60 for element in document.cssselect(addon.configuration["css_selector"]):
61 text = element.text
62 if (
63 element.getchildren()
64 or element.tag in IGNORED_TAGS
65 or not text
66 or not text.strip()
67 or text in source_units
68 or text in units
69 ):
70 continue
71 units.append(text)
73 # Actually create units
74 for text in units:
75 source_translation.add_unit(
76 request=None,
77 context=calculate_checksum(text),
78 source=text,
79 target=None,
80 author=addon.addon.user,
81 )
83 if errors:
84 component.add_alert("CDNAddonError", occurrences=errors)
85 else:
86 component.delete_alert("CDNAddonError")
89@app.task(
90 trail=False,
91 autoretry_for=(WeblateLockTimeoutError,),
92 retry_backoff=600,
93 retry_backoff_max=3600,
94)
95@transaction.atomic
96def language_consistency(
97 addon_id: int,
98 language_ids: list[int],
99 project_id: int,
100 activity_log_id: int | None = None,
101) -> None:
102 try:
103 addon = Addon.objects.get(pk=addon_id)
104 except Addon.DoesNotExist:
105 return
106 project = Project.objects.get(pk=project_id)
107 languages = Language.objects.filter(id__in=language_ids)
108 request = HttpRequest()
109 request.user = addon.addon.user
111 # Filter components with missing translation
112 components = project.component_set.annotate(
113 translation_count=Count(
114 "translation", filter=Q(translation__language__in=languages)
115 )
116 ).exclude(translation_count=languages.count())
118 log_result: list[str] = []
120 for component in components.iterator():
121 # Avoid two language consistency add-ons working at same on a single component
122 with component.lock:
123 missing = languages.exclude(
124 Q(translation__component=component) | Q(component=component)
125 )
126 if not missing:
127 continue
128 component.commit_pending("language consistency", None)
129 for language in missing:
130 new_lang = component.add_new_language(
131 language,
132 request,
133 send_signal=False,
134 create_translations=False,
135 )
136 if new_lang is None:
137 log_result.append(
138 f"{component.full_slug}: Could not add {language} language consistency: {component.new_lang_error_message}"
139 )
140 else:
141 log_result.append(
142 f"{component.full_slug}: Added {language} for language consistency"
143 )
144 try:
145 component.create_translations_immediate()
146 except FileParseError as error:
147 log_result.append(
148 f"{component.full_slug}: Could not parse translation files: {error}"
149 )
151 if activity_log_id and log_result:
152 update_addon_activity_log(activity_log_id, "\n".join(log_result))
155@app.task(trail=False)
156def daily_addons(modulo: bool = True) -> None:
157 def daily_callback(
158 addon: Addon, component: Component, *, activity_log_id: int | None = None
159 ) -> None:
160 addon.addon.daily(component, activity_log_id=activity_log_id)
162 today = timezone.now()
163 addons = Addon.objects.filter(event__event=AddonEvent.EVENT_DAILY)
164 if modulo: 164 ↛ 166line 164 didn't jump to line 166 because the condition on line 164 was always true
165 addons = addons.annotate(hourmod=F("id") % 24).filter(hourmod=today.hour)
166 handle_addon_event(
167 AddonEvent.EVENT_DAILY,
168 daily_callback,
169 addon_queryset=addons,
170 auto_scope=True,
171 )
174def update_addon_activity_log(
175 pk: int, result: str = "", error_occurred: bool = False, pending: bool | None = None
176) -> None:
177 addon_activity_log = AddonActivityLog.objects.select_for_update().get(id=pk)
178 addon_activity_log.details["error"] = error_occurred
179 if result:
180 addon_activity_log.update_result(result)
181 if pending is not None:
182 addon_activity_log.pending = pending
183 addon_activity_log.save(update_fields=["details", "pending"])
186@app.task(trail=False)
187def cleanup_addon_activity_log() -> None:
188 """Cleanup old add-on activity log entries."""
189 from weblate.addons.models import AddonActivityLog
191 AddonActivityLog.objects.filter(
192 created__lt=now() - timedelta(days=settings.ADDON_ACTIVITY_LOG_EXPIRY)
193 ).delete()
196@app.task(
197 trail=False,
198 autoretry_for=(WeblateLockTimeoutError,),
199 retry_backoff=60,
200)
201@transaction.atomic
202def postconfigure_addon(addon_id: int, addon=None) -> None:
203 if addon is None:
204 addon = Addon.objects.get(pk=addon_id)
205 addon.addon.post_configure_run()
208@app.on_after_finalize.connect
209def setup_periodic_tasks(sender, **kwargs) -> None:
210 sender.add_periodic_task(crontab(minute=45), daily_addons.s(), name="daily-addons")
211 sender.add_periodic_task(
212 crontab(hour=0, minute=40), # Not to run on minute 0 to spread the load
213 cleanup_addon_activity_log.s(),
214 name="cleanup-addon-activity-log",
215 )
218@app.task(trail=True)
219def addon_change(change_ids: list[int], **kwargs) -> None:
220 """
221 Process add-on change events for a list of changes.
223 This task retrieves add-ons that are subscribed to change events and
224 applies the change event to each relevant add-on.
225 """
226 addons = Addon.objects.filter(event__event=AddonEvent.EVENT_CHANGE).select_related(
227 "component", "project"
228 )
230 for change in Change.objects.filter(pk__in=change_ids).prefetch_for_render():
231 change.fill_in_prefetched()
232 # Filter addons for this change
233 change_addons = [
234 addon
235 for addon in addons
236 if (not addon.component or addon.component == change.component)
237 and (not addon.project or addon.project == change.project)
238 ]
239 if change_addons: 239 ↛ 240line 239 didn't jump to line 240 because the condition on line 239 was never true
240 handle_addon_event(
241 AddonEvent.EVENT_CHANGE,
242 "change_event",
243 (change,),
244 addon_queryset=change_addons,
245 project=change.project,
246 component=change.component,
247 translation=change.translation,
248 )