Coverage for app/venv/lib/python3.14/site-packages/weblate/trans/tasks.py: 37%
386 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-10-07 07:15 +0000
1# Copyright © Michal Čihař <michal@weblate.org>
2#
3# SPDX-License-Identifier: GPL-3.0-or-later
5from __future__ import annotations
7import os
8import time
9from datetime import datetime, timedelta
10from glob import glob
11from operator import itemgetter
12from pathlib import Path
13from typing import TYPE_CHECKING, Literal
15from celery.schedules import crontab
16from django.conf import settings
17from django.contrib.staticfiles.storage import staticfiles_storage
18from django.core.cache import cache
19from django.db import IntegrityError, transaction
20from django.db.models import (
21 Count,
22 DateTimeField,
23 ExpressionWrapper,
24 F,
25 Value,
26)
27from django.db.models.functions import Now
28from django.http import Http404
29from django.utils import timezone
30from django.utils.timezone import make_aware
31from django.utils.translation import override
33from weblate.addons.models import Addon
34from weblate.auth.models import AuthenticatedHttpRequest, User, get_anonymous
35from weblate.lang.models import Language
36from weblate.logger import LOGGER
37from weblate.trans.actions import ActionEvents
38from weblate.trans.autotranslate import AutoTranslate
39from weblate.trans.exceptions import FileParseError
40from weblate.trans.functions import MySQLTimestampAdd
41from weblate.trans.models import (
42 Category,
43 Change,
44 Comment,
45 Component,
46 ComponentList,
47 Project,
48 Suggestion,
49 Translation,
50)
51from weblate.utils.celery import app
52from weblate.utils.data import data_dir
53from weblate.utils.db import using_postgresql
54from weblate.utils.errors import report_error
55from weblate.utils.files import remove_tree
56from weblate.utils.lock import WeblateLockTimeoutError
57from weblate.utils.stats import prefetch_stats
58from weblate.utils.views import parse_path
59from weblate.vcs.base import RepositoryError
61if TYPE_CHECKING: 61 ↛ 62line 61 didn't jump to line 62 because the condition on line 61 was never true
62 from collections.abc import Callable
64 from django.db.models import (
65 Expression,
66 )
69@app.task(
70 trail=False,
71 autoretry_for=(WeblateLockTimeoutError,),
72 retry_backoff=600,
73 retry_backoff_max=3600,
74)
75def perform_update(
76 cls: Literal["Project", "Component"],
77 pk: int,
78 auto: bool = False,
79 obj=None,
80 user_id: int = 0,
81) -> None:
82 request: AuthenticatedHttpRequest | None = None
83 if user_id:
84 request = AuthenticatedHttpRequest()
85 request.user = User.objects.get(pk=user_id)
86 try:
87 if obj is None:
88 if cls == "Project":
89 obj = Project.objects.get(pk=pk)
90 else:
91 obj = Component.objects.get(pk=pk)
92 if settings.AUTO_UPDATE in {"full", True} or not auto:
93 obj.do_update(request)
94 else:
95 obj.update_remote_branch()
96 except (FileParseError, RepositoryError, FileNotFoundError):
97 # This is stored as alert, so we can silently ignore here
98 return
101@app.task(
102 trail=False,
103 autoretry_for=(WeblateLockTimeoutError,),
104 retry_backoff=600,
105 retry_backoff_max=3600,
106)
107def perform_load(
108 pk: int,
109 *,
110 force: bool = False,
111 force_scan: bool = False,
112 langs: list[str] | None = None,
113 changed_template: bool = False,
114 from_link: bool = False,
115 change: int | None = None,
116 user_id: int | None = None,
117) -> None:
118 request: AuthenticatedHttpRequest | None = None
119 if user_id:
120 request = AuthenticatedHttpRequest()
121 request.user = User.objects.get(pk=user_id)
122 component = Component.objects.get(pk=pk)
123 component.create_translations_immediate(
124 force=force,
125 force_scan=force_scan,
126 langs=langs,
127 changed_template=changed_template,
128 from_link=from_link,
129 change=change,
130 request=request,
131 )
134@app.task(
135 trail=False,
136 autoretry_for=(WeblateLockTimeoutError,),
137 retry_backoff=600,
138 retry_backoff_max=3600,
139)
140def perform_commit(
141 pk,
142 reason: str,
143 *,
144 user_id: int | None = None,
145 force_scan: bool = False,
146 previous_head: str | None = None,
147) -> None:
148 user = User.objects.get(pk=user_id) if user_id else None
149 component = Component.objects.get(pk=pk)
150 with component.repository.lock:
151 component.commit_pending(reason, user=user)
152 if force_scan:
153 component.trigger_post_update(
154 previous_head=previous_head,
155 skip_push=False,
156 )
157 component.create_translations(force=True)
160@app.task(
161 trail=False,
162 autoretry_for=(WeblateLockTimeoutError,),
163 retry_backoff=600,
164 retry_backoff_max=3600,
165)
166def perform_push(pk, *args, **kwargs) -> None:
167 component = Component.objects.get(pk=pk)
168 component.do_push(*args, **kwargs)
171@app.task(trail=False)
172def commit_pending(
173 hours: int | None = None,
174 pks: set[int] | None = None,
175 logger: Callable[[str], None] | None = None,
176) -> None:
177 if pks is None: 177 ↛ 180line 177 didn't jump to line 180 because the condition on line 177 was always true
178 components = Component.objects.all()
179 else:
180 components = Component.objects.filter(translation__pk__in=pks)
182 hours_expr = F("commit_pending_age") if hours is None else Value(hours)
183 age_cutoff: Expression
184 if using_postgresql(): 184 ↛ 189line 184 didn't jump to line 189 because the condition on line 184 was always true
185 age_cutoff = ExpressionWrapper(
186 Now() - hours_expr * timedelta(hours=1), output_field=DateTimeField()
187 )
188 else:
189 age_cutoff = MySQLTimestampAdd("HOUR", -hours_expr, Now())
191 components = components.filter(
192 translation__unit__pending_changes__timestamp__lt=age_cutoff,
193 ).distinct()
195 for component in prefetch_stats(components.prefetch()): 195 ↛ 196line 195 didn't jump to line 196 because the loop on line 195 never started
196 if logger:
197 logger(f"Committing {component}")
199 perform_commit.delay(component.pk, "commit_pending")
202@app.task(trail=False)
203def cleanup_component(pk) -> None:
204 """
205 Perform cleanup of component models.
207 - Remove stale source Unit objects.
208 - Update variants.
209 """
210 try:
211 component = Component.objects.get(pk=pk)
212 except Component.DoesNotExist:
213 return
215 # Skip monolingual components, these handle cleanups based on the template
216 if component.template: 216 ↛ 217line 216 didn't jump to line 217 because the condition on line 216 was never true
217 return
219 # Remove stale variants
220 with transaction.atomic():
221 component.update_variants()
223 translation = component.source_translation
224 # Skip translations with a filename (eg. when POT file is present)
225 if translation.filename: 225 ↛ 229line 225 didn't jump to line 229 because the condition on line 225 was always true
226 return
228 # Remove all units where there is just one referenced unit (self)
229 with transaction.atomic():
230 deleted, details = (
231 translation.unit_set.annotate(Count("unit"))
232 .filter(unit__count__lte=1)
233 .delete()
234 )
235 if deleted:
236 translation.log_info("removed leaf units: %s", details)
239@app.task(trail=False)
240def cleanup_suggestions() -> None:
241 # Process suggestions
242 anonymous_user = get_anonymous()
243 suggestions = Suggestion.objects.prefetch_related("unit")
244 for suggestion in suggestions:
245 with transaction.atomic():
246 # Remove suggestions with same text as real translation
247 if (
248 suggestion.unit.target == suggestion.target
249 and suggestion.unit.translated
250 ):
251 suggestion.delete_log(
252 anonymous_user, change=ActionEvents.SUGGESTION_CLEANUP
253 )
254 continue
256 # Remove duplicate suggestions
257 sugs = Suggestion.objects.filter(
258 unit=suggestion.unit, target=suggestion.target
259 ).exclude(id=suggestion.id)
260 # Do not rely on the SQL as MySQL compares strings case insensitive
261 for other in sugs:
262 if other.target == suggestion.target:
263 suggestion.delete_log(
264 anonymous_user, change=ActionEvents.SUGGESTION_CLEANUP
265 )
266 break
269@app.task(trail=False)
270def update_remotes() -> None:
271 """Update all remote branches (without attempt to merge)."""
272 if settings.AUTO_UPDATE not in {"full", "remote", True, False}: 272 ↛ 273line 272 didn't jump to line 273 because the condition on line 272 was never true
273 return
275 now = timezone.now()
276 components = (
277 Component.objects.with_repo()
278 .annotate(hourmod=F("id") % 24)
279 .filter(hourmod=now.hour)
280 )
281 for component in components.prefetch().iterator(chunk_size=100): 281 ↛ 282line 281 didn't jump to line 282 because the loop on line 281 never started
282 perform_update("Component", -1, auto=True, obj=component)
285@app.task(trail=False)
286def cleanup_stale_repos(root: Path | None = None) -> bool:
287 vcs_root = Path(data_dir("vcs"))
288 if root is None:
289 root = vcs_root
291 yesterday = time.time() - 86400
293 empty_dir = True
294 for path in root.glob("*"):
295 if not path.is_dir():
296 empty_dir = False
297 # Possibly a lock file
298 continue
299 git_dir = path / ".git"
300 mercurial_dir = path / ".hg"
301 if not git_dir.exists() and not mercurial_dir.exists():
302 # Category dir
303 if not cleanup_stale_repos(path):
304 empty_dir = False
305 continue
307 # Skip recently modified paths
308 if path.stat().st_mtime > yesterday:
309 empty_dir = False
310 continue
312 try:
313 # Find matching components
314 component: Component = parse_path(
315 None, path.relative_to(vcs_root).parts, (Component,)
316 )
317 except Http404:
318 # Remove stale dir
319 LOGGER.info("removing stale VCS path (not found): %s", path)
320 remove_tree(path)
321 else:
322 if component.is_repo_link:
323 LOGGER.info("removing stale VCS path (uses link): %s", root)
324 remove_tree(path)
325 else:
326 empty_dir = False
328 if empty_dir and root != vcs_root:
329 try:
330 # Find matching components
331 parse_path(None, root.relative_to(vcs_root).parts, (Category, Project))
332 except Http404:
333 LOGGER.info("removing stale VCS path (not found): %s", root)
334 root.rmdir()
335 else:
336 empty_dir = False
337 return empty_dir
340@app.task(trail=False)
341def cleanup_old_suggestions() -> None:
342 if not settings.SUGGESTION_CLEANUP_DAYS:
343 return
344 cutoff = timezone.now() - timedelta(days=settings.SUGGESTION_CLEANUP_DAYS)
345 Suggestion.objects.filter(timestamp__lt=cutoff).delete()
348@app.task(trail=False)
349def cleanup_old_comments() -> None:
350 if not settings.COMMENT_CLEANUP_DAYS:
351 return
352 cutoff = timezone.now() - timedelta(days=settings.COMMENT_CLEANUP_DAYS)
353 Comment.objects.filter(timestamp__lt=cutoff).delete()
356@app.task(trail=False)
357def repository_alerts(threshold=settings.REPOSITORY_ALERT_THRESHOLD) -> None:
358 non_linked = Component.objects.with_repo()
359 for component in non_linked.iterator():
360 try:
361 if component.repository.count_missing() > threshold:
362 component.add_alert("RepositoryOutdated")
363 else:
364 component.delete_alert("RepositoryOutdated")
365 if component.repository.count_outgoing() > threshold:
366 component.add_alert("RepositoryChanges")
367 else:
368 component.delete_alert("RepositoryChanges")
369 except RepositoryError as error:
370 report_error("Could not check repository status", project=component.project)
371 component.add_alert("MergeFailure", error=component.error_text(error))
374@app.task(trail=False)
375def component_alerts(component_ids=None) -> None:
376 if component_ids:
377 components = Component.objects.filter(pk__in=component_ids)
378 else:
379 now = timezone.now()
380 components = Component.objects.annotate(hourmod=F("id") % 24).filter(
381 hourmod=now.hour
382 )
383 for component in components.prefetch().iterator(chunk_size=100):
384 with transaction.atomic():
385 component.update_alerts()
388@app.task(
389 trail=False,
390 autoretry_for=(Component.DoesNotExist, WeblateLockTimeoutError),
391 retry_backoff=60,
392)
393@transaction.atomic
394def component_after_save(
395 pk: int,
396 changed_git: bool,
397 changed_setup: bool,
398 changed_template: bool,
399 changed_variant: bool,
400 changed_enforced_checks: bool,
401 skip_push: bool,
402 create: bool,
403) -> dict[Literal["component"], int]:
404 component = Component.objects.get(pk=pk)
405 component.after_save(
406 changed_git=changed_git,
407 changed_setup=changed_setup,
408 changed_template=changed_template,
409 changed_variant=changed_variant,
410 changed_enforced_checks=changed_enforced_checks,
411 skip_push=skip_push,
412 create=create,
413 )
414 return {"component": pk}
417@app.task(
418 trail=False,
419 autoretry_for=(Component.DoesNotExist, WeblateLockTimeoutError),
420 retry_backoff=60,
421)
422@transaction.atomic
423def update_enforced_checks(component: int | Component) -> None:
424 if isinstance(component, int):
425 component = Component.objects.get(pk=component)
426 component.update_enforced_checks()
429@app.task(trail=False)
430@transaction.atomic
431def component_removal(pk: int, uid: int) -> None:
432 user = User.objects.get(pk=uid)
433 try:
434 component = Component.objects.get(pk=pk)
435 except Component.DoesNotExist:
436 return
437 component.acting_user = user
438 component.project.change_set.create(
439 action=ActionEvents.REMOVE_COMPONENT,
440 target=component.slug,
441 user=user,
442 author=user,
443 )
444 component.delete()
445 if component.allow_translation_propagation:
446 components = component.project.component_set.filter(
447 allow_translation_propagation=True
448 ).exclude(pk=component.pk)
449 for component in components.iterator():
450 component.schedule_update_checks()
453@app.task(trail=False)
454@transaction.atomic
455def category_removal(pk: int, uid: int) -> None:
456 user = User.objects.get(pk=uid)
457 try:
458 category = Category.objects.get(pk=pk)
459 except Category.DoesNotExist:
460 return
461 for child in category.category_set.all(): 461 ↛ 462line 461 didn't jump to line 462 because the loop on line 461 never started
462 category_removal(child.pk, uid)
463 for component_id in category.component_set.values_list("id", flat=True): 463 ↛ 464line 463 didn't jump to line 464 because the loop on line 463 never started
464 component_removal(component_id, uid)
465 category.project.change_set.create(
466 action=ActionEvents.REMOVE_CATEGORY,
467 target=category.slug,
468 user=user,
469 author=user,
470 )
471 category.delete()
474@app.task(
475 trail=False,
476 autoretry_for=(IntegrityError,),
477 retry_backoff=600,
478 retry_backoff_max=3600,
479)
480def actual_project_removal(pk: int, uid: int | None) -> None:
481 """
482 Remove project.
484 This is separated from project_removal to allow retry on integrity errors.
485 """
486 with transaction.atomic():
487 user = get_anonymous() if uid is None else User.objects.get(pk=uid)
488 try:
489 project = Project.objects.get(pk=pk)
490 except Project.DoesNotExist:
491 return
492 Change.objects.create(
493 action=ActionEvents.REMOVE_PROJECT,
494 target=project.slug,
495 user=user,
496 author=user,
497 )
498 project.delete()
499 transaction.on_commit(project.stats.update_parents)
502@app.task(trail=False)
503def project_removal(pk: int, uid: int | None) -> None:
504 """Backup project and schedule actual removal."""
505 create_project_backup(pk)
506 actual_project_removal.delay(pk, uid)
509@app.task(
510 trail=False,
511 autoretry_for=(WeblateLockTimeoutError,),
512 retry_backoff=600,
513 retry_backoff_max=3600,
514)
515def auto_translate(
516 *,
517 user_id: int | None,
518 translation_id: int,
519 mode: str,
520 q: str,
521 auto_source: Literal["mt", "others"],
522 component: int | None,
523 engines: list[str],
524 threshold: int,
525 component_wide: bool = False,
526 unit_ids: list[int] | None = None,
527):
528 translation = Translation.objects.get(pk=translation_id)
529 user = User.objects.get(pk=user_id) if user_id else None
530 with override(user.profile.language if user else "en"):
531 auto = AutoTranslate(
532 user=user,
533 translation=translation,
534 q=q,
535 mode=mode,
536 component_wide=component_wide,
537 unit_ids=unit_ids,
538 )
539 message = auto.perform(
540 auto_source=auto_source,
541 engines=engines,
542 threshold=threshold,
543 source=component,
544 )
545 return {"translation": translation_id, "message": message}
548@app.task(
549 trail=False,
550 autoretry_for=(WeblateLockTimeoutError,),
551 retry_backoff=600,
552 retry_backoff_max=3600,
553)
554def auto_translate_component(
555 component_id: int,
556 mode: str,
557 q: str,
558 auto_source: Literal["mt", "others"],
559 engines: list[str],
560 threshold: int,
561 component: int | None = None,
562):
563 component_obj = Component.objects.get(pk=component_id)
565 for translation in component_obj.translation_set.iterator():
566 if translation.is_source:
567 continue
569 auto = AutoTranslate(
570 user=None,
571 translation=translation,
572 q=q,
573 mode=mode,
574 component_wide=True,
575 )
576 auto.perform(
577 auto_source=auto_source,
578 engines=engines,
579 threshold=threshold,
580 source=component,
581 )
582 component_obj.update_source_checks()
583 component_obj.run_batched_checks()
584 return {"component": component_obj.id}
587@app.task(trail=False)
588def create_component(copy_from=None, copy_addons=False, in_task=False, **kwargs):
589 kwargs["project"] = Project.objects.get(pk=kwargs["project"])
590 kwargs["source_language"] = Language.objects.get(pk=kwargs["source_language"])
591 component = Component(**kwargs)
592 # Perform validation to avoid creating duplicate components via background
593 # tasks in discovery
594 component.full_clean()
595 component.save(force_insert=True)
596 component.change_set.create(action=ActionEvents.CREATE_COMPONENT)
597 if copy_from:
598 # Copy non-automatic component lists
599 for clist in ComponentList.objects.filter(
600 components__id=copy_from, autocomponentlist__isnull=True
601 ):
602 clist.components.add(component)
603 # Copy add-ons
604 if copy_addons:
605 addons = Addon.objects.filter(component__pk=copy_from, repo_scope=False)
606 for addon in addons:
607 # Avoid installing duplicate addons
608 if component.addon_set.filter(name=addon.name).exists():
609 continue
610 if not addon.addon.can_install(component, None):
611 continue
612 addon.addon.create(
613 component=component, configuration=addon.configuration
614 )
615 if in_task:
616 return {"component": component.id}
617 return component
620@app.task(trail=False)
621@transaction.atomic
622def update_checks(pk: int, update_token: str, update_state: bool = False) -> None:
623 try:
624 component = Component.objects.get(pk=pk)
625 except Component.DoesNotExist:
626 return
628 # Skip when further updates are scheduled
629 latest_token = cache.get(component.update_checks_key)
630 if latest_token and update_token != latest_token:
631 return
633 component.batch_checks = True
634 # Source translation as last
635 translations = (
636 *component.translation_set.exclude(
637 pk=component.source_translation.pk
638 ).prefetch(),
639 component.source_translation,
640 )
641 for translation in translations:
642 units = translation.unit_set.prefetch()
643 if update_state:
644 units = units.select_for_update()
645 for unit in units.prefetch_all_checks():
646 # Reuse object to avoid fetching from the database
647 unit.source_unit.translation = component.source_translation
648 if update_state:
649 unit.update_state()
650 unit.run_checks()
651 component.run_batched_checks()
652 component.invalidate_cache()
655@app.task(trail=False)
656def daily_update_checks() -> None:
657 if settings.BACKGROUND_TASKS == "never": 657 ↛ 658line 657 didn't jump to line 658 because the condition on line 657 was never true
658 return
659 today = timezone.now()
660 components = Component.objects.annotate(hourmod=F("id") % 24).filter(
661 hourmod=today.hour
662 )
663 if settings.BACKGROUND_TASKS == "monthly": 663 ↛ 665line 663 didn't jump to line 665 because the condition on line 663 was always true
664 components = components.annotate(idmod=F("id") % 30).filter(idmod=today.day)
665 elif settings.BACKGROUND_TASKS == "weekly":
666 components = components.annotate(idmod=F("id") % 7).filter(
667 idmod=today.weekday()
668 )
669 for component in components.iterator(): 669 ↛ 670line 669 didn't jump to line 670 because the loop on line 669 never started
670 component.schedule_update_checks()
673@app.task(trail=False)
674def cleanup_project_backups() -> None:
675 from weblate.trans.backups import PROJECTBACKUP_PREFIX
677 # This intentionally does not use Project objects to remove stale backups
678 # for removed projects as well.
679 rootdir = data_dir(PROJECTBACKUP_PREFIX)
680 backup_cutoff = timezone.now() - timedelta(days=settings.PROJECT_BACKUP_KEEP_DAYS)
681 for projectdir in glob(os.path.join(rootdir, "*")):
682 if not os.path.isdir(projectdir):
683 continue
684 if projectdir.endswith("import"):
685 # Keep imports for shorter time, but more of them
686 cutoff = timezone.now() - timedelta(days=1)
687 max_count = 30
688 else:
689 cutoff = backup_cutoff
690 max_count = settings.PROJECT_BACKUP_KEEP_COUNT
691 backups = sorted(
692 (
693 (
694 path,
695 make_aware(
696 datetime.fromtimestamp(int(path.split(".")[0])) # noqa: DTZ006
697 ),
698 )
699 for path in os.listdir(projectdir)
700 if path.endswith((".zip", ".zip.part"))
701 ),
702 key=itemgetter(1),
703 reverse=True,
704 )
705 while len(backups) > max_count:
706 remove = backups.pop()
707 os.unlink(os.path.join(projectdir, remove[0]))
709 for backup in backups:
710 if backup[1] < cutoff:
711 os.unlink(os.path.join(projectdir, backup[0]))
714@app.task(trail=False)
715def create_project_backup(pk) -> None:
716 from weblate.trans.backups import ProjectBackup
718 project = Project.objects.get(pk=pk)
719 ProjectBackup().backup_project(project)
722@app.task(trail=False)
723def remove_project_backup_download(name: str) -> None:
724 if staticfiles_storage.exists(name):
725 staticfiles_storage.delete(name)
728@app.task(trail=False)
729def cleanup_project_backup_download() -> None:
730 from weblate.trans.backups import PROJECTBACKUP_PREFIX
732 if not staticfiles_storage.exists(PROJECTBACKUP_PREFIX): 732 ↛ 734line 732 didn't jump to line 734 because the condition on line 732 was always true
733 return
734 cutoff = timezone.now() - timedelta(hours=2)
735 for name in staticfiles_storage.listdir(PROJECTBACKUP_PREFIX)[1]:
736 full_name = os.path.join(PROJECTBACKUP_PREFIX, name)
737 if staticfiles_storage.get_created_time(full_name) < cutoff:
738 staticfiles_storage.delete(full_name)
741@app.on_after_finalize.connect
742def setup_periodic_tasks(sender, **kwargs) -> None:
743 sender.add_periodic_task(3600, commit_pending.s(), name="commit-pending")
744 sender.add_periodic_task(3600, update_remotes.s(), name="update-remotes")
745 sender.add_periodic_task(
746 crontab(minute=30), daily_update_checks.s(), name="daily-update-checks"
747 )
748 sender.add_periodic_task(
749 crontab(hour=3, minute=45), repository_alerts.s(), name="repository-alerts"
750 )
751 sender.add_periodic_task(3600, component_alerts.s(), name="component-alerts")
752 sender.add_periodic_task(
753 crontab(hour=0, minute=40), cleanup_suggestions.s(), name="suggestions-cleanup"
754 )
755 sender.add_periodic_task(
756 crontab(hour=0, minute=40), cleanup_stale_repos.s(), name="cleanup-stale-repos"
757 )
758 sender.add_periodic_task(
759 crontab(hour=0, minute=45),
760 cleanup_old_suggestions.s(),
761 name="cleanup-old-suggestions",
762 )
763 sender.add_periodic_task(
764 crontab(hour=0, minute=50),
765 cleanup_old_comments.s(),
766 name="cleanup-old-comments",
767 )
768 sender.add_periodic_task(
769 crontab(hour=2, minute=30),
770 cleanup_project_backups.s(),
771 name="cleanup-project-backups",
772 )
773 sender.add_periodic_task(
774 3600,
775 cleanup_project_backup_download.s(),
776 name="cleanup-project-backup-download",
777 )