Coverage for app/venv/lib/python3.14/site-packages/weblate/trans/models/component.py: 58%
1936 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-10-07 07:15 +0000
1# Copyright © Michal Čihař <michal@weblate.org>
2#
3# SPDX-License-Identifier: GPL-3.0-or-later
5from __future__ import annotations
7import os
8import re
9import time
10from collections import defaultdict
11from glob import glob
12from itertools import chain
13from typing import TYPE_CHECKING, Any, TypedDict, cast
14from urllib.parse import quote as urlquote
15from urllib.parse import urlparse
17import sentry_sdk
18from celery import current_task
19from celery.result import AsyncResult
20from django.conf import settings
21from django.core.cache import cache
22from django.core.exceptions import (
23 ImproperlyConfigured,
24 ObjectDoesNotExist,
25 ValidationError,
26)
27from django.core.validators import MaxValueValidator
28from django.db import IntegrityError, connection, models, transaction
29from django.db.models import Count, F, Q
30from django.db.models.signals import m2m_changed
31from django.dispatch import receiver
32from django.utils.functional import cached_property
33from django.utils.html import format_html, format_html_join
34from django.utils.safestring import mark_safe
35from django.utils.text import format_lazy
36from django.utils.timezone import localtime
37from django.utils.translation import gettext, gettext_lazy, ngettext, pgettext
38from weblate_language_data.ambiguous import AMBIGUOUS
40from weblate.checks.flags import Flags
41from weblate.checks.models import CHECKS
42from weblate.formats.models import FILE_FORMATS
43from weblate.lang.models import Language, get_default_lang
44from weblate.memory.tasks import import_memory
45from weblate.trans.actions import ActionEvents
46from weblate.trans.defines import (
47 BRANCH_LENGTH,
48 COMPONENT_NAME_LENGTH,
49 FILENAME_LENGTH,
50 PROJECT_NAME_LENGTH,
51 REPO_LENGTH,
52)
53from weblate.trans.exceptions import FileParseError, InvalidTemplateError
54from weblate.trans.fields import RegexField
55from weblate.trans.file_format_params import FILE_FORMATS_PARAMS
56from weblate.trans.mixins import (
57 CacheKeyMixin,
58 ComponentCategoryMixin,
59 LockMixin,
60 PathMixin,
61)
62from weblate.trans.models.alert import ALERTS, ALERTS_IMPORT, Alert, update_alerts
63from weblate.trans.models.change import Change
64from weblate.trans.models.pending import PendingUnitChange
65from weblate.trans.models.translation import Translation
66from weblate.trans.models.unit import Unit
67from weblate.trans.models.variant import Variant
68from weblate.trans.signals import (
69 component_post_update,
70 translation_post_add,
71 vcs_post_commit,
72 vcs_post_push,
73 vcs_post_update,
74 vcs_pre_push,
75 vcs_pre_update,
76)
77from weblate.trans.util import (
78 PRIORITY_CHOICES,
79 cleanup_path,
80 cleanup_repo_url,
81 count_words,
82 is_repo_link,
83 path_separator,
84)
85from weblate.trans.validators import (
86 validate_autoaccept,
87 validate_check_flags,
88 validate_file_format_parameters,
89 validate_filemask,
90 validate_language_code,
91)
92from weblate.utils import messages
93from weblate.utils.celery import get_task_progress
94from weblate.utils.colors import ColorChoices
95from weblate.utils.decorators import disable_for_loaddata
96from weblate.utils.errors import report_error
97from weblate.utils.fields import EmailField
98from weblate.utils.html import format_html_join_comma, list_to_tuples
99from weblate.utils.licenses import (
100 get_license_choices,
101 get_license_name,
102 get_license_url,
103 is_libre,
104)
105from weblate.utils.lock import WeblateLock
106from weblate.utils.random import get_random_identifier
107from weblate.utils.render import (
108 render_template,
109 validate_render_addon,
110 validate_render_commit,
111 validate_render_component,
112 validate_repoweb,
113)
114from weblate.utils.site import get_site_url
115from weblate.utils.state import (
116 STATE_APPROVED,
117 STATE_FUZZY,
118 STATE_READONLY,
119 STATE_TRANSLATED,
120)
121from weblate.utils.stats import ComponentStats
122from weblate.utils.validators import (
123 validate_filename,
124 validate_re_nonempty,
125 validate_slug,
126)
127from weblate.vcs.base import RepositoryError
128from weblate.vcs.git import GitMergeRequestBase, LocalRepository
129from weblate.vcs.models import VCS_REGISTRY
130from weblate.vcs.ssh import add_host_key
132if TYPE_CHECKING: 132 ↛ 133line 132 didn't jump to line 133 because the condition on line 132 was never true
133 from collections.abc import Iterable
134 from datetime import datetime
136 from weblate.addons.models import Addon
137 from weblate.auth.models import AuthenticatedHttpRequest, User
138 from weblate.checks.base import BaseCheck
139 from weblate.formats.base import TranslationFormat
140 from weblate.trans.models.unit import UnitAttributesDict
141 from weblate.vcs.base import Repository
143NEW_LANG_CHOICES = (
144 # Translators: Action when adding new translation
145 ("contact", gettext_lazy("Contact maintainers")),
146 # Translators: Action when adding new translation
147 ("url", gettext_lazy("Point to translation instructions URL")),
148 # Translators: Action when adding new translation
149 ("add", gettext_lazy("Create new language file")),
150 # Translators: Action when adding new translation
151 ("none", gettext_lazy("Disable adding new translations")),
152)
153LANGUAGE_CODE_STYLE_CHOICES = (
154 ("", gettext_lazy("Default based on the file format")),
155 ("posix", gettext_lazy("POSIX style using underscore as a separator")),
156 (
157 "posix_lowercase",
158 gettext_lazy("POSIX style using underscore as a separator, lower cased"),
159 ),
160 ("bcp", gettext_lazy("BCP style using hyphen as a separator")),
161 (
162 "posix_long",
163 gettext_lazy(
164 "POSIX style using underscore as a separator, including country code"
165 ),
166 ),
167 (
168 "posix_long_lowercase",
169 gettext_lazy(
170 "POSIX style using underscore as a separator, including country code, lower cased"
171 ),
172 ),
173 (
174 "bcp_long",
175 gettext_lazy("BCP style using hyphen as a separator, including country code"),
176 ),
177 (
178 "bcp_legacy",
179 gettext_lazy("BCP style using hyphen as a separator, legacy language codes"),
180 ),
181 ("bcp_lower", gettext_lazy("BCP style using hyphen as a separator, lower cased")),
182 ("android", gettext_lazy("Android style")),
183 ("appstore", gettext_lazy("Apple App Store metadata style")),
184 ("googleplay", gettext_lazy("Google Play metadata style")),
185 ("linux", gettext_lazy("Linux style")),
186 ("linux_lowercase", gettext_lazy("Linux style, lower cased")),
187)
189MERGE_CHOICES = (
190 ("merge", gettext_lazy("Merge")),
191 ("rebase", gettext_lazy("Rebase")),
192 ("merge_noff", gettext_lazy("Merge without fast-forward")),
193)
195LOCKING_ALERTS = {"MergeFailure", "UpdateFailure", "PushFailure", "ParseError"}
197BITBUCKET_GIT_REPOS_REGEXP = [
198 r"(?:ssh|https):\/\/(?:(?:git@|)bitbucket.org)\/([^/]*)\/([^/]*)",
199 r"git@bitbucket.org:([^/]*)\/([^/]*)",
200]
202GITHUB_REPOS_REGEXP = [
203 r"(?:git|https):\/\/(?:github.com)\/([^/]*)\/([^/]*)",
204 r"git@github.com:([^/]*)\/([^/]*)",
205]
207PAGURE_REPOS_REGEXP = [r"(?:ssh|https):\/\/(?:(?:git@|)pagure.io)\/([^/]*)\/([^/]*)"]
209AZURE_REPOS_REGEXP = [
210 r"(?:https):\/\/(?:dev.azure.com)\/([^/]*)\/([^/]*)\/(?:_git)\/([^/]*)",
211 r"(?:https):\/\/(?:([^/]*).visualstudio.com)\/([^/]*)\/(?:_git)\/([^/]*)",
212 r"(?:[^/]*)\@vs-ssh.visualstudio.com:v3\/([^/]*)\/([^/]*)\/([^/]*)",
213 r"(?:git@ssh.dev.azure.com:v3)\/([^/]*)\/([^/]*)\/([^/]*)",
214]
217def perform_on_link(func):
218 """Perform operation on repository link."""
220 def on_link_wrapper(self, *args, **kwargs):
221 linked = self.linked_component
222 if linked: 222 ↛ 224line 222 didn't jump to line 224 because the condition on line 222 was never true
223 # Avoid loading project next time if matches
224 if linked.project_id == self.project_id:
225 linked.project = self.project
226 # Call same method on linked component
227 return getattr(linked, func.__name__)(*args, **kwargs)
228 return func(self, *args, **kwargs)
230 return on_link_wrapper
233def prefetch_tasks(components):
234 """Prefetch update tasks."""
235 lookup = {component.update_key: component for component in components}
236 if lookup:
237 results_dict = cache.get_many(lookup.keys())
238 results: dict[str, AsyncResult] = {
239 value: AsyncResult(value) for value in results_dict.values() if value
240 }
242 for item, value in results_dict.items():
243 if not value:
244 continue
245 lookup[item].__dict__["background_task"] = results[value]
246 lookup.pop(item)
247 for component in lookup.values():
248 component.__dict__["background_task"] = None
249 return components
252def translation_prefetch_tasks(translations):
253 prefetch_tasks([translation.component for translation in translations])
254 return translations
257def prefetch_glossary_terms(components) -> None:
258 if not components:
259 return
260 lookup = {component.glossary_sources_key: component for component in components}
261 for item, value in cache.get_many(lookup.keys()).items():
262 lookup[item].__dict__["glossary_sources"] = value
265class ComponentQuerySet(models.QuerySet):
266 def prefetch(self, alerts: bool = True, defer: bool = True):
267 result = self
268 linked_component: str | models.Prefetch
269 if defer:
270 result = result.defer_huge()
271 linked_component = models.Prefetch(
272 "linked_component", queryset=Component.objects.defer_huge()
273 )
274 else:
275 linked_component = "linked_component"
276 if alerts: 276 ↛ 285line 276 didn't jump to line 285 because the condition on line 276 was always true
277 result = result.prefetch_related(
278 models.Prefetch(
279 "alert_set",
280 queryset=Alert.objects.filter(dismissed=False),
281 to_attr="all_active_alerts",
282 ),
283 )
285 return result.prefetch_related(
286 "project",
287 "category",
288 "category__project",
289 "category__category",
290 "category__category__project",
291 "category__category__category",
292 "category__category__category__project",
293 linked_component,
294 "linked_component__project",
295 )
297 def defer_huge(self):
298 return self.defer(
299 "commit_message",
300 "add_message",
301 "delete_message",
302 "merge_message",
303 "addon_message",
304 "pull_message",
305 )
307 def filter_by_path(self, path: str) -> ComponentQuerySet:
308 try:
309 project, *categories, component = path.split("/")
310 except ValueError as error:
311 raise Component.DoesNotExist from error
312 kwargs: dict[str, str | None] = {}
313 prefix = ""
314 for category in reversed(categories):
315 kwargs[f"{prefix}category__slug"] = category
316 prefix = f"category__{prefix}"
317 if not kwargs:
318 kwargs["category"] = None
319 return self.filter(
320 slug__iexact=component, project__slug__iexact=project, **kwargs
321 )
323 def get_by_path(self, path: str) -> Component:
324 return self.filter_by_path(path).get()
326 def get_linked(self, val):
327 """Return component for linked repo."""
328 if not is_repo_link(val): 328 ↛ 330line 328 didn't jump to line 330 because the condition on line 328 was always true
329 return None
330 return self.get_by_path(val[10:])
332 def order_project(self):
333 """Ordering in global scope by project name."""
334 return self.order_by("project__name", "name")
336 def order(self):
337 """Ordering in project scope by priority."""
338 return self.order_by("priority", "is_glossary", "name")
340 def with_repo(self):
341 return self.exclude(repo__startswith="weblate:")
343 def filter_access(self, user: User):
344 result = self
345 if user.needs_project_filter: 345 ↛ 346line 345 didn't jump to line 346 because the condition on line 345 was never true
346 result = result.filter(project__in=user.allowed_projects)
347 if user.needs_component_restrictions_filter: 347 ↛ 348line 347 didn't jump to line 348 because the condition on line 347 was never true
348 result = result.filter(
349 Q(restricted=False) | Q(id__in=user.component_permissions)
350 )
351 return result
353 def search(self, query: str):
354 return self.filter(
355 Q(name__icontains=query) | Q(slug__icontains=query)
356 ).select_related(
357 "project",
358 "category__project",
359 "category__category",
360 "category__category__project",
361 "category__category__category",
362 "category__category__category__project",
363 )
366class OldComponentSettings(TypedDict):
367 check_flags: str
370class Component(
371 models.Model, PathMixin, CacheKeyMixin, ComponentCategoryMixin, LockMixin
372):
373 name = models.CharField(
374 verbose_name=gettext_lazy("Component name"),
375 max_length=COMPONENT_NAME_LENGTH,
376 help_text=gettext_lazy("Display name"),
377 )
378 slug = models.SlugField(
379 verbose_name=gettext_lazy("URL slug"),
380 max_length=COMPONENT_NAME_LENGTH,
381 help_text=gettext_lazy("Name used in URLs and filenames."),
382 validators=[validate_slug],
383 )
384 project = models.ForeignKey(
385 "trans.Project",
386 verbose_name=gettext_lazy("Project"),
387 on_delete=models.deletion.CASCADE,
388 db_index=False,
389 )
390 category = models.ForeignKey(
391 "trans.Category",
392 verbose_name=gettext_lazy("Category"),
393 on_delete=models.deletion.CASCADE,
394 null=True,
395 blank=True,
396 )
397 vcs = models.CharField(
398 verbose_name=gettext_lazy("Version control system"),
399 max_length=20,
400 help_text=gettext_lazy(
401 "Version control system to use to access your "
402 "repository containing translations. You can also choose "
403 "additional integration with third party providers to "
404 "submit merge requests."
405 ),
406 choices=VCS_REGISTRY.get_choices(),
407 default=settings.DEFAULT_VCS,
408 )
409 repo = models.CharField(
410 verbose_name=gettext_lazy("Source code repository"),
411 max_length=REPO_LENGTH,
412 help_text=gettext_lazy(
413 "URL of a repository, use weblate://project/component "
414 "to share it with other component."
415 ),
416 )
417 linked_component = models.ForeignKey(
418 "trans.Component",
419 verbose_name=gettext_lazy("Project"),
420 on_delete=models.deletion.CASCADE,
421 null=True,
422 editable=False,
423 )
424 push = models.CharField(
425 verbose_name=gettext_lazy("Repository push URL"),
426 max_length=REPO_LENGTH,
427 help_text=gettext_lazy(
428 "URL of a push repository, pushing is turned off if empty."
429 ),
430 blank=True,
431 )
432 repoweb = models.CharField(
433 verbose_name=gettext_lazy("Repository browser"),
434 max_length=200,
435 help_text=gettext_lazy(
436 "Link to repository browser, use {{branch}} for branch, "
437 "{{filename}} and {{line}} as filename and line placeholders. "
438 "You might want to strip leading directory by using {{filename|parentdir}}."
439 ),
440 validators=[validate_repoweb],
441 blank=True,
442 )
443 git_export = models.CharField(
444 verbose_name=gettext_lazy("Exported repository URL"),
445 max_length=60 + PROJECT_NAME_LENGTH + COMPONENT_NAME_LENGTH,
446 help_text=gettext_lazy(
447 "URL of repository where users can fetch changes from Weblate"
448 ),
449 blank=True,
450 )
451 report_source_bugs = EmailField(
452 verbose_name=gettext_lazy("Source string bug reporting address"),
453 help_text=gettext_lazy(
454 "E-mail address for reports on errors in source strings. "
455 "Leave empty for no e-mails."
456 ),
457 blank=True,
458 )
459 branch = models.CharField(
460 verbose_name=gettext_lazy("Repository branch"),
461 max_length=BRANCH_LENGTH,
462 help_text=gettext_lazy("Repository branch to translate"),
463 default="",
464 blank=True,
465 )
466 push_branch = models.CharField(
467 verbose_name=gettext_lazy("Push branch"),
468 max_length=BRANCH_LENGTH,
469 help_text=gettext_lazy(
470 "Branch for pushing changes, leave empty to use repository branch"
471 ),
472 default="",
473 blank=True,
474 )
475 filemask = models.CharField(
476 verbose_name=gettext_lazy("File mask"),
477 max_length=FILENAME_LENGTH,
478 validators=[validate_filemask, validate_filename],
479 help_text=gettext_lazy(
480 "Path of files to translate relative to repository root,"
481 " use * instead of language code, "
482 "for example: po/*.po or locale/*/LC_MESSAGES/django.po."
483 ),
484 )
485 screenshot_filemask = models.CharField(
486 verbose_name=gettext_lazy("Screenshot file mask"),
487 max_length=FILENAME_LENGTH,
488 blank=True,
489 validators=[validate_filemask, validate_filename],
490 help_text=gettext_lazy(
491 "Path of screenshots relative to repository root, "
492 "for example: docs/screenshots/*.png."
493 ),
494 )
495 template = models.CharField(
496 verbose_name=gettext_lazy("Monolingual base language file"),
497 max_length=FILENAME_LENGTH,
498 blank=True,
499 help_text=gettext_lazy(
500 "Filename of translation base file, containing all strings "
501 "and their source "
502 "for monolingual translations."
503 ),
504 validators=[validate_filename],
505 )
506 edit_template = models.BooleanField(
507 verbose_name=gettext_lazy("Edit base file"),
508 default=True,
509 help_text=gettext_lazy(
510 "Whether users will be able to edit the base file "
511 "for monolingual translations."
512 ),
513 )
514 intermediate = models.CharField(
515 verbose_name=gettext_lazy("Intermediate language file"),
516 max_length=FILENAME_LENGTH,
517 blank=True,
518 help_text=gettext_lazy(
519 "Filename of intermediate translation file. In most cases "
520 "this is a translation file provided by developers and is "
521 "used when creating actual source strings."
522 ),
523 validators=[validate_filename],
524 )
526 new_base = models.CharField(
527 verbose_name=gettext_lazy("Template for new translations"),
528 max_length=FILENAME_LENGTH,
529 blank=True,
530 help_text=gettext_lazy(
531 "Filename of file used for creating new translations. "
532 "For gettext choose .pot file."
533 ),
534 validators=[validate_filename],
535 )
536 file_format = models.CharField(
537 verbose_name=gettext_lazy("File format"),
538 max_length=50,
539 choices=FILE_FORMATS.get_choices(),
540 blank=False,
541 )
543 file_format_params = models.JSONField(
544 verbose_name=gettext_lazy("File format parameters"),
545 default=dict,
546 blank=True,
547 validators=[validate_file_format_parameters],
548 )
550 locked = models.BooleanField(
551 verbose_name=gettext_lazy("Locked"),
552 default=False,
553 help_text=gettext_lazy(
554 "Locked component will not get any translation updates."
555 ),
556 )
557 hide_glossary_matches = models.BooleanField(
558 verbose_name=gettext_lazy("Do not show glossary matches"),
559 default=False,
560 help_text=gettext_lazy("Hides the glossary panel in the translation editor."),
561 )
562 allow_translation_propagation = models.BooleanField(
563 verbose_name=gettext_lazy("Allow translation propagation"),
564 default=settings.DEFAULT_TRANSLATION_PROPAGATION,
565 help_text=gettext_lazy(
566 "Whether translation updates in other components "
567 "will cause automatic translation in this one"
568 ),
569 )
570 contribute_project_tm = models.BooleanField(
571 verbose_name=gettext_lazy("Contribute to project translation memory"),
572 default=True,
573 help_text=gettext_lazy(
574 "Allow this component's translations to be added to the project-level"
575 " translation memory."
576 ),
577 )
578 # This should match definition in WorkflowSetting
579 enable_suggestions = models.BooleanField(
580 verbose_name=gettext_lazy("Turn on suggestions"),
581 default=True,
582 help_text=gettext_lazy("Whether to allow translation suggestions at all."),
583 )
584 # This should match definition in WorkflowSetting
585 suggestion_voting = models.BooleanField(
586 verbose_name=gettext_lazy("Suggestion voting"),
587 default=False,
588 help_text=gettext_lazy(
589 "Users can only vote for suggestions and can’t make direct translations."
590 ),
591 )
592 # This should match definition in WorkflowSetting
593 suggestion_autoaccept = models.PositiveSmallIntegerField(
594 verbose_name=gettext_lazy("Automatically accept suggestions"),
595 default=0,
596 help_text=gettext_lazy(
597 "Automatically accept suggestions with this number of votes,"
598 " use 0 to turn it off."
599 ),
600 validators=[validate_autoaccept],
601 )
602 check_flags = models.TextField(
603 verbose_name=gettext_lazy("Translation flags"),
604 default="",
605 help_text=gettext_lazy(
606 "Additional comma-separated flags to influence Weblate behavior."
607 ),
608 validators=[validate_check_flags],
609 blank=True,
610 )
611 enforced_checks = models.JSONField(
612 verbose_name=gettext_lazy("Enforced checks"),
613 help_text=gettext_lazy("List of checks which can not be dismissed."),
614 default=list,
615 blank=True,
616 )
618 # Licensing
619 license = models.CharField(
620 verbose_name=gettext_lazy("Translation license"),
621 max_length=150,
622 blank=not settings.LICENSE_REQUIRED,
623 default="",
624 choices=get_license_choices(),
625 )
626 agreement = models.TextField(
627 verbose_name=gettext_lazy("Contributor license agreement"),
628 blank=True,
629 default="",
630 help_text=gettext_lazy(
631 "Contributor license agreement which needs to be approved before a user can "
632 "translate this component."
633 ),
634 )
636 # Adding new language
637 new_lang = models.CharField(
638 verbose_name=gettext_lazy("Adding new translation"),
639 max_length=10,
640 choices=NEW_LANG_CHOICES,
641 default="add",
642 help_text=gettext_lazy("How to handle requests for creating new translations."),
643 )
644 language_code_style = models.CharField(
645 verbose_name=gettext_lazy("Language code style"),
646 max_length=20,
647 choices=LANGUAGE_CODE_STYLE_CHOICES,
648 default="",
649 blank=True,
650 help_text=gettext_lazy(
651 "Customize language code used to generate the filename for "
652 "translations created by Weblate."
653 ),
654 )
655 manage_units = models.BooleanField(
656 verbose_name=gettext_lazy("Manage strings"),
657 default=False,
658 help_text=gettext_lazy(
659 "Enables adding and removing strings straight from Weblate. If your "
660 "strings are extracted from the source code or managed externally you "
661 "probably want to keep it disabled."
662 ),
663 )
665 # VCS config
666 merge_style = models.CharField(
667 verbose_name=gettext_lazy("Merge style"),
668 max_length=10,
669 choices=MERGE_CHOICES,
670 default=settings.DEFAULT_MERGE_STYLE,
671 help_text=gettext_lazy(
672 "Define whether Weblate should merge the upstream repository "
673 "or rebase changes onto it."
674 ),
675 )
676 commit_message = models.TextField(
677 verbose_name=gettext_lazy("Commit message when translating"),
678 help_text=gettext_lazy(
679 "You can use template language for various info, "
680 "please consult the documentation for more details."
681 ),
682 validators=[validate_render_commit],
683 default=settings.DEFAULT_COMMIT_MESSAGE,
684 )
685 add_message = models.TextField(
686 verbose_name=gettext_lazy("Commit message when adding translation"),
687 help_text=gettext_lazy(
688 "You can use template language for various info, "
689 "please consult the documentation for more details."
690 ),
691 validators=[validate_render_commit],
692 default=settings.DEFAULT_ADD_MESSAGE,
693 )
694 delete_message = models.TextField(
695 verbose_name=gettext_lazy("Commit message when removing translation"),
696 help_text=gettext_lazy(
697 "You can use template language for various info, "
698 "please consult the documentation for more details."
699 ),
700 validators=[validate_render_commit],
701 default=settings.DEFAULT_DELETE_MESSAGE,
702 )
703 merge_message = models.TextField(
704 # Translators: The commit message, for when merging the translation
705 verbose_name=gettext_lazy("Commit message when merging translation"),
706 help_text=gettext_lazy(
707 "You can use template language for various info, "
708 "please consult the documentation for more details."
709 ),
710 validators=[validate_render_component],
711 default=settings.DEFAULT_MERGE_MESSAGE,
712 )
713 addon_message = models.TextField(
714 verbose_name=gettext_lazy("Commit message when add-on makes a change"),
715 help_text=gettext_lazy(
716 "You can use template language for various info, "
717 "please consult the documentation for more details."
718 ),
719 validators=[validate_render_addon],
720 default=settings.DEFAULT_ADDON_MESSAGE,
721 )
722 pull_message = models.TextField(
723 verbose_name=gettext_lazy("Merge request message"),
724 help_text=gettext_lazy(
725 "You can use template language for various info, "
726 "please consult the documentation for more details."
727 ),
728 validators=[validate_render_addon],
729 default=settings.DEFAULT_PULL_MESSAGE,
730 )
731 push_on_commit = models.BooleanField(
732 verbose_name=gettext_lazy("Push on commit"),
733 default=settings.DEFAULT_PUSH_ON_COMMIT,
734 help_text=gettext_lazy(
735 "Whether the repository should be pushed upstream on every commit."
736 ),
737 )
738 commit_pending_age = models.SmallIntegerField(
739 verbose_name=gettext_lazy("Age of changes to commit"),
740 default=settings.COMMIT_PENDING_HOURS,
741 validators=[MaxValueValidator(2160)],
742 help_text=gettext_lazy(
743 "Time in hours after which any pending changes will be "
744 "committed to the VCS."
745 ),
746 )
747 auto_lock_error = models.BooleanField(
748 verbose_name=gettext_lazy("Lock on error"),
749 default=settings.DEFAULT_AUTO_LOCK_ERROR,
750 help_text=gettext_lazy(
751 "Whether the component should be locked on repository errors."
752 ),
753 )
755 source_language = models.ForeignKey(
756 Language,
757 verbose_name=gettext_lazy("Source language"),
758 help_text=gettext_lazy("Language used for source strings in all components"),
759 default=get_default_lang,
760 on_delete=models.deletion.CASCADE,
761 )
762 language_regex = RegexField(
763 verbose_name=gettext_lazy("Language filter"),
764 max_length=500,
765 default="^[^.]+$",
766 help_text=gettext_lazy(
767 "Regular expression used to filter "
768 "translation files when scanning for file mask."
769 ),
770 )
771 variant_regex = RegexField(
772 verbose_name=gettext_lazy("Variants regular expression"),
773 validators=[validate_re_nonempty],
774 max_length=190,
775 default="",
776 blank=True,
777 help_text=gettext_lazy(
778 "Regular expression used to determine variants of a string."
779 ),
780 )
782 priority = models.IntegerField(
783 default=100,
784 choices=PRIORITY_CHOICES,
785 verbose_name=gettext_lazy("Priority"),
786 help_text=gettext_lazy(
787 "Components with higher priority are offered first to translators."
788 ),
789 )
790 restricted = models.BooleanField(
791 verbose_name=gettext_lazy("Restricted component"),
792 default=settings.DEFAULT_RESTRICTED_COMPONENT,
793 db_index=True,
794 help_text=gettext_lazy(
795 "Restrict access to the component to only "
796 "those explicitly given permission."
797 ),
798 )
800 links = models.ManyToManyField(
801 "trans.Project",
802 verbose_name=gettext_lazy("Share in projects"),
803 blank=True,
804 related_name="shared_components",
805 help_text=gettext_lazy(
806 "Choose additional projects where this component will be listed."
807 ),
808 )
810 # Glossary management
811 is_glossary = models.BooleanField(
812 verbose_name=gettext_lazy("Use as a glossary"),
813 default=False,
814 db_index=True,
815 help_text=gettext_lazy(
816 "Glossaries are different from regular translations but help keep track of and share consistent terminology."
817 ),
818 )
819 glossary_color = models.CharField(
820 verbose_name=gettext_lazy("Glossary color"),
821 max_length=30,
822 choices=ColorChoices.choices,
823 blank=False,
824 default=ColorChoices.SILVER,
825 )
826 remote_revision = models.CharField(max_length=200, default="", blank=True)
827 local_revision = models.CharField(max_length=200, default="", blank=True)
828 processed_revision = models.CharField(max_length=200, default="", blank=True)
830 key_filter = RegexField(
831 verbose_name=gettext_lazy("Key filter"),
832 max_length=500,
833 default="",
834 help_text=gettext_lazy(
835 "Regular expression used to filter keys. This is only available for monolingual formats."
836 ),
837 blank=True,
838 )
840 secondary_language = models.ForeignKey(
841 Language,
842 verbose_name=gettext_lazy("Secondary language"),
843 help_text=format_lazy(
844 "{} {}",
845 gettext_lazy(
846 "Additional language to show together with the source language while translating."
847 ),
848 gettext_lazy("This setting is inherited from the project if left empty."),
849 ),
850 default=None,
851 blank=True,
852 null=True,
853 related_name="component_secondary_languages",
854 on_delete=models.deletion.CASCADE,
855 )
857 objects = ComponentQuerySet.as_manager()
859 is_lockable = True
860 remove_permission = "component.edit"
861 settings_permission = "component.edit"
863 class Meta:
864 app_label = "trans"
865 verbose_name = "Component"
866 verbose_name_plural = "Components"
867 indexes = [ # noqa: RUF012
868 models.Index(fields=["project", "allow_translation_propagation"]),
869 ]
870 constraints = [ # noqa: RUF012
871 models.UniqueConstraint(
872 name="component_slug_unique",
873 fields=["project", "category", "slug"],
874 nulls_distinct=False,
875 ),
876 models.UniqueConstraint(
877 name="component_name_unique",
878 fields=["project", "category", "name"],
879 nulls_distinct=False,
880 ),
881 ]
883 def __str__(self) -> str:
884 return f"{self.category or self.project}/{self.name}"
886 def __init__(self, *args, **kwargs) -> None:
887 super().__init__(*args, **kwargs)
888 self._file_format = None
889 self.stats = ComponentStats(self)
890 self.needs_cleanup = False
891 self.alerts_trigger: dict[str, list[dict]] = {}
892 self.updated_sources: dict[int, Unit] = {}
893 self.old_component_settings: OldComponentSettings = {
894 "check_flags": self.check_flags
895 }
896 self._sources: dict[int, Unit] = {}
897 self._sources_prefetched = False
898 self.logs: list[str] = []
899 self.translations_count: int | None = None
900 self.translations_progress = 0
901 self.acting_user: User | None = None
902 self.batch_checks = False
903 self.batched_checks: set[str] = set()
904 self.needs_variants_update = False
905 self._invalidate_scheduled = False
906 self._alerts_scheduled = False
907 self._template_check_done = False
908 self._glossary_sync_scheduled = False
909 self.new_lang_error_message: str | None = None
911 def save(self, *args, **kwargs) -> None:
912 """
913 Save wrapper.
915 It updates the back-end repository and regenerates translation data.
916 """
917 from weblate.trans.tasks import component_after_save
919 self.drop_file_format_cache()
920 self.set_default_branch()
922 # Linked component cache
923 self.linked_component = Component.objects.get_linked(self.repo)
925 # Detect if VCS config has changed (so that we have to pull the repo)
926 changed_git = True
927 changed_setup = False
928 changed_template = False
929 changed_variant = False
930 changed_enforced_checks = False
931 create = True
933 # Sets the key_filter to blank if the file format is bilingual
934 if self.key_filter and not self.has_template(): 934 ↛ 935line 934 didn't jump to line 935 because the condition on line 934 was never true
935 self.key_filter = ""
937 update_tm = self.contribute_project_tm
939 if self.id:
940 old = Component.objects.get(pk=self.id)
941 changed_git = (
942 (old.vcs != self.vcs)
943 or (old.repo != self.repo)
944 or (old.branch != self.branch)
945 or (old.filemask != self.filemask)
946 or (old.language_regex != self.language_regex)
947 )
948 changed_template = (old.intermediate != self.intermediate) or (
949 old.template != self.template
950 )
951 changed_setup = (
952 (old.file_format != self.file_format)
953 or (old.edit_template != self.edit_template)
954 or (old.new_base != self.new_base)
955 or changed_template
956 or old.key_filter != self.key_filter
957 )
958 if changed_setup:
959 old.commit_pending("changed setup", None)
960 if old.key_filter != self.key_filter: 960 ↛ 961line 960 didn't jump to line 961 because the condition on line 960 was never true
961 self.drop_key_filter_cache()
963 changed_variant = old.variant_regex != self.variant_regex
964 # Generate change entries for changes
965 self.generate_changes(old)
966 # Detect slug changes and rename Git repo
967 self.check_rename(old)
968 # Rename linked repos
969 if ( 969 ↛ 974line 969 didn't jump to line 974 because the condition on line 969 was never true
970 old.slug != self.slug
971 or old.project != self.project
972 or old.category != self.category
973 ):
974 old.component_set.update(repo=self.get_repo_link_url())
975 if changed_git: 975 ↛ 976line 975 didn't jump to line 976 because the condition on line 975 was never true
976 self.drop_repository_cache()
978 changed_enforced_checks = (
979 old.enforced_checks != self.enforced_checks and self.enforced_checks
980 )
982 create = False
984 # detect if the component had TM contribution disabled but changed to enabled
985 update_tm = self.contribute_project_tm and not old.contribute_project_tm
986 elif self.is_glossary:
987 # Creating new glossary
989 # Turn on unit management for glossary and disable adding languages
990 # as they are added automatically
991 self.manage_units = True
992 self.new_lang = "none"
993 # Make sure it is listed in project glossaries now
994 self.project.glossaries.append(self)
996 # Remove leading ./ from paths
997 self.filemask = cleanup_path(self.filemask)
998 self.screenshot_filemask = cleanup_path(self.screenshot_filemask)
999 self.template = cleanup_path(self.template)
1000 self.intermediate = cleanup_path(self.intermediate)
1001 self.new_base = cleanup_path(self.new_base)
1003 # Save/Create object
1004 super().save(*args, **kwargs)
1006 if create:
1007 self.install_autoaddon()
1009 # Ensure source translation is existing, otherwise we might
1010 # be hitting race conditions between background update and frontend displaying
1011 # the newly created component
1012 bool(self.source_translation)
1014 if settings.CELERY_TASK_ALWAYS_EAGER: 1014 ↛ 1015line 1014 didn't jump to line 1015 because the condition on line 1014 was never true
1015 self.after_save(
1016 changed_git=changed_git,
1017 changed_setup=changed_setup,
1018 changed_template=changed_template,
1019 changed_variant=changed_variant,
1020 changed_enforced_checks=changed_enforced_checks,
1021 skip_push=kwargs.get("force_insert", False),
1022 create=create,
1023 )
1024 else:
1025 component_after_save.delay_on_commit(
1026 self.pk,
1027 changed_git=changed_git,
1028 changed_setup=changed_setup,
1029 changed_template=changed_template,
1030 changed_variant=changed_variant,
1031 changed_enforced_checks=changed_enforced_checks,
1032 skip_push=kwargs.get("force_insert", False),
1033 create=create,
1034 )
1036 if self.old_component_settings["check_flags"] != self.check_flags: 1036 ↛ 1037line 1036 didn't jump to line 1037 because the condition on line 1036 was never true
1037 transaction.on_commit(
1038 lambda: self.schedule_update_checks(update_state=True)
1039 )
1041 # Invalidate source language cache just to be sure, as it is relatively
1042 # cheap to update
1043 self.project.invalidate_source_language_cache()
1044 for project in self.cached_links: 1044 ↛ 1045line 1044 didn't jump to line 1045 because the loop on line 1044 never started
1045 project.invalidate_source_language_cache()
1047 if update_tm:
1048 import_memory.delay_on_commit(self.project.id, self.pk)
1050 @cached_property
1051 def cached_links(self) -> models.QuerySet[Component]:
1052 return self.links.all()
1054 def generate_changes(self, old) -> None:
1055 def getvalue(base, attribute):
1056 result = getattr(base, attribute)
1057 if result is None:
1058 return ""
1059 # Use slug for Category/Project instances
1060 return getattr(result, "slug", result)
1062 tracked = (
1063 ("license", ActionEvents.LICENSE_CHANGE),
1064 ("agreement", ActionEvents.AGREEMENT_CHANGE),
1065 ("slug", ActionEvents.RENAME_COMPONENT),
1066 ("category", ActionEvents.MOVE_COMPONENT),
1067 ("project", ActionEvents.MOVE_COMPONENT),
1068 )
1069 for attribute, action in tracked:
1070 old_value = getvalue(old, attribute)
1071 current_value = getvalue(self, attribute)
1073 if old_value != current_value: 1073 ↛ 1074line 1073 didn't jump to line 1074 because the condition on line 1073 was never true
1074 self.change_set.create(
1075 action=action,
1076 old=old_value,
1077 target=current_value,
1078 user=self.acting_user,
1079 )
1081 def install_autoaddon(self) -> None:
1082 """Installs automatically enabled addons from file format."""
1083 from weblate.addons.models import ADDONS, Addon
1085 for name, configuration in chain( 1085 ↛ 1088line 1085 didn't jump to line 1088 because the loop on line 1085 never started
1086 self.file_format_cls.autoaddon.items(), settings.DEFAULT_ADDONS.items()
1087 ):
1088 try:
1089 addon = ADDONS[name]
1090 except KeyError:
1091 self.log_warning("could not enable addon %s, not found", name)
1092 continue
1094 if (
1095 addon.project_scope
1096 and Addon.objects.filter(
1097 component__project=self.project, name=name
1098 ).exists()
1099 ):
1100 self.log_warning(
1101 "could not enable addon %s, already installed on project", name
1102 )
1103 continue
1105 component = self
1106 if addon.repo_scope and self.linked_component:
1107 component = self.linked_component
1109 if component.addon_set.filter(name=name).exists():
1110 component.log_warning(
1111 "could not enable addon %s, already installed", name
1112 )
1113 continue
1115 if addon.has_settings():
1116 form = addon.get_add_form(None, component=component, data=configuration)
1117 if form is None:
1118 raise TypeError
1119 if not form.is_valid():
1120 component.log_warning(
1121 "could not enable addon %s, invalid settings", name
1122 )
1123 continue
1125 if not addon.can_install(component, None):
1126 component.log_warning("could not enable addon %s, not compatible", name)
1127 continue
1129 component.log_info("enabling addon %s", name)
1130 # Running is disabled now, it is triggered in after_save
1131 addon.create(component=component, run=False, configuration=configuration)
1133 def create_glossary(self) -> None:
1134 project = self.project
1136 # Does glossary already exist?
1137 if self.is_glossary or project.glossaries or len(project.child_components) > 2:
1138 return
1140 component_names = {component.name for component in project.child_components}
1141 component_slugs = {component.slug for component in project.child_components}
1142 if "Glossary" in component_names or "glossary" in component_slugs: 1142 ↛ 1143line 1142 didn't jump to line 1143 because the condition on line 1142 was never true
1143 return
1145 # Create glossary component
1146 project.scratch_create_component(
1147 project.name if project.name not in component_names else "Glossary",
1148 "glossary",
1149 self.source_language,
1150 "tbx",
1151 is_glossary=True,
1152 has_template=False,
1153 allow_translation_propagation=False,
1154 license=self.license,
1155 )
1157 @cached_property
1158 def lock(self):
1159 return WeblateLock(
1160 lock_path=self.project.full_path,
1161 scope="component-update",
1162 key=self.pk,
1163 slug=self.slug,
1164 cache_template="{scope}-lock-{key}",
1165 file_template="{slug}-update.lock",
1166 timeout=5,
1167 origin=self.full_slug,
1168 )
1170 @cached_property
1171 def update_key(self) -> str:
1172 return f"component-update-{self.pk}"
1174 def delete_background_task(self) -> None:
1175 cache.delete(self.update_key)
1177 def store_background_task(self, task=None) -> None:
1178 if task is None: 1178 ↛ 1182line 1178 didn't jump to line 1182 because the condition on line 1178 was always true
1179 if not current_task:
1180 return
1181 task = current_task.request
1182 cache.set(self.update_key, task.id, 6 * 3600)
1184 @cached_property
1185 def background_task_id(self):
1186 return cache.get(self.update_key)
1188 @cached_property
1189 def background_task(self):
1190 task_id = self.background_task_id
1191 if not task_id:
1192 return None
1193 return AsyncResult(task_id)
1195 def progress_step(self, progress=None) -> None:
1196 # No task (for example eager mode)
1197 if not current_task or not current_task.request.id:
1198 return
1199 # Operate on linked component if needed
1200 if self.translations_count == -1: 1200 ↛ 1201line 1200 didn't jump to line 1201 because the condition on line 1200 was never true
1201 if self.linked_component:
1202 self.linked_component.progress_step(progress)
1203 return
1204 # Calculate progress for translations
1205 if progress is None:
1206 self.translations_progress += 1
1207 progress = 100 * self.translations_progress // self.translations_count
1208 # Store task state
1209 current_task.update_state(
1210 state="PROGRESS", meta={"progress": progress, "component": self.pk}
1211 )
1213 def store_log(self, slug, msg, *args) -> None:
1214 if self.translations_count == -1 and self.linked_component: 1214 ↛ 1215line 1214 didn't jump to line 1215 because the condition on line 1214 was never true
1215 self.linked_component.store_log(slug, msg, *args)
1216 return
1217 self.logs.append(f"{slug}: {msg % args}")
1218 if current_task and current_task.request.id:
1219 cache.set(f"task-log-{current_task.request.id}", self.logs, 2 * 3600)
1221 def log_hook(self, level, msg, *args) -> None:
1222 if level != "DEBUG":
1223 self.store_log(self.full_slug, msg, *args)
1225 def get_progress(self):
1226 task = self.background_task
1227 if task is None:
1228 return 100, []
1229 progress = get_task_progress(task)
1230 return (progress, cache.get(f"task-log-{task.id}", []))
1232 def in_progress(self):
1233 return (
1234 not settings.CELERY_TASK_ALWAYS_EAGER
1235 and self.background_task is not None
1236 and not self.background_task.ready()
1237 )
1239 def get_source_translation(self):
1240 """
1241 Return source translation object if it exists.
1243 In some cases we do not want to create source translation object as
1244 source_translation property does, but we want to utilize its cache.
1245 """
1246 if "source_translation" in self.__dict__:
1247 return self.__dict__["source_translation"]
1248 try:
1249 result = self.translation_set.get(language_id=self.source_language_id)
1250 except ObjectDoesNotExist:
1251 return None
1252 self.__dict__["source_translation"] = result
1253 return result
1255 @cached_property
1256 def source_translation(self):
1257 # This is basically copy of get_or_create, but avoids additional
1258 # SQL query to get source_language in case the source translation
1259 # already exists. The source_language is only fetched in the slow
1260 # path when creating the translation.
1261 language = self.source_language
1262 try:
1263 result = self.translation_set.select_related("plural").get(
1264 language=language
1265 )
1266 except ObjectDoesNotExist:
1267 try:
1268 with transaction.atomic():
1269 return self.translation_set.create(
1270 language=language,
1271 check_flags="read-only",
1272 filename=self.template,
1273 plural=self.file_format_cls.get_plural(language),
1274 language_code=language.code,
1275 )
1276 except IntegrityError:
1277 try:
1278 return self.translation_set.get(language_id=self.source_language_id)
1279 except self.translation_set.model.DoesNotExist:
1280 pass
1281 raise
1282 else:
1283 result.language = language
1284 return result
1286 def preload_sources(self, sources=None) -> None:
1287 """Preload source objects to improve performance on load."""
1288 if sources is not None:
1289 self._sources = sources
1290 else:
1291 self._sources = {
1292 source.id_hash: source
1293 for source in self.source_translation.unit_set.all()
1294 }
1295 self._sources_prefetched = True
1297 def get_all_sources(self):
1298 if not self._sources_prefetched: 1298 ↛ 1300line 1298 didn't jump to line 1300 because the condition on line 1298 was always true
1299 self.preload_sources()
1300 return list(self._sources.values())
1302 def unload_sources(self) -> None:
1303 self._sources = {}
1304 self._sources_prefetched = False
1306 def _process_new_source(self, source: Unit, *, save: bool = True) -> Change:
1307 # Avoid fetching empty list of checks from the database
1308 source.all_checks = []
1309 source.source_updated = True
1310 change = source.generate_change(
1311 self.acting_user,
1312 self.acting_user,
1313 ActionEvents.NEW_SOURCE,
1314 check_new=False,
1315 save=save,
1316 )
1317 self.updated_sources[source.id] = source
1318 return change
1320 def bulk_create_sources(self, attributes_list: list[UnitAttributesDict]) -> None:
1321 """Ensure that all sources are stored in the database."""
1322 # Can not bulk create with getting primary key, resort to one by one creation
1323 if not connection.features.can_return_rows_from_bulk_insert: 1323 ↛ 1324line 1323 didn't jump to line 1324 because the condition on line 1323 was never true
1324 return
1326 # Make sure we load all the sources
1327 if not self._sources_prefetched: 1327 ↛ 1328line 1327 didn't jump to line 1328 because the condition on line 1327 was never true
1328 self.preload_sources()
1330 # Filter out missing ones
1331 missing = [
1332 attributes
1333 for attributes in attributes_list
1334 if attributes["id_hash"] not in self._sources
1335 ]
1337 if not missing:
1338 return
1340 # Bulk create
1341 # Needs to be in sync with Unit.update_source_unit
1342 units = [
1343 Unit(
1344 translation=self.source_translation,
1345 id_hash=attributes["id_hash"],
1346 source=attributes["source"],
1347 target=attributes["source"],
1348 context=attributes["context"],
1349 position=attributes["pos"],
1350 note=attributes["note"],
1351 location=attributes["location"],
1352 explanation=attributes["explanation"],
1353 flags=attributes["flags"].format(),
1354 num_words=count_words(attributes["source"], self.source_language),
1355 state=STATE_TRANSLATED
1356 if self.template and self.edit_template
1357 else STATE_READONLY,
1358 )
1359 for attributes in missing
1360 ]
1361 Unit.objects.bulk_create(units)
1363 changes: list[Change] = []
1365 for unit in units:
1366 # Fill in source_unit
1367 unit.source_unit = unit
1369 # Track in local cache
1370 self._sources[unit.id_hash] = unit
1372 # Postprocess and create change
1373 changes.append(self._process_new_source(unit, save=False))
1375 # Update source unit in the database
1376 Unit.objects.bulk_update(units, fields=["source_unit"])
1378 # Store changes in the database
1379 Change.objects.bulk_create(changes)
1381 def get_source(self, id_hash, create=None):
1382 """Get source info with caching."""
1383 from weblate.trans.models import Unit
1385 # Preload sources when creating units
1386 if not self._sources_prefetched and create: 1386 ↛ 1387line 1386 didn't jump to line 1387 because the condition on line 1386 was never true
1387 self.preload_sources()
1389 try:
1390 return self._sources[id_hash]
1391 except KeyError:
1392 source_units = self.source_translation.unit_set
1393 if not self._sources_prefetched:
1394 # Fetch one by one for case getting for single unit, if not prefetch
1395 # was done, this will raise an exception in case of error
1396 source = source_units.get(id_hash=id_hash)
1397 elif create:
1398 # Create in case of parsing translations
1399 # Set correct state depending on template editing
1400 if self.template and self.edit_template:
1401 create["state"] = STATE_TRANSLATED
1402 else:
1403 create["state"] = STATE_READONLY
1405 # Create source unit
1406 source = Unit(
1407 translation=self.source_translation, id_hash=id_hash, **create
1408 )
1409 source.save(force_insert=True, only_save=True)
1410 self._process_new_source(source)
1411 else:
1412 # We are not supposed to create new one
1413 msg = "Could not find source unit"
1414 raise Unit.DoesNotExist(msg) from None
1416 self._sources[id_hash] = source
1417 return source
1419 @property
1420 def filemask_re(self):
1421 # We used to rely on fnmask.translate here, but since Python 3.9
1422 # it became super optimized beast producing regexp with possibly
1423 # several groups making it hard to modify later for our needs.
1424 result: list[str] = []
1425 raw: list[str] = []
1427 def append(text: str | None) -> None:
1428 if raw:
1429 result.append(re.escape("".join(raw)))
1430 raw.clear()
1431 if text is not None:
1432 result.append(text)
1434 for char in self.filemask:
1435 if char == ".":
1436 append(r"\.")
1437 elif char == "*":
1438 append("([^/]*)")
1439 else:
1440 raw.append(char)
1441 append(None)
1442 regex = "".join(result)
1443 return re.compile(f"^{regex}$")
1445 def get_url_path(self):
1446 parent = self.category or self.project
1447 return (*parent.get_url_path(), self.slug)
1449 def get_widgets_url(self) -> str:
1450 """Return absolute URL for widgets."""
1451 return f"{self.project.get_widgets_url()}?component={self.pk}"
1453 def get_share_url(self):
1454 """Return absolute shareable URL."""
1455 return self.project.get_share_url()
1457 @perform_on_link
1458 def _get_path(self):
1459 """Return full path to component VCS repository."""
1460 return super()._get_path()
1462 @perform_on_link
1463 def has_push_configuration(self):
1464 return bool(self.push)
1466 @perform_on_link
1467 def can_push(self):
1468 """Return true if push is possible for this component."""
1469 return self.has_push_configuration() or not self.repository_class.needs_push_url
1471 @property
1472 def is_repo_link(self):
1473 """Check whether a repository is just a link to another one."""
1474 return is_repo_link(self.repo)
1476 @property
1477 def is_repo_local(self) -> bool:
1478 """Check whether a repository is local-only."""
1479 return self.vcs == "local"
1481 @property
1482 def repository_class(self) -> type[Repository]:
1483 try:
1484 return VCS_REGISTRY[self.vcs]
1485 except KeyError as error:
1486 msg = f"Component using VCS {self.vcs}, but it not configured"
1487 raise ImproperlyConfigured(msg) from error
1489 @cached_property
1490 def repository(self) -> Repository:
1491 """Get VCS repository object."""
1492 if self.linked_component is not None: 1492 ↛ 1493line 1492 didn't jump to line 1493 because the condition on line 1492 was never true
1493 return self.linked_component.repository
1494 return self.repository_class(self.full_path, branch=self.branch, component=self)
1496 @perform_on_link
1497 def get_last_remote_commit(self):
1498 """Return latest locally known remote commit."""
1499 if self.is_repo_local or not self.remote_revision: 1499 ↛ 1501line 1499 didn't jump to line 1501 because the condition on line 1499 was always true
1500 return None
1501 try:
1502 return self.repository.get_revision_info(self.remote_revision)
1503 except RepositoryError:
1504 return None
1506 def get_last_commit(self):
1507 """Return latest locally known remote commit."""
1508 if self.is_repo_local or not self.local_revision: 1508 ↛ 1510line 1508 didn't jump to line 1510 because the condition on line 1508 was always true
1509 return None
1510 try:
1511 return self.repository.get_revision_info(self.local_revision)
1512 except RepositoryError:
1513 try:
1514 self.store_local_revision()
1515 except RepositoryError:
1516 return None
1517 return self.repository.get_revision_info(self.local_revision)
1519 @perform_on_link
1520 def get_repo_url(self):
1521 """Return link to repository."""
1522 if not settings.HIDE_REPO_CREDENTIALS:
1523 return self.repo
1524 return cleanup_repo_url(self.repo)
1526 @perform_on_link
1527 def get_repo_branch(self):
1528 """Return branch in repository."""
1529 return self.branch
1531 @perform_on_link
1532 def get_export_url(self):
1533 """Return URL of exported VCS repository."""
1534 return self.git_export
1536 def get_repoweb_link(
1537 self,
1538 filename: str,
1539 line: str,
1540 template: str | None = None,
1541 user=None,
1542 ):
1543 """
1544 Generate link to source code browser for given file and line.
1546 For linked repositories, it is possible to override the linked repository path
1547 here.
1548 """
1549 if not template:
1550 if self.repoweb:
1551 template = self.repoweb
1552 elif user and user.has_perm("vcs.view", self):
1553 template = getattr(
1554 self,
1555 f"get_{self.vcs}_repoweb_template",
1556 self.get_git_repoweb_template,
1557 )()
1558 if self.linked_component is not None:
1559 return self.linked_component.get_repoweb_link(
1560 filename, line, template, user=self.acting_user
1561 )
1562 if not template:
1563 if filename.startswith("https://"):
1564 return filename
1565 return None
1567 return render_template(
1568 template,
1569 filename=urlquote(filename),
1570 line=urlquote(line),
1571 branch=urlquote(self.branch),
1572 component=self,
1573 )
1575 def get_git_repoweb_template(self):
1576 """Return the template link for a specific vcs."""
1577 repo = self.repo
1578 if repo == "local:":
1579 return None
1581 parsed_url = urlparse(repo)
1583 # Make sure this is a string
1584 parsed_hostname = parsed_url.hostname or ""
1586 if repo.startswith("git@bitbucket.org") or parsed_hostname == "bitbucket.org":
1587 return self.get_bitbucket_git_repoweb_template()
1589 if repo.startswith("git@github.com") or parsed_hostname == "github.com":
1590 return self.get_github_repoweb_template()
1592 if parsed_hostname == "pagure.io":
1593 return self.get_pagure_repoweb_template()
1595 if (
1596 repo.startswith("git@ssh.dev.azure.com:v3")
1597 or parsed_hostname == "dev.azure.com"
1598 or parsed_hostname.endswith(".visualstudio.com")
1599 ):
1600 return self.get_azure_repoweb_template()
1602 return None
1604 def get_clean_slug(self, slug):
1605 if slug.endswith(".git"):
1606 return slug[:-4]
1607 return slug
1609 def get_bitbucket_git_repoweb_template(self) -> str | None:
1610 owner, slug, matches = None, None, None
1611 domain = "bitbucket.org"
1612 matches = re.match(BITBUCKET_GIT_REPOS_REGEXP[0], self.repo)
1613 if matches is None:
1614 matches = re.match(BITBUCKET_GIT_REPOS_REGEXP[1], self.repo)
1615 if matches:
1616 owner = matches.group(1)
1617 slug = self.get_clean_slug(matches.group(2))
1618 if owner and slug:
1619 return (
1620 f"https://{domain}/{owner}/{slug}/blob/{{branch}}/{{filename}}#{{line}}"
1621 )
1623 return None
1625 def get_github_repoweb_template(self) -> str | None:
1626 owner, slug, matches = None, None, None
1627 domain = "github.com"
1628 matches = re.match(GITHUB_REPOS_REGEXP[0], self.repo)
1629 if matches is None:
1630 matches = re.match(GITHUB_REPOS_REGEXP[1], self.repo)
1631 if matches:
1632 owner = matches.group(1)
1633 slug = self.get_clean_slug(matches.group(2))
1634 if owner and slug:
1635 return f"https://{domain}/{owner}/{slug}/blob/{{branch}}/{{filename}}#L{{line}}"
1637 return None
1639 def get_pagure_repoweb_template(self) -> str | None:
1640 owner, slug = None, None
1641 domain = "pagure.io"
1642 if matches := re.match(PAGURE_REPOS_REGEXP[0], self.repo):
1643 owner = matches.group(1)
1644 slug = matches.group(2)
1646 if owner and slug:
1647 return f"https://{domain}/{owner}/{slug}/blob/{{branch}}/f/{{filename}}/#_{{line}}"
1649 return None
1651 def get_azure_repoweb_template(self) -> str | None:
1652 organization, project, repository, matches = None, None, None, None
1653 domain = "dev.azure.com"
1654 matches = re.match(AZURE_REPOS_REGEXP[0], self.repo)
1655 if matches is None:
1656 matches = re.match(AZURE_REPOS_REGEXP[1], self.repo)
1657 if matches is None:
1658 matches = re.match(AZURE_REPOS_REGEXP[2], self.repo)
1659 if matches is None:
1660 matches = re.match(AZURE_REPOS_REGEXP[3], self.repo)
1662 if matches:
1663 organization = matches.group(1)
1664 project = matches.group(2)
1665 repository = matches.group(3)
1667 if organization and project and repository:
1668 return f"https://{domain}/{organization}/{project}/_git/{repository}/blob/{{branch}}/{{filename}}#L{{line}}"
1670 return None
1672 def error_text(self, error):
1673 """Return text message for a RepositoryError."""
1674 message = error.get_message()
1675 if not settings.HIDE_REPO_CREDENTIALS: 1675 ↛ 1676line 1675 didn't jump to line 1676 because the condition on line 1675 was never true
1676 return message
1677 return cleanup_repo_url(self.repo, message)
1679 def add_ssh_host_key(self) -> None:
1680 """
1681 Add SSH key for current repo as trusted.
1683 This is essentially a TOFU approach.
1684 """
1686 def add(repo) -> None:
1687 self.log_info("checking for key to add for %s", repo)
1688 parsed = urlparse(repo)
1689 if not parsed.hostname:
1690 parsed = urlparse(f"ssh://{repo}")
1691 if not parsed.hostname:
1692 return
1693 try:
1694 port = parsed.port
1695 except ValueError:
1696 port = ""
1697 self.log_info("adding SSH key for %s:%s", parsed.hostname, port)
1698 add_host_key(None, parsed.hostname, port)
1700 add(self.repo)
1701 if self.push:
1702 add(self.push)
1704 def handle_update_error(self, error_text, retry) -> None:
1705 if "Host key verification failed" in error_text: 1705 ↛ 1706line 1705 didn't jump to line 1706 because the condition on line 1705 was never true
1706 if retry:
1707 # Add ssh key and retry
1708 self.add_ssh_host_key()
1709 return
1710 raise ValidationError(
1711 {
1712 "repo": gettext(
1713 "Could not verify SSH host key, please add "
1714 "them in SSH page in the admin interface."
1715 )
1716 }
1717 )
1718 if "terminal prompts disabled" in error_text: 1718 ↛ 1719line 1718 didn't jump to line 1719 because the condition on line 1718 was never true
1719 raise ValidationError(
1720 {
1721 "repo": gettext(
1722 "The repository requires authentication, please specify "
1723 "credentials in the URL or use SSH access instead."
1724 )
1725 }
1726 )
1727 raise ValidationError(
1728 {"repo": gettext("Could not fetch the repository: %s") % error_text}
1729 )
1731 @perform_on_link
1732 def update_remote_branch(self, validate=False, retry=True):
1733 """Pull from remote repository."""
1734 # Update
1735 self.log_info("updating repository")
1736 try:
1737 with self.repository.lock:
1738 start = time.monotonic()
1739 try:
1740 previous_revision = self.repository.last_remote_revision
1741 except RepositoryError:
1742 # Repository not yet configured
1743 previous_revision = None
1744 self.repository.update_remote()
1745 timediff = time.monotonic() - start
1746 self.log_info("update took %.2f seconds", timediff)
1747 except RepositoryError as error:
1748 report_error(
1749 "Could not update the repository",
1750 project=self.project,
1751 skip_sentry=not settings.DEBUG,
1752 )
1753 error_text = self.error_text(error)
1754 if validate: 1754 ↛ 1757line 1754 didn't jump to line 1757 because the condition on line 1754 was always true
1755 self.handle_update_error(error_text, retry)
1756 return self.update_remote_branch(True, False)
1757 if self.id:
1758 self.add_alert("UpdateFailure", error=error_text)
1759 return False
1761 for line in self.repository.last_output.splitlines():
1762 self.log_debug("update: %s", line)
1763 try:
1764 # This can actually fail without a remote repo
1765 remote_revision = self.repository.last_remote_revision
1766 except RepositoryError:
1767 remote_revision = None
1768 if previous_revision and remote_revision: 1768 ↛ 1777line 1768 didn't jump to line 1777 because the condition on line 1768 was always true
1769 if previous_revision == remote_revision: 1769 ↛ 1772line 1769 didn't jump to line 1772 because the condition on line 1769 was always true
1770 self.log_info("repository up to date at %s", previous_revision)
1771 else:
1772 self.log_info(
1773 "repository updated from %s to %s",
1774 previous_revision,
1775 remote_revision,
1776 )
1777 if self.id:
1778 self.delete_alert("UpdateFailure")
1779 if remote_revision and previous_revision != remote_revision: 1779 ↛ 1780line 1779 didn't jump to line 1780 because the condition on line 1779 was never true
1780 self.remote_revision = remote_revision
1781 Component.objects.filter(pk=self.pk).update(
1782 remote_revision=remote_revision
1783 )
1784 return True
1786 def configure_repo(self, validate=False, pull=True) -> None:
1787 """Ensure repository is correctly set up."""
1788 if self.is_repo_link: 1788 ↛ 1789line 1788 didn't jump to line 1789 because the condition on line 1788 was never true
1789 return
1791 if self.is_repo_local:
1792 if not os.path.exists(os.path.join(self.full_path, ".git")):
1793 if ( 1793 ↛ 1801line 1793 didn't jump to line 1801 because the condition on line 1793 was never true
1794 not self.template
1795 and not self.file_format_cls.create_empty_bilingual
1796 and not hasattr(self.file_format_cls, "update_bilingual")
1797 ) or (
1798 self.template
1799 and self.file_format_cls.get_new_file_content() is None
1800 ):
1801 raise ValidationError({"template": gettext("File does not exist.")})
1802 LocalRepository.from_files(
1803 self.full_path,
1804 {self.template: self.file_format_cls.get_new_file_content()}
1805 if self.template
1806 else {},
1807 )
1808 return
1810 with self.repository.lock:
1811 self.repository.configure_remote(
1812 self.repo, self.push, self.branch, fast=not self.id
1813 )
1814 self.repository.set_committer(
1815 settings.DEFAULT_COMMITER_NAME, settings.DEFAULT_COMMITER_EMAIL
1816 )
1818 if pull: 1818 ↛ exitline 1818 didn't jump to the function exit
1819 self.update_remote_branch(validate)
1821 def configure_branch(self) -> None:
1822 """Ensure local tracking branch exists and is checked out."""
1823 if self.is_repo_link: 1823 ↛ 1824line 1823 didn't jump to line 1824 because the condition on line 1823 was never true
1824 return
1826 with self.repository.lock:
1827 self.repository.configure_branch(self.branch)
1829 def uses_changed_files(self, changed):
1830 """Detect whether list of changed files matches configuration."""
1831 for filename in [self.template, self.intermediate, self.new_base]:
1832 if filename and filename in changed: 1832 ↛ 1833line 1832 didn't jump to line 1833 because the condition on line 1832 was never true
1833 return True
1834 return any(self.filemask_re.match(path) for path in changed)
1836 def needs_commit_upstream(self) -> bool:
1837 """
1838 Detect whether commit is needed because of upstream changes.
1840 Inspect changed files in the upstream repository to see if any of them
1841 would trigger parsing of translation files. In case there is none, the
1842 repository can be merged without committing pending changes.
1843 """
1844 changed = self.repository.get_changed_files()
1845 if self.uses_changed_files(changed): 1845 ↛ 1846line 1845 didn't jump to line 1846 because the condition on line 1845 was never true
1846 return True
1847 for component in self.linked_children: 1847 ↛ 1848line 1847 didn't jump to line 1848 because the loop on line 1847 never started
1848 if component.uses_changed_files(changed):
1849 return True
1850 return False
1852 @perform_on_link
1853 def do_update(self, request: AuthenticatedHttpRequest | None = None, method=None):
1854 """Perform repository update."""
1855 self.translations_progress = 0
1856 self.translations_count = 0
1857 # Hold lock all time here to avoid somebody writing between commit
1858 # and merge/rebase.
1859 with self.repository.lock:
1860 self.store_background_task()
1861 self.progress_step(0)
1862 self.configure_repo(pull=False)
1864 # pull remote
1865 if not self.update_remote_branch(): 1865 ↛ 1866line 1865 didn't jump to line 1866 because the condition on line 1865 was never true
1866 return False
1868 self.configure_branch()
1870 # do we have something to merge?
1871 try:
1872 needs_merge = self.repo_needs_merge()
1873 except RepositoryError:
1874 # Not yet configured repository
1875 needs_merge = True
1877 if not needs_merge and method != "rebase":
1878 self.delete_alert("MergeFailure")
1879 self.delete_alert("RepositoryOutdated")
1880 return True
1882 # commit possible pending changes if needed
1883 if self.needs_commit_upstream(): 1883 ↛ 1884line 1883 didn't jump to line 1884 because the condition on line 1883 was never true
1884 self.commit_pending(
1885 "update", request.user if request else None, skip_push=True
1886 )
1888 # update local branch
1889 try:
1890 result = self.update_branch(request, method=method, skip_push=True)
1891 except RepositoryError:
1892 result = False
1894 if result: 1894 ↛ 1896line 1894 didn't jump to line 1896 because the condition on line 1894 was never true
1895 # create translation objects for all files
1896 self.create_translations(request=request)
1898 # Push after possible merge
1899 self.push_if_needed(do_update=False)
1901 if not self.repo_needs_push(): 1901 ↛ 1904line 1901 didn't jump to line 1904 because the condition on line 1901 was always true
1902 self.delete_alert("RepositoryChanges")
1904 self.progress_step(100)
1905 self.translations_count = None
1907 return result
1909 @perform_on_link
1910 def push_if_needed(self, do_update=True) -> None:
1911 """
1912 Push changes to upstream if needed.
1914 Checks for:
1916 * Pushing on commit
1917 * Configured push
1918 * Whether there is something to push
1919 """
1920 if not self.push_on_commit:
1921 self.log_info("skipped push: push on commit disabled")
1922 return
1923 if not self.can_push(): 1923 ↛ 1926line 1923 didn't jump to line 1926 because the condition on line 1923 was always true
1924 self.log_info("skipped push: upstream not configured")
1925 return
1926 if not self.repo_needs_push():
1927 self.log_info(
1928 "skipped push: nothing to push (%d/%d outgoing)",
1929 self.count_repo_outgoing,
1930 self.count_push_branch_outgoing,
1931 )
1932 return
1933 if settings.CELERY_TASK_ALWAYS_EAGER:
1934 self.do_push(None, force_commit=False, do_update=do_update)
1935 else:
1936 from weblate.trans.tasks import perform_push
1938 self.log_info("scheduling push")
1939 perform_push.delay_on_commit(
1940 self.pk, None, force_commit=False, do_update=do_update
1941 )
1943 @perform_on_link
1944 def push_repo(self, request: AuthenticatedHttpRequest, retry: bool = True):
1945 """Push repository changes upstream."""
1946 with self.repository.lock:
1947 self.log_info("pushing to remote repo")
1948 try:
1949 self.repository.push(self.push_branch)
1950 except RepositoryError as error:
1951 error_text = self.error_text(error)
1952 report_error(
1953 "Could not push the repo",
1954 project=self.project,
1955 skip_sentry=not settings.DEBUG,
1956 )
1957 self.change_set.create(
1958 action=ActionEvents.FAILED_PUSH,
1959 target=error_text,
1960 user=request.user if request else self.acting_user,
1961 )
1962 if retry:
1963 if "Host key verification failed" in error_text:
1964 # Try adding SSH key and retry
1965 self.add_ssh_host_key()
1966 return self.push_repo(request, retry=False)
1967 if "fetch first" in error_text:
1968 # Upstream has moved, try additional update via calling do_push
1969 return self.do_push(request, retry=False)
1970 if (
1971 "shallow update not allowed" in error_text
1972 or "expected old/new/ref, got 'shallow" in error_text
1973 ):
1974 try:
1975 self.repository.unshallow()
1976 except RepositoryError:
1977 report_error(
1978 "Could not unshallow the repo",
1979 project=self.project,
1980 skip_sentry=not settings.DEBUG,
1981 )
1982 else:
1983 return self.push_repo(request, retry=False)
1984 messages.error(
1985 request,
1986 gettext("Could not push %(component)s: %(error_text)s")
1987 % {
1988 "component": self,
1989 "error_text": error_text,
1990 },
1991 )
1992 self.add_alert("PushFailure", error=error_text)
1993 return False
1994 self.delete_alert("RepositoryChanges")
1995 self.delete_alert("PushFailure")
1996 return True
1998 @property
1999 def pushes_to_different_location(self) -> bool:
2000 return (
2001 self.branch != self.push_branch
2002 or self.repository.pushes_to_different_location
2003 )
2005 @perform_on_link
2006 def do_push(
2007 self,
2008 request,
2009 force_commit: bool = True,
2010 do_update: bool = True,
2011 retry: bool = True,
2012 ) -> bool:
2013 """Push changes to remote repo."""
2014 # Skip push for local only repo
2015 if self.is_repo_local: 2015 ↛ 2019line 2015 didn't jump to line 2019 because the condition on line 2015 was always true
2016 return True
2018 # Do we have push configured
2019 if not self.can_push():
2020 messages.error(request, gettext("Push is turned off for %s.") % self)
2021 return False
2023 # Commit any pending changes
2024 if force_commit:
2025 self.commit_pending(
2026 "push", request.user if request else None, skip_push=True
2027 )
2029 # Do we have anything to push?
2030 if not self.repo_needs_push():
2031 return True
2033 if do_update:
2034 # Update the repo
2035 self.do_update(request)
2037 # Were all changes merged?
2038 if not self.pushes_to_different_location and self.repo_needs_merge():
2039 return False
2041 # Send pre push signal
2042 vcs_pre_push.send(sender=self.__class__, component=self)
2043 for component in self.linked_children:
2044 vcs_pre_push.send(sender=component.__class__, component=component)
2046 # Do actual push
2047 result = self.push_repo(request, retry=retry)
2048 if not result:
2049 return False
2051 self.change_set.create(
2052 action=ActionEvents.PUSH,
2053 user=request.user if request else self.acting_user,
2054 )
2056 vcs_post_push.send(sender=self.__class__, component=self)
2057 for component in self.linked_children:
2058 vcs_post_push.send(sender=component.__class__, component=component)
2060 return True
2062 @perform_on_link
2063 def do_reset(
2064 self,
2065 request: AuthenticatedHttpRequest | None = None,
2066 *,
2067 keep_changes: bool = False,
2068 ) -> bool:
2069 """Reset repo to match remote."""
2070 from weblate.trans.tasks import perform_commit
2072 with self.repository.lock:
2073 previous_head = self.repository.last_revision
2074 # First check we're up to date
2075 self.update_remote_branch()
2077 if keep_changes:
2078 # Mark all strings as pending when keeping changes
2079 self.do_file_sync(request, do_commit=False)
2080 else:
2081 # Explicitly remove all pending changes
2082 PendingUnitChange.objects.filter(
2083 unit__translation__component=self
2084 ).delete()
2086 # Do actual reset
2087 try:
2088 self.log_info("resetting to remote repo")
2089 self.repository.reset()
2090 except RepositoryError:
2091 report_error(
2092 "Could not reset the repository",
2093 project=self.project,
2094 skip_sentry=not settings.DEBUG,
2095 )
2096 messages.error(
2097 request,
2098 gettext("Could not reset to remote branch on %s.") % self,
2099 )
2100 return False
2102 self.change_set.create(
2103 action=ActionEvents.RESET,
2104 user=request.user if request else self.acting_user,
2105 details={
2106 "new_head": self.repository.last_revision,
2107 "previous_head": previous_head,
2108 },
2109 )
2110 self.delete_alert("MergeFailure")
2111 self.delete_alert("RepositoryOutdated")
2112 self.delete_alert("PushFailure")
2114 if not keep_changes:
2115 self.trigger_post_update(
2116 previous_head=previous_head,
2117 skip_push=False,
2118 )
2120 # create translation objects for all files
2121 self.create_translations(request=request, force=True)
2123 if keep_changes:
2124 # Trigger commit and scan in the background
2125 perform_commit.delay_on_commit(
2126 self.pk,
2127 "reset-sync",
2128 user_id=request.user.id if request else None,
2129 force_scan=True,
2130 previous_head=previous_head,
2131 )
2132 return True
2134 @perform_on_link
2135 def do_cleanup(self, request: AuthenticatedHttpRequest | None = None) -> bool:
2136 """Clean up the repository."""
2137 with self.repository.lock:
2138 try:
2139 self.log_info("cleaning up the repo")
2140 self.repository.cleanup()
2141 except RepositoryError:
2142 report_error(
2143 "Could not clean the repository",
2144 project=self.project,
2145 skip_sentry=not settings.DEBUG,
2146 )
2147 messages.error(
2148 request,
2149 gettext("Could not clean the repository on %s.") % self,
2150 )
2151 return False
2153 return True
2155 @perform_on_link
2156 @transaction.atomic
2157 def do_file_sync(
2158 self, request: AuthenticatedHttpRequest | None = None, *, do_commit: bool = True
2159 ) -> None:
2160 from weblate.trans.models import Unit
2161 from weblate.trans.tasks import perform_commit
2163 for unit in Unit.objects.filter(
2164 Q(translation__component=self)
2165 | Q(translation__component__linked_component=self)
2166 ).exclude(
2167 Q(translation__language_id=F("translation__component__source_language_id"))
2168 | Q(translation__filename="")
2169 ):
2170 PendingUnitChange.store_unit_change(unit)
2172 self.change_set.create(
2173 action=ActionEvents.FORCE_SYNC,
2174 user=self.acting_user or (request.user if request else None),
2175 )
2177 if do_commit:
2178 perform_commit.delay_on_commit(
2179 self.pk, "file-sync", user_id=request.user.id if request else None
2180 )
2182 @perform_on_link
2183 @transaction.atomic
2184 def do_file_scan(self, request: AuthenticatedHttpRequest | None = None) -> bool:
2185 self.commit_pending("file-scan", request.user if request else None)
2186 self.change_set.create(
2187 action=ActionEvents.FORCE_SCAN,
2188 user=self.acting_user or (request.user if request else None),
2189 )
2191 self.create_translations(request=request, force=True)
2192 return True
2194 def get_repo_link_url(self):
2195 return "weblate://{}".format("/".join(self.get_url_path()))
2197 @cached_property
2198 def linked_children(self) -> ComponentQuerySet:
2199 """Return list of components which links repository to us."""
2200 if self.is_repo_link: 2200 ↛ 2201line 2200 didn't jump to line 2201 because the condition on line 2200 was never true
2201 return self.component_set.none()
2202 children = self.component_set.prefetch()
2203 for child in children: 2203 ↛ 2204line 2203 didn't jump to line 2204 because the loop on line 2203 never started
2204 child.linked_component = self
2205 return children
2207 def get_linked_children_for_template(self):
2208 return [
2209 {
2210 "project_name": linked.project.name,
2211 "name": linked.name,
2212 "url": get_site_url(linked.get_absolute_url()),
2213 }
2214 for linked in self.linked_children
2215 ]
2217 @perform_on_link
2218 def commit_pending( # noqa: C901
2219 self, reason: str, user: User | None, skip_push: bool = False
2220 ) -> bool:
2221 """Check whether there is any translation to be committed."""
2222 from weblate.auth.models import User
2224 def reuse_self(translation: Translation) -> Translation:
2225 if translation.component_id == self.id: 2225 ↛ 2227line 2225 didn't jump to line 2227 because the condition on line 2225 was always true
2226 translation.component = self
2227 if translation.component.linked_component_id == self.id: 2227 ↛ 2228line 2227 didn't jump to line 2228 because the condition on line 2227 was never true
2228 translation.component.linked_component = self
2229 if translation.pk == translation.component.source_translation.pk: 2229 ↛ 2230line 2229 didn't jump to line 2230 because the condition on line 2229 was never true
2230 translation = translation.component.source_translation
2231 return translation
2233 if user is None:
2234 user = User.objects.get_or_create_bot(
2235 scope="weblate", name="commit", verbose="Background commit"
2236 )
2238 # Get all translation with pending changes, source translation first
2239 translations = sorted(
2240 Translation.objects.filter(unit__pending_changes__isnull=False)
2241 .filter(Q(component=self) | Q(component__linked_component=self))
2242 .distinct()
2243 .prefetch_related("component"),
2244 key=lambda translation: not translation.is_source,
2245 )
2246 components = {}
2247 skipped = set()
2248 source_updated_components = []
2249 translation_pks = defaultdict(list)
2250 was_changed: bool = False
2252 if not translations:
2253 return True
2255 # Commit pending changes
2256 with self.repository.lock:
2257 for translation in translations:
2258 translation = reuse_self(translation)
2259 component = translation.component
2260 if component.pk in skipped: 2260 ↛ 2262line 2260 didn't jump to line 2262 because the condition on line 2260 was never true
2261 # We already failed at this component
2262 continue
2263 if component.pk not in components:
2264 # Validate template is valid
2265 if component.has_template(): 2265 ↛ 2266line 2265 didn't jump to line 2266 because the condition on line 2265 was never true
2266 try:
2267 component.template_store # noqa: B018
2268 except FileParseError as error:
2269 if not isinstance(error.__cause__, FileNotFoundError):
2270 report_error(
2271 "Could not parse template file on commit",
2272 project=self.project,
2273 )
2274 component.log_error(
2275 "skipping commit due to error: %s", error
2276 )
2277 component.update_import_alerts(delete=False)
2278 skipped.add(component.pk)
2279 continue
2281 components[component.pk] = component
2282 with self.start_sentry_span("commit_pending"):
2283 translation_changed = translation._commit_pending(reason, user) # noqa: SLF001
2284 was_changed |= translation_changed
2285 if translation_changed and component.has_template(): 2285 ↛ 2286line 2285 didn't jump to line 2286 because the condition on line 2285 was never true
2286 translation_pks[component.pk].append(translation.pk)
2287 if translation.is_source:
2288 source_updated_components.append(component)
2290 # Update hash of other translations, otherwise they would be seen as having change
2291 for component in source_updated_components: 2291 ↛ 2292line 2291 didn't jump to line 2292 because the loop on line 2291 never started
2292 for translation in component.translation_set.exclude(
2293 pk__in=translation_pks[component.pk]
2294 ):
2295 translation.store_hash()
2297 if was_changed: 2297 ↛ 2309line 2297 didn't jump to line 2309 because the condition on line 2297 was always true
2298 self.store_local_revision()
2300 # Fire postponed post commit signals
2301 for component in components.values():
2302 component.send_post_commit_signal()
2303 component.update_import_alerts(delete=False)
2305 # Push if enabled
2306 if not skip_push: 2306 ↛ 2309line 2306 didn't jump to line 2309 because the condition on line 2306 was always true
2307 self.push_if_needed()
2309 return True
2311 def commit_files(
2312 self,
2313 *,
2314 template: str | None = None,
2315 author: str | None = None,
2316 timestamp: datetime | None = None,
2317 files: list[str] | None = None,
2318 signals: bool = True,
2319 skip_push: bool = False,
2320 extra_context: dict[str, Any] | None = None,
2321 message: str | None = None,
2322 component: models.Model | None = None,
2323 store_hash: bool = True,
2324 ):
2325 """Commit files to the repository."""
2326 linked = self.linked_component
2327 if linked: 2327 ↛ 2328line 2327 didn't jump to line 2328 because the condition on line 2327 was never true
2328 return linked.commit_files(
2329 template=template,
2330 author=author,
2331 timestamp=timestamp,
2332 files=files,
2333 signals=signals,
2334 skip_push=skip_push,
2335 extra_context=extra_context,
2336 message=message,
2337 component=self,
2338 )
2340 with self.start_sentry_span("commit_files"):
2341 if message is None: 2341 ↛ 2354line 2341 didn't jump to line 2354 because the condition on line 2341 was always true
2342 if template is None: 2342 ↛ 2343line 2342 didn't jump to line 2343 because the condition on line 2342 was never true
2343 msg = "Missing template when message is not specified"
2344 raise ValueError(msg)
2345 # Handle context
2346 context = {"component": component or self, "author": author}
2347 if extra_context: 2347 ↛ 2351line 2347 didn't jump to line 2351 because the condition on line 2347 was always true
2348 context.update(extra_context)
2350 # Generate commit message
2351 message = render_template(template, **context)
2353 # Actual commit
2354 if not self.repository.commit(message, author, localtime(timestamp), files):
2355 return False
2357 # Send post commit signal
2358 if signals:
2359 self.send_post_commit_signal(store_hash=store_hash)
2361 self.store_local_revision()
2363 # Push if we should
2364 if not skip_push:
2365 self.push_if_needed()
2367 return True
2369 def send_post_commit_signal(self, store_hash: bool = True) -> None:
2370 vcs_post_commit.send(
2371 sender=self.__class__, component=self, store_hash=store_hash
2372 )
2374 def get_parse_error_message(self, error) -> str:
2375 error_message = getattr(error, "strerror", "")
2376 if not error_message:
2377 error_message = getattr(error, "message", "")
2378 if not error_message:
2379 error_message = str(error).replace(self.full_path, "")
2380 return error_message
2382 def handle_parse_error(
2383 self, error, translation=None, filename=None, reraise: bool = True
2384 ) -> None:
2385 """Process parse errors."""
2386 error_message = self.get_parse_error_message(error)
2387 if filename is None:
2388 filename = self.template if translation is None else translation.filename
2389 self.trigger_alert("ParseError", error=error_message, filename=filename)
2390 if self.id:
2391 self.change_set.create(
2392 translation=translation,
2393 action=ActionEvents.PARSE_ERROR,
2394 details={"error_message": error_message, "filename": filename},
2395 user=self.acting_user,
2396 )
2397 if reraise:
2398 raise FileParseError(error_message) from error
2400 def store_local_revision(self) -> None:
2401 """Store current revision in the database."""
2402 self.local_revision = self.repository.last_revision
2403 # Avoid using using save as that does complex things and we
2404 # just want to update the database
2405 Component.objects.filter(Q(pk=self.pk) | Q(linked_component=self)).update(
2406 local_revision=self.local_revision
2407 )
2409 @perform_on_link
2410 def update_branch(
2411 self,
2412 request: AuthenticatedHttpRequest | None = None,
2413 method: str | None = None,
2414 skip_push: bool = False,
2415 ) -> bool:
2416 """Update current branch to match remote (if possible)."""
2417 if method is None:
2418 method = self.merge_style
2419 user = request.user if request else self.acting_user
2420 # run pre update hook
2421 vcs_pre_update.send(sender=self.__class__, component=self)
2422 for component in self.linked_children: 2422 ↛ 2423line 2422 didn't jump to line 2423 because the loop on line 2422 never started
2423 vcs_pre_update.send(sender=component.__class__, component=component)
2425 # Apply logic for merge or rebase
2426 if method == "rebase":
2427 method_func = self.repository.rebase
2428 error_msg = gettext("Could not rebase local branch onto remote branch %s.")
2429 action = ActionEvents.REBASE
2430 action_failed = ActionEvents.FAILED_REBASE
2431 kwargs = {}
2432 else:
2433 method_func = self.repository.merge
2434 error_msg = gettext("Could not merge remote branch into %s.")
2435 action = ActionEvents.MERGE
2436 action_failed = ActionEvents.FAILED_MERGE
2437 kwargs = {"message": render_template(self.merge_message, component=self)}
2438 if method == "merge_noff": 2438 ↛ 2439line 2438 didn't jump to line 2439 because the condition on line 2438 was never true
2439 kwargs["no_ff"] = True
2441 with self.repository.lock:
2442 try:
2443 previous_head = self.repository.last_revision
2444 # Try to merge it
2445 method_func(**kwargs)
2446 new_head = self.repository.last_revision
2447 self.log_info(
2448 "%s remote into repo %s..%s", method, previous_head, new_head
2449 )
2450 except RepositoryError as error:
2451 # Report error
2452 report_error(
2453 f"Failed {method}",
2454 project=self.project,
2455 skip_sentry=not settings.DEBUG,
2456 )
2458 # In case merge has failure recover
2459 error = self.error_text(error)
2460 status = self.repository.status()
2462 # Log error
2463 if self.id:
2464 self.change_set.create(
2465 action=action_failed,
2466 target=error,
2467 user=user,
2468 details={"error": error, "status": status},
2469 )
2470 self.add_alert("MergeFailure", error=error)
2472 # Reset repo back
2473 method_func(abort=True)
2475 # Tell user (if there is any)
2476 messages.error(request, error_msg % self)
2478 raise
2480 # Delete alerts
2481 if self.id: 2481 ↛ 2488line 2481 didn't jump to line 2488 because the condition on line 2481 was always true
2482 self.delete_alert("MergeFailure")
2483 self.delete_alert("RepositoryOutdated")
2484 if not self.repo_needs_push(): 2484 ↛ 2488line 2484 didn't jump to line 2488 because the condition on line 2484 was always true
2485 self.delete_alert("PushFailure")
2487 # New upstream matches previous local revision
2488 if self.local_revision == new_head:
2489 return False
2491 if self.id: 2491 ↛ 2511line 2491 didn't jump to line 2511
2492 self.store_local_revision()
2494 # Record change
2495 self.change_set.create(
2496 action=action,
2497 user=user,
2498 details={"new_head": new_head, "previous_head": previous_head},
2499 )
2501 # The files have been updated and the signal receivers (addons)
2502 # might need to access the template
2503 self.drop_template_store_cache()
2505 # Run post update hook, this should be done with repo lock held
2506 # to avoid possible race with another update
2507 self.trigger_post_update(
2508 previous_head=previous_head,
2509 skip_push=skip_push,
2510 )
2511 return True
2513 @perform_on_link
2514 def trigger_post_update(self, *, previous_head: str, skip_push: bool) -> None:
2515 vcs_post_update.send(
2516 sender=self.__class__,
2517 component=self,
2518 previous_head=previous_head,
2519 skip_push=skip_push,
2520 )
2521 for component in self.linked_children: 2521 ↛ 2522line 2521 didn't jump to line 2522 because the loop on line 2521 never started
2522 vcs_post_update.send(
2523 sender=component.__class__,
2524 component=component,
2525 previous_head=previous_head,
2526 skip_push=skip_push,
2527 )
2529 def get_mask_matches(self) -> list[str]:
2530 """Return files matching current mask."""
2531 prefix = path_separator(os.path.join(self.full_path, ""))
2532 matches = set()
2533 for filename in glob(os.path.join(self.full_path, self.filemask)):
2534 path = path_separator(filename).replace(prefix, "")
2535 code = self.get_lang_code(path)
2536 if re.match(self.language_regex, code) and code != "source": 2536 ↛ 2539line 2536 didn't jump to line 2539 because the condition on line 2536 was always true
2537 matches.add(path)
2538 else:
2539 self.log_info("skipping language %s [%s]", code, path)
2541 # Remove symlinked translations
2542 for filename in list(matches):
2543 resolved = self.repository.resolve_symlinks(filename)
2544 if resolved != filename and resolved in matches: 2544 ↛ 2545line 2544 didn't jump to line 2545 because the condition on line 2544 was never true
2545 matches.discard(filename)
2547 if self.has_template(): 2547 ↛ 2549line 2547 didn't jump to line 2549 because the condition on line 2547 was never true
2548 # We do not want to show intermediate translation standalone
2549 if self.intermediate:
2550 matches.discard(self.intermediate)
2551 # We want to list template among translations as well
2552 matches.discard(self.template)
2553 return [self.template, *sorted(matches)]
2554 return sorted(matches)
2556 def update_source_checks(self) -> None:
2557 self.log_info("running source checks for %d strings", len(self.updated_sources))
2558 for unit in self.updated_sources.values():
2559 unit.is_batch_update = True
2560 unit.run_checks()
2561 self.updated_sources = {}
2563 @cached_property
2564 def all_active_alerts(self):
2565 result = self.alert_set.filter(dismissed=False)
2566 list(result)
2567 return result
2569 @cached_property
2570 def all_alerts(self):
2571 return {alert.name: alert for alert in self.alert_set.all()}
2573 @property
2574 def lock_alerts(self):
2575 if not self.auto_lock_error:
2576 return []
2577 return [
2578 alert for alert in self.all_active_alerts if alert.name in LOCKING_ALERTS
2579 ]
2581 def trigger_alert(self, name: str, **kwargs) -> None:
2582 if name in self.alerts_trigger:
2583 self.alerts_trigger[name].append(kwargs)
2584 else:
2585 self.alerts_trigger[name] = [kwargs]
2587 def delete_alert(self, alert: str) -> None:
2588 if alert in self.all_alerts:
2589 self.all_alerts[alert].delete()
2590 del self.all_alerts[alert]
2591 if ( 2591 ↛ 2605line 2591 didn't jump to line 2605 because the condition on line 2591 was never true
2592 self.locked
2593 and self.auto_lock_error
2594 and alert in LOCKING_ALERTS
2595 and not self.alert_set.filter(name__in=LOCKING_ALERTS).exists()
2596 and getattr(
2597 # The object might not exist
2598 self.change_set.filter(action=ActionEvents.LOCK)
2599 .order_by("-id")
2600 .first(),
2601 "auto_status",
2602 None,
2603 )
2604 ):
2605 self.do_lock(user=None, lock=False, auto=True)
2607 if ALERTS[alert].link_wide:
2608 for component in self.linked_children: 2608 ↛ 2609line 2608 didn't jump to line 2609 because the loop on line 2608 never started
2609 component.delete_alert(alert)
2611 def add_alert(self, alert: str, noupdate: bool = False, **details) -> None:
2612 if alert in self.all_alerts:
2613 obj = self.all_alerts[alert]
2614 created = False
2615 else:
2616 obj, created = self.alert_set.get_or_create(
2617 name=alert, defaults={"details": details}
2618 )
2619 self.all_alerts[alert] = obj
2621 # Automatically lock on error
2622 if created and self.auto_lock_error and alert in LOCKING_ALERTS: 2622 ↛ 2623line 2622 didn't jump to line 2623 because the condition on line 2622 was never true
2623 self.do_lock(user=None, lock=True, auto=True)
2625 # Update details with exception of component removal
2626 if not created and not noupdate:
2627 obj.details = details
2628 obj.save()
2630 if ALERTS[alert].link_wide: 2630 ↛ 2631line 2630 didn't jump to line 2631 because the condition on line 2630 was never true
2631 for component in self.linked_children:
2632 component.add_alert(alert, noupdate=noupdate, **details)
2634 def update_import_alerts(self, delete: bool = True) -> None:
2635 self.log_info("checking triggered alerts")
2636 for alert in ALERTS_IMPORT:
2637 if alert in self.alerts_trigger:
2638 self.add_alert(alert, occurrences=self.alerts_trigger[alert])
2639 elif delete:
2640 self.delete_alert(alert)
2641 self.alerts_trigger = {}
2643 def create_translations(
2644 self,
2645 *,
2646 force: bool = False,
2647 force_scan: bool = False,
2648 langs: list[str] | None = None,
2649 request: AuthenticatedHttpRequest | None = None,
2650 changed_template: bool = False,
2651 from_link: bool = False,
2652 change: int | None = None,
2653 ) -> bool:
2654 """Load translations from VCS."""
2655 if settings.CELERY_TASK_ALWAYS_EAGER: 2655 ↛ 2659line 2655 didn't jump to line 2659 because the condition on line 2655 was never true
2656 # Asynchronous processing not available, run the update
2657 # directly from the request processing.
2658 # NOTE: In case the lock cannot be acquired, an error will be raised.
2659 return self.create_translations_immediate(
2660 force=force,
2661 force_scan=force_scan,
2662 langs=langs,
2663 request=request,
2664 changed_template=changed_template,
2665 from_link=from_link,
2666 change=change,
2667 )
2669 from weblate.trans.tasks import perform_load
2671 self.log_info("scheduling update in background")
2672 perform_load.delay_on_commit(
2673 pk=self.pk,
2674 force=force,
2675 force_scan=force_scan,
2676 langs=langs,
2677 changed_template=changed_template,
2678 from_link=from_link,
2679 change=change,
2680 user_id=request.user.id if request is not None else None,
2681 )
2682 return False
2684 def create_translations_immediate(
2685 self,
2686 *,
2687 force: bool = False,
2688 force_scan: bool = False,
2689 langs: list[str] | None = None,
2690 request: AuthenticatedHttpRequest | None = None,
2691 changed_template: bool = False,
2692 from_link: bool = False,
2693 change: int | None = None,
2694 ) -> bool:
2695 """
2696 Load translations from VCS synchronously.
2698 Should not be called directly, except from Celery tasks.
2699 """
2700 # In case the lock cannot be acquired, an error will be raised.
2701 with self.lock, self.start_sentry_span("create_translations"): # pylint: disable=not-context-manager
2702 return self._create_translations(
2703 force=force,
2704 force_scan=force_scan,
2705 langs=langs,
2706 request=request,
2707 changed_template=changed_template,
2708 from_link=from_link,
2709 change=change,
2710 )
2712 def check_template_valid(self) -> None:
2713 if self._template_check_done:
2714 return
2715 if self.has_template(): 2715 ↛ 2717line 2715 didn't jump to line 2717 because the condition on line 2715 was never true
2716 # Avoid parsing if template is invalid
2717 try:
2718 self.template_store.check_valid()
2719 except (ValueError, FileParseError) as exc:
2720 raise InvalidTemplateError(info=str(exc)) from exc
2721 self._template_check_done = True
2723 def _create_translations( # noqa: C901,PLR0915
2724 self,
2725 *,
2726 force: bool = False,
2727 force_scan: bool = False,
2728 langs: list[str] | None = None,
2729 request: AuthenticatedHttpRequest | None = None,
2730 changed_template: bool = False,
2731 from_link: bool = False,
2732 change: int | None = None,
2733 ) -> bool:
2734 """Load translations from VCS."""
2735 from weblate.trans.tasks import update_enforced_checks
2737 self.store_background_task()
2739 # Store the revision as add-ons might update it later
2740 current_revision = self.local_revision
2742 if (
2743 self.processed_revision == current_revision
2744 and self.local_revision
2745 and not force
2746 and not force_scan
2747 ):
2748 self.log_info("this revision has been already parsed, skipping update")
2749 self.progress_step(100)
2750 return False
2752 # Ensure we start from fresh template
2753 self.drop_template_store_cache()
2754 self.unload_sources()
2755 self.needs_cleanup = False
2756 self.updated_sources = {}
2757 self.alerts_trigger = {}
2758 self.start_batched_checks()
2759 was_change = False
2760 translations = {}
2761 languages = {}
2762 matches = self.get_mask_matches()
2764 source_file = self.template
2766 if not self.has_template(): 2766 ↛ 2787line 2766 didn't jump to line 2787 because the condition on line 2766 was always true
2767 # This creates the translation when necessary
2768 translation = self.source_translation
2770 if (
2771 self.file_format == "po"
2772 and self.new_base.endswith(".pot")
2773 and os.path.exists(self.get_new_base_filename())
2774 ):
2775 # Process pot file as source to include additional metadata
2776 matches = [self.new_base, *matches]
2777 source_file = self.new_base
2778 else:
2779 # Always include source language to avoid parsing matching files
2780 languages[self.source_language.code] = translation
2781 translations[translation.id] = translation
2783 # Delete old source units after change from monolingual to bilingual
2784 if changed_template: 2784 ↛ 2785line 2784 didn't jump to line 2785 because the condition on line 2784 was never true
2785 translation.unit_set.all().delete()
2787 if self.translations_count != -1: 2787 ↛ 2792line 2787 didn't jump to line 2792 because the condition on line 2787 was always true
2788 self.translations_progress = 0
2789 self.translations_count = len(matches) + sum(
2790 c.translation_set.count() for c in self.linked_children
2791 )
2792 for pos, path in enumerate(matches):
2793 if not self._sources_prefetched and path != source_file:
2794 self.preload_sources()
2795 with transaction.atomic():
2796 if path == source_file:
2797 code = self.source_language.code
2798 else:
2799 code = self.get_lang_code(path)
2800 if langs is not None and code not in langs: 2800 ↛ 2801line 2800 didn't jump to line 2801 because the condition on line 2800 was never true
2801 self.log_info("skipping %s", path)
2802 continue
2804 self.log_info(
2805 "checking %s (%s) [%d/%d]", path, code, pos + 1, len(matches)
2806 )
2807 lang = Language.objects.auto_get_or_create(
2808 code=self.get_language_alias(code),
2809 languages_cache=self.project.languages_cache,
2810 )
2811 if lang.code in languages: 2811 ↛ 2812line 2811 didn't jump to line 2812 because the condition on line 2811 was never true
2812 codes = f"{code}, {languages[lang.code].language_code}"
2813 filenames = f"{path}, {languages[lang.code].filename}"
2814 detail = f"{lang.code} ({codes})"
2815 self.log_warning("duplicate language found: %s", detail)
2816 self.trigger_alert(
2817 "DuplicateLanguage",
2818 codes=codes,
2819 language_code=lang.code,
2820 filenames=filenames,
2821 )
2822 continue
2823 try:
2824 translation = Translation.objects.check_sync(
2825 self,
2826 lang,
2827 code,
2828 path,
2829 force,
2830 request=request,
2831 change=change,
2832 )
2833 except InvalidTemplateError as error:
2834 self.log_warning(
2835 "skipping update due to error in parsing template: %s",
2836 error.__cause__,
2837 )
2838 self.handle_parse_error(error.__cause__, filename=self.template)
2839 self.update_import_alerts()
2840 raise error.__cause__ from error # pylint: disable=E0710
2841 was_change |= bool(translation.reason)
2842 translations[translation.id] = translation
2843 languages[lang.code] = translation
2844 # Unload the store to save memory as we won't need it again
2845 translation.drop_store_cache()
2846 # Remove fuzzy flag on template name change
2847 if changed_template and self.template: 2847 ↛ 2848line 2847 didn't jump to line 2848 because the condition on line 2847 was never true
2848 translation.unit_set.filter(state=STATE_FUZZY).update(
2849 state=STATE_TRANSLATED
2850 )
2851 self.progress_step()
2853 # Delete possibly no longer existing translations
2854 if langs is None: 2854 ↛ 2868line 2854 didn't jump to line 2868 because the condition on line 2854 was always true
2855 todelete = self.translation_set.exclude(id__in=translations.keys())
2856 if todelete.exists(): 2856 ↛ 2857line 2856 didn't jump to line 2857 because the condition on line 2856 was never true
2857 self.needs_cleanup = True
2858 with transaction.atomic():
2859 self.log_info(
2860 "removing stale translations: %s",
2861 ",".join(trans.language.code for trans in todelete),
2862 )
2863 todelete.delete()
2864 # Indicate a change to invalidate stats
2865 was_change = True
2867 # Update import alerts
2868 self.update_import_alerts()
2869 # Clean no matches alert if there are translations:
2870 if translations: 2870 ↛ 2874line 2870 didn't jump to line 2874 because the condition on line 2870 was always true
2871 self.delete_alert("NoMaskMatches")
2873 # Process linked repos
2874 for pos, component in enumerate(self.linked_children): 2874 ↛ 2875line 2874 didn't jump to line 2875 because the loop on line 2874 never started
2875 self.log_info(
2876 "updating linked project %s [%d/%d]",
2877 component,
2878 pos + 1,
2879 len(self.linked_children),
2880 )
2881 component.translations_count = -1
2882 try:
2883 # Do not run these linked repos update as other background tasks.
2884 was_change |= component.create_translations_immediate(
2885 force=force, langs=langs, request=request, from_link=True
2886 )
2887 except FileParseError as error:
2888 if not isinstance(error.__cause__, FileNotFoundError):
2889 report_error("Failed linked component update", project=self.project)
2890 continue
2892 # Run source checks on updated source strings
2893 if self.updated_sources:
2894 self.update_source_checks()
2896 # Update flags
2897 if was_change: 2897 ↛ 2901line 2897 didn't jump to line 2901 because the condition on line 2897 was always true
2898 self.invalidate_cache()
2900 # Schedule background cleanup if needed
2901 if self.needs_cleanup and not self.template:
2902 from weblate.trans.tasks import cleanup_component
2904 cleanup_component.delay_on_commit(self.id)
2906 if was_change: 2906 ↛ 2912line 2906 didn't jump to line 2912 because the condition on line 2906 was always true
2907 if self.needs_variants_update: 2907 ↛ 2908line 2907 didn't jump to line 2908 because the condition on line 2907 was never true
2908 self.update_variants()
2909 component_post_update.send(sender=self.__class__, component=self)
2910 self.schedule_sync_terminology()
2912 self.unload_sources()
2913 self.run_batched_checks()
2915 # Update last processed revision
2916 self.processed_revision = current_revision
2917 # Avoid using save() here
2918 Component.objects.filter(pk=self.pk).update(processed_revision=current_revision)
2920 if self.enforced_checks: 2920 ↛ 2921line 2920 didn't jump to line 2921 because the condition on line 2920 was never true
2921 update_enforced_checks.delay_on_commit(component=self.pk)
2923 self.log_info("updating completed")
2924 return was_change
2926 def start_batched_checks(self) -> None:
2927 self.batch_checks = True
2928 self.batched_checks = set()
2930 def run_batched_checks(self) -> None:
2931 # Batch checks
2932 if self.batched_checks:
2933 from weblate.checks.tasks import batch_update_checks
2935 batched_checks = list(self.batched_checks)
2936 if settings.CELERY_TASK_ALWAYS_EAGER: 2936 ↛ 2937line 2936 didn't jump to line 2937 because the condition on line 2936 was never true
2937 batch_update_checks(self.id, batched_checks, component=self)
2938 else:
2939 batch_update_checks.delay_on_commit(self.id, batched_checks)
2940 self.batch_checks = False
2941 self.batched_checks = set()
2943 def _invalidate_trigger(self) -> None:
2944 self._invalidate_scheduled = False
2945 self.log_info("updating stats caches")
2946 self.stats.update_language_stats()
2947 self.invalidate_glossary_cache()
2949 def invalidate_cache(self) -> None:
2950 if self._invalidate_scheduled: 2950 ↛ 2951line 2950 didn't jump to line 2951 because the condition on line 2950 was never true
2951 return
2953 self._invalidate_scheduled = True
2954 transaction.on_commit(self._invalidate_trigger)
2956 @cached_property
2957 def glossary_sources_key(self) -> str:
2958 return f"component-glossary-{self.pk}"
2960 @cached_property
2961 def glossary_sources(self):
2962 from weblate.glossary.models import get_glossary_sources
2964 result = cache.get(self.glossary_sources_key)
2965 if result is None:
2966 result = get_glossary_sources(self)
2967 cache.set(self.glossary_sources_key, result, 24 * 3600)
2968 return result
2970 def invalidate_glossary_cache(self) -> None:
2971 if not self.is_glossary:
2972 return
2973 cache.delete(self.glossary_sources_key)
2974 self.project.invalidate_glossary_cache()
2975 for project in self.cached_links: 2975 ↛ 2976line 2975 didn't jump to line 2976 because the loop on line 2975 never started
2976 project.invalidate_glossary_cache()
2977 if "glossary_sources" in self.__dict__: 2977 ↛ 2978line 2977 didn't jump to line 2978 because the condition on line 2977 was never true
2978 del self.__dict__["glossary_sources"]
2980 def get_lang_code(self, path, validate=False):
2981 """Parse language code from path."""
2982 # Directly return source language code unless validating
2983 if not validate and path == self.template: 2983 ↛ 2984line 2983 didn't jump to line 2984 because the condition on line 2983 was never true
2984 return self.source_language.code
2985 # Parse filename
2986 matches = self.filemask_re.match(path)
2988 if not matches or not matches.lastindex: 2988 ↛ 2989line 2988 didn't jump to line 2989 because the condition on line 2988 was never true
2989 if path == self.template:
2990 return self.source_language.code
2991 return ""
2993 # Use longest matched code
2994 code = max(matches.groups(), key=len)
2996 # Remove possible encoding part
2997 if "." in code and (".utf" in code.lower() or ".iso" in code.lower()): 2997 ↛ 2998line 2997 didn't jump to line 2998 because the condition on line 2997 was never true
2998 return code.split(".")[0]
2999 return code
3001 def sync_git_repo(
3002 self,
3003 *,
3004 validate: bool = False,
3005 skip_push: bool = False,
3006 skip_commit: bool = False,
3007 ) -> None:
3008 """Bring VCS repo in sync with current model."""
3009 if self.is_repo_link: 3009 ↛ 3010line 3009 didn't jump to line 3010 because the condition on line 3009 was never true
3010 return
3011 if skip_push is None: 3011 ↛ 3012line 3011 didn't jump to line 3012 because the condition on line 3011 was never true
3012 skip_push = validate
3013 self.configure_repo(validate)
3014 if not skip_commit and self.id:
3015 self.commit_pending("sync", None, skip_push=skip_push)
3016 self.configure_branch()
3017 if self.id:
3018 # Update existing repo
3019 self.update_branch(skip_push=skip_push)
3020 else:
3021 # Reset to upstream in case not yet saved model (this is called
3022 # from the clean method only)
3023 with self.repository.lock:
3024 self.update_remote_branch()
3025 self.repository.reset()
3027 def set_default_branch(self) -> None:
3028 """Set default VCS branch if empty."""
3029 if not self.branch and not self.is_repo_link:
3030 self.branch = self.repository_class.get_remote_branch(self.repo)
3032 def clean_category(self) -> None:
3033 if self.category: 3033 ↛ 3034line 3033 didn't jump to line 3034 because the condition on line 3033 was never true
3034 if self.category.project != self.project:
3035 raise ValidationError(
3036 {"category": gettext("Category does not belong to this project.")}
3037 )
3038 if self.pk and self.links.exists():
3039 raise ValidationError(
3040 gettext("Categorized component can not be shared.")
3041 )
3043 def clean_repo_link(self) -> None:
3044 """Validate repository link."""
3045 if self.is_repo_link: 3045 ↛ 3046line 3045 didn't jump to line 3046 because the condition on line 3045 was never true
3046 try:
3047 repo = Component.objects.get_linked(self.repo)
3048 except (Component.DoesNotExist, ValueError) as error:
3049 raise ValidationError(
3050 {
3051 "repo": gettext(
3052 "Invalid link to a Weblate project, "
3053 "use weblate://project/component."
3054 )
3055 }
3056 ) from error
3057 else:
3058 if repo is not None and repo.is_repo_link:
3059 raise ValidationError(
3060 {
3061 "repo": gettext(
3062 "Invalid link to a Weblate project, "
3063 "cannot link to linked repository!"
3064 )
3065 }
3066 )
3067 if repo.pk == self.pk:
3068 raise ValidationError(
3069 {
3070 "repo": gettext(
3071 "Invalid link to a Weblate project, "
3072 "cannot link it to itself!"
3073 )
3074 }
3075 )
3076 # Push repo is not used with link
3077 for setting in ("push", "branch", "push_branch"):
3078 if getattr(self, setting):
3079 raise ValidationError(
3080 {
3081 setting: gettext(
3082 "Option is not available for linked repositories. "
3083 "Setting from linked component will be used."
3084 )
3085 }
3086 )
3087 # Make sure we are not using stale link even if link is not present
3088 self.linked_component = Component.objects.get_linked(self.repo)
3090 def clean_lang_codes(self, matches: list[str]) -> None:
3091 """Validate that there are no double language codes."""
3092 if not matches and not self.is_valid_base_for_new():
3093 raise ValidationError(
3094 {"filemask": gettext("The file mask did not match any files.")}
3095 )
3096 langs: dict[str, str] = {}
3097 existing_langs: set[str] = set()
3099 for match in matches:
3100 code = self.get_lang_code(match, validate=True)
3101 lang = validate_language_code(self.get_language_alias(code), match, True)
3102 if lang.code in langs: 3102 ↛ 3103line 3102 didn't jump to line 3103 because the condition on line 3102 was never true
3103 message = gettext(
3104 "There is more than one file for %(language)s language: "
3105 "%(filename1)s, %(filename2)s "
3106 "Please adjust the file mask and use components for translating "
3107 "different resources."
3108 ) % {
3109 "language": lang,
3110 "filename1": match,
3111 "filename2": langs[lang.code],
3112 }
3113 raise ValidationError({"filemask": message})
3115 langs[lang.code] = match
3116 if lang.id: 3116 ↛ 3099line 3116 didn't jump to line 3099 because the condition on line 3116 was always true
3117 existing_langs.add(lang.code)
3119 # No languages matched our definition
3120 if not existing_langs and langs: 3120 ↛ 3121line 3120 didn't jump to line 3121 because the condition on line 3120 was never true
3121 message = gettext(
3122 "Could not find any matching language, please check the file mask."
3123 )
3124 raise ValidationError({"filemask": message})
3126 def clean_files(self, matches: list[str]) -> None:
3127 """Validate that translation files can be parsed."""
3128 errors: list[tuple[str, Exception]] = []
3129 dir_path = self.full_path
3130 for match in matches:
3131 try:
3132 store = self.file_format_cls( # pylint: disable=too-many-function-args,unexpected-keyword-arg
3133 os.path.join(dir_path, match),
3134 self.template_store,
3135 file_format_params=self.file_format_params,
3136 )
3137 store.check_valid()
3138 except Exception as error:
3139 errors.append((match, error))
3140 if errors:
3141 if len(errors) == 1: 3141 ↛ 3142line 3141 didn't jump to line 3142 because the condition on line 3141 was never true
3142 msg = format_html(
3143 gettext("Could not parse {file}: {error}"),
3144 file=format_html("<code>{}</code>", errors[0][0]),
3145 error=errors[0][1],
3146 )
3147 else:
3148 msg = format_html(
3149 "{}<br>{}",
3150 ngettext(
3151 "Could not parse %d matched file.",
3152 "Could not parse %d matched files.",
3153 len(errors),
3154 )
3155 % len(errors),
3156 format_html_join(
3157 mark_safe("<br>"),
3158 "<code>{}</code>: {}",
3159 errors,
3160 ),
3161 )
3162 raise ValidationError({"filemask": msg})
3164 def is_valid_base_for_new(
3165 self, errors: list[Exception] | None = None, fast: bool = False
3166 ) -> bool:
3167 filename = self.get_new_base_filename()
3168 template = self.has_template()
3169 return self.file_format_cls.is_valid_base_for_new(
3170 filename,
3171 template,
3172 errors,
3173 fast=fast,
3174 file_format_params=self.file_format_params,
3175 )
3177 def clean_new_lang(self) -> None:
3178 """Validate new language choices."""
3179 # Validate if new base is configured or language adding is set
3180 if (not self.new_base and self.new_lang != "add") or not self.file_format:
3181 return
3182 # File is valid or no file is needed
3183 errors: list[Exception] = []
3184 if self.is_valid_base_for_new(errors): 3184 ↛ 3187line 3184 didn't jump to line 3187 because the condition on line 3184 was always true
3185 return
3186 # File is needed, but not present
3187 if not self.new_base:
3188 message = gettext(
3189 "You have set up Weblate to add new translation "
3190 "files, but did not provide a base file to do that."
3191 )
3192 raise ValidationError({"new_base": message, "new_lang": message})
3193 filename = self.get_new_base_filename()
3194 # File is present, but does not exist
3195 if not os.path.exists(filename):
3196 raise ValidationError({"new_base": gettext("File does not exist.")})
3197 # File is present, but it is not valid
3198 if errors:
3199 message = gettext(
3200 "Could not parse base file for new translations: %s"
3201 ) % format_html_join_comma("{}", list_to_tuples(errors))
3202 raise ValidationError({"new_base": message})
3203 raise ValidationError(
3204 {"new_base": gettext("Unrecognized base file for new translations.")}
3205 )
3207 def clean_template(self) -> None:
3208 """Validate template value."""
3209 # Test for unexpected template usage
3210 if ( 3210 ↛ 3215line 3210 didn't jump to line 3215 because the condition on line 3210 was never true
3211 self.template
3212 and self.file_format
3213 and self.file_format_cls.monolingual is False
3214 ):
3215 msg = gettext("You can not use a base file for bilingual translation.")
3216 raise ValidationError({"template": msg, "file_format": msg})
3218 if self.edit_template and not self.file_format_cls.can_edit_base:
3219 msg = gettext("Editing template is not supported with this file format.")
3220 raise ValidationError({"edit_template": msg})
3222 # Prohibit intermediate usage without template
3223 if self.intermediate and not self.template: 3223 ↛ 3224line 3223 didn't jump to line 3224 because the condition on line 3223 was never true
3224 msg = gettext(
3225 "An intermediate language file can not be used "
3226 "without an editing template."
3227 )
3228 raise ValidationError({"template": msg, "intermediate": msg})
3229 if self.intermediate and self.intermediate == self.template: 3229 ↛ 3230line 3229 didn't jump to line 3230 because the condition on line 3229 was never true
3230 raise ValidationError(
3231 {
3232 # Translators: it refers to the intermediate field in the form
3233 "intermediate": gettext(
3234 "An intermediate language file has to be different from "
3235 "monolingual base language file. You can probably keep it "
3236 "empty."
3237 )
3238 }
3239 )
3240 if self.intermediate and not self.edit_template: 3240 ↛ 3241line 3240 didn't jump to line 3241 because the condition on line 3240 was never true
3241 msg = gettext(
3242 "An intermediate language file can not be used "
3243 "without an editing template."
3244 )
3245 raise ValidationError({"edit_template": msg, "intermediate": msg})
3247 # Special case for Gettext
3248 if self.template.endswith(".pot") and self.filemask.endswith(".po"): 3248 ↛ 3249line 3248 didn't jump to line 3249 because the condition on line 3248 was never true
3249 msg = gettext("Using a .pot file as base file is unsupported.")
3250 raise ValidationError({"template": msg})
3252 if not self.file_format: 3252 ↛ 3253line 3252 didn't jump to line 3253 because the condition on line 3252 was never true
3253 return
3255 # Validate template loading
3256 if self.has_template(): 3256 ↛ 3257line 3256 didn't jump to line 3257 because the condition on line 3256 was never true
3257 self.create_template_if_missing()
3258 full_path = os.path.join(self.full_path, self.template)
3259 if not os.path.exists(full_path):
3260 raise ValidationError({"template": gettext("File does not exist.")})
3262 try:
3263 self.template_store.check_valid()
3264 except (FileParseError, ValueError) as error:
3265 msg = gettext("Could not parse translation base file: %s") % str(error)
3266 raise ValidationError({"template": msg}) from error
3268 code = self.get_lang_code(self.template, validate=True)
3269 lang = validate_language_code(
3270 self.get_language_alias(code), self.template, required=False
3271 )
3272 if lang:
3273 lang_code = lang.base_code
3274 if lang_code != self.source_language.base_code:
3275 msg = gettext(
3276 "Template language ({0}) does not match source language ({1})!"
3277 ).format(lang_code, self.source_language.code)
3278 raise ValidationError({"template": msg, "source_language": msg})
3280 elif self.file_format_cls.monolingual:
3281 msg = gettext(
3282 "You can not use a monolingual translation without a base file."
3283 )
3284 raise ValidationError({"template": msg})
3286 def clean_repo(self) -> None:
3287 self.clean_repo_link()
3289 # Baild out on failed repo validation
3290 if self.repo is None: 3290 ↛ 3291line 3290 didn't jump to line 3291 because the condition on line 3290 was never true
3291 return
3293 if not self.is_repo_local and self.repo == "local:":
3294 raise ValidationError(
3295 {"vcs": gettext("Choose No remote repository for local: URL.")}
3296 )
3298 if self.is_repo_local and self.push:
3299 raise ValidationError(
3300 {"push": gettext("Push URL is not used without a remote repository.")}
3301 )
3303 # Validate VCS repo
3304 try:
3305 self.set_default_branch()
3307 self.sync_git_repo(validate=True, skip_push=True)
3308 except RepositoryError as error:
3309 text = self.error_text(error)
3310 if "terminal prompts disabled" in text: 3310 ↛ 3311line 3310 didn't jump to line 3311 because the condition on line 3310 was never true
3311 raise ValidationError(
3312 {
3313 "repo": gettext(
3314 "Your push URL seems to miss credentials. Either provide "
3315 "them in the URL or use SSH with key based authentication."
3316 )
3317 }
3318 ) from error
3319 msg = gettext("Could not update repository: %s") % text
3320 raise ValidationError({"repo": msg}) from error
3322 if ( 3322 ↛ 3327line 3322 didn't jump to line 3327 because the condition on line 3322 was never true
3323 issubclass(self.repository_class, GitMergeRequestBase)
3324 and self.repo == self.push
3325 and self.branch == self.push_branch
3326 ):
3327 msg = gettext(
3328 "Pull and push branches cannot be the same when using merge requests."
3329 )
3330 raise ValidationError({"push_branch": msg})
3332 def clean_file_format_params(self) -> None:
3333 for param in [ 3333 ↛ 3336line 3333 didn't jump to line 3336 because the loop on line 3333 never started
3334 p for p in FILE_FORMATS_PARAMS if p.name in self.file_format_params
3335 ]:
3336 if self.file_format not in param.file_formats:
3337 message = gettext(
3338 "The parameter '%(param)s' is not applicable for the file format '%(format)s'."
3339 ) % {"param": param.name, "format": self.file_format}
3340 raise ValidationError({"file_format_params": message})
3342 def clean(self) -> None:
3343 """
3344 Validate component parameters.
3346 - validation fetches repository
3347 - it tries to find translation files and checks that they are valid
3348 """
3349 self.drop_file_format_cache()
3350 if self.new_lang == "url" and not self.project.instructions:
3351 msg = gettext(
3352 "Please either fill in an instruction URL "
3353 "or use a different option for adding a new language."
3354 )
3355 raise ValidationError({"new_lang": msg})
3357 # Skip validation if we don't have valid project
3358 if self.project_id is None or not self.file_format: 3358 ↛ 3359line 3358 didn't jump to line 3359 because the condition on line 3358 was never true
3359 return
3361 # Check if we should rename
3362 changed_git = True
3363 if self.id:
3364 old = Component.objects.get(pk=self.id)
3365 self.check_rename(old, validate=True)
3366 changed_git = (
3367 (old.vcs != self.vcs)
3368 or (old.repo != self.repo)
3369 or (old.push != self.push)
3370 or (old.branch != self.branch)
3371 or (old.filemask != self.filemask)
3372 or (old.language_regex != self.language_regex)
3373 )
3374 if old.source_language != self.source_language: 3374 ↛ 3380line 3374 didn't jump to line 3380 because the condition on line 3374 was never true
3375 # Might be implemented in future, but needs to handle:
3376 # - properly toggle read-only flag for source translation
3377 # - remap screenshots to different units
3378 # - remap source string comments
3379 # - possibly adjust other metadata
3380 raise ValidationError(
3381 {
3382 "source_language": gettext(
3383 "Source language can not be changed, "
3384 "please recreate the component instead."
3385 )
3386 }
3387 )
3389 self.clean_unique_together()
3391 # Check repo if config was changes
3392 if changed_git:
3393 self.drop_repository_cache()
3394 self.clean_repo()
3396 self.clean_category()
3398 # Template validation
3399 self.clean_template()
3401 # New language options
3402 self.clean_new_lang()
3404 # File format parameters
3405 self.clean_file_format_params()
3407 try:
3408 matches = self.get_mask_matches()
3410 # Verify language codes
3411 self.clean_lang_codes(matches)
3413 # Try parsing files
3414 self.clean_files(matches)
3415 except re.error as error:
3416 raise ValidationError(
3417 gettext(
3418 "Can not validate file matches due to invalid regular expression."
3419 )
3420 ) from error
3422 # Suggestions
3423 if ( 3423 ↛ 3428line 3423 didn't jump to line 3428 because the condition on line 3423 was never true
3424 hasattr(self, "suggestion_autoaccept")
3425 and self.suggestion_autoaccept
3426 and not self.suggestion_voting
3427 ):
3428 msg = gettext(
3429 "Accepting suggestions automatically only works with voting turned on."
3430 )
3431 raise ValidationError(
3432 {"suggestion_autoaccept": msg, "suggestion_voting": msg}
3433 )
3435 if self.key_filter and not self.has_template(): 3435 ↛ 3436line 3435 didn't jump to line 3436 because the condition on line 3435 was never true
3436 raise ValidationError(
3437 gettext("To use the key filter, the file format must be monolingual.")
3438 )
3440 def get_template_filename(self):
3441 """Create absolute filename for template."""
3442 return os.path.join(self.full_path, self.template)
3444 def get_intermediate_filename(self):
3445 """Create absolute filename for intermediate."""
3446 return os.path.join(self.full_path, self.intermediate)
3448 def get_new_base_filename(self):
3449 """Create absolute filename for base file for new translations."""
3450 if not self.new_base:
3451 return None
3452 return os.path.join(self.full_path, self.new_base)
3454 def create_template_if_missing(self) -> None:
3455 """Create blank template in case intermediate language is enabled."""
3456 fullname = self.get_template_filename()
3457 if ( 3457 ↛ 3464line 3457 didn't jump to line 3464 because the condition on line 3457 was always true
3458 not self.intermediate
3459 or not self.is_valid_base_for_new()
3460 or os.path.exists(fullname)
3461 or not os.path.exists(self.get_intermediate_filename())
3462 ):
3463 return
3464 self.file_format_cls.add_language(
3465 fullname,
3466 self.source_language,
3467 self.get_new_base_filename(),
3468 file_format_params=self.file_format_params,
3469 )
3471 # Skip commit in case Component is not yet saved (called during validation)
3472 if not self.pk:
3473 return
3475 with self.repository.lock:
3476 self.commit_files(
3477 template=self.add_message,
3478 author="Weblate <noreply@weblate.org>",
3479 extra_context={
3480 "translation": Translation(
3481 filename=self.template,
3482 language_code=self.source_language.code,
3483 language=self.source_language,
3484 component=self,
3485 pk=-1,
3486 )
3487 },
3488 files=[fullname],
3489 )
3491 def after_save(
3492 self,
3493 *,
3494 changed_git: bool,
3495 changed_setup: bool,
3496 changed_template: bool,
3497 changed_variant: bool,
3498 changed_enforced_checks: bool,
3499 skip_push: bool,
3500 create: bool,
3501 ) -> None:
3502 self.store_background_task()
3503 self.translations_progress = 0
3504 self.translations_count = 0
3505 self.progress_step(0)
3506 # Configure git repo if there were changes
3507 if changed_git:
3508 # Bring VCS repo in sync with current model
3509 self.sync_git_repo(skip_push=skip_push, skip_commit=create)
3511 # Create template in case intermediate file is present
3512 self.create_template_if_missing()
3514 # Rescan for possibly new translations if there were changes, needs to
3515 # be done after actual creating the object above
3516 was_change = False
3517 if changed_setup:
3518 was_change = self.create_translations(
3519 force=True, changed_template=changed_template
3520 )
3521 elif changed_git:
3522 was_change = self.create_translations()
3524 # Update variants (create_translation does this on change)
3525 if changed_variant and not was_change: 3525 ↛ 3526line 3525 didn't jump to line 3526 because the condition on line 3525 was never true
3526 self.update_variants()
3528 # Update changed enforced checks
3529 if changed_enforced_checks: 3529 ↛ 3530line 3529 didn't jump to line 3530 because the condition on line 3529 was never true
3530 self.update_enforced_checks()
3532 self.progress_step(100)
3533 self.translations_count = None
3535 # Invalidate stats on template change
3536 if changed_template: 3536 ↛ 3537line 3536 didn't jump to line 3537 because the condition on line 3536 was never true
3537 self.invalidate_cache()
3539 # Update alerts after stats update
3540 self.update_alerts()
3541 if self.linked_component: 3541 ↛ 3542line 3541 didn't jump to line 3542 because the condition on line 3541 was never true
3542 self.linked_component.update_alerts()
3544 # Make sure we create glossary
3545 if create and settings.CREATE_GLOSSARIES:
3546 self.create_glossary()
3548 # Make sure all languages are present
3549 self.schedule_sync_terminology()
3551 # Run automatically installed addons. They are run upon installation,
3552 # but there are no translations created at that point. This complements
3553 # installation in install_autoaddon.
3554 for addon in self.addons_cache["__all__"]: 3554 ↛ 3556line 3554 didn't jump to line 3556 because the loop on line 3554 never started
3555 # Skip addons installed elsewhere (repo/project wide)
3556 if addon.component_id != self.id:
3557 continue
3558 self.log_debug("triggering add-on: %s", addon.name)
3559 addon.addon.post_configure_run()
3561 def update_variants(self, updated_units=None) -> None:
3562 from weblate.trans.models import Unit
3564 component_units = Unit.objects.filter(translation__component=self, variant=None)
3566 if updated_units is None: 3566 ↛ 3571line 3566 didn't jump to line 3571 because the condition on line 3566 was always true
3567 # Assume all units without a variant were updated
3568 process_units = component_units
3569 updated_unit_id_hashes = set()
3570 else:
3571 process_units = updated_units
3572 updated_unit_id_hashes = {unit.id_hash for unit in updated_units}
3574 # Delete stale regex variants
3575 self.variant_set.exclude(variant_regex__in=("", self.variant_regex)).delete()
3577 # Handle regex based variants
3578 if self.variant_regex: 3578 ↛ 3579line 3578 didn't jump to line 3579 because the condition on line 3578 was never true
3579 variant_re = re.compile(self.variant_regex)
3580 units = process_units.filter(context__regex=self.variant_regex)
3581 variant_updates: dict[str, tuple[Variant, list[int]]] = {}
3582 for unit in units.iterator():
3583 if variant_re.findall(unit.context):
3584 key = variant_re.sub("", unit.context)
3585 if key in variant_updates:
3586 variant = variant_updates[key][0]
3587 else:
3588 variant = Variant.objects.get_or_create(
3589 key=key, component=self, variant_regex=self.variant_regex
3590 )[0]
3591 variant_updates[key] = (variant, [])
3592 variant_updates[key][1].append(unit.pk)
3594 if variant_updates:
3595 for variant, unit_ids in variant_updates.values():
3596 # Update matching units
3597 Unit.objects.filter(pk__in=unit_ids).update(variant=variant)
3598 # Update variant links for keys
3599 component_units.filter(context=variant.key).update(variant=variant)
3601 # Update variant links
3602 for variant in self.variant_set.filter(variant_regex="").prefetch_related( 3602 ↛ 3605line 3602 didn't jump to line 3605 because the loop on line 3602 never started
3603 "defining_units"
3604 ):
3605 defining_units = {unit.id_hash for unit in variant.defining_units.all()}
3606 if updated_unit_id_hashes and not updated_unit_id_hashes & defining_units:
3607 continue
3608 # Link based on source string or defining units
3609 component_units.filter(
3610 Q(source=variant.key) | Q(id_hash__in=defining_units)
3611 ).update(variant=variant)
3613 # Delete stale variant links
3614 self.variant_set.annotate(unit_count=Count("defining_units")).filter(
3615 variant_regex="", unit_count=0
3616 ).delete()
3618 def _update_alerts(self) -> None:
3619 self._alerts_scheduled = False
3620 # Flush alerts case, mostly needed for tests
3621 self.__dict__.pop("all_alerts", None)
3623 update_alerts(self)
3625 # Update libre checklist upon save on all components in a project
3626 if ( 3626 ↛ 3631line 3626 didn't jump to line 3631 because the condition on line 3626 was never true
3627 settings.OFFER_HOSTING
3628 and self.project.billings
3629 and self.project.billing.plan.price == 0
3630 ):
3631 for component in self.project.child_components:
3632 update_alerts(component, {"NoLibreConditions"})
3634 def update_alerts(self) -> None:
3635 if self._alerts_scheduled: 3635 ↛ 3636line 3635 didn't jump to line 3636 because the condition on line 3635 was never true
3636 return
3638 self._alerts_scheduled = True
3639 transaction.on_commit(self._update_alerts)
3641 def get_ambiguous_translations(self):
3642 return self.translation_set.filter(language__code__in=AMBIGUOUS.keys())
3644 @property
3645 def count_pending_units(self):
3646 """Return count of pending units."""
3647 from weblate.trans.models import Unit
3649 return (
3650 Unit.objects.filter(
3651 translation__component=self, pending_changes__isnull=False
3652 )
3653 .distinct()
3654 .count()
3655 )
3657 @property
3658 def count_repo_missing(self):
3659 try:
3660 return self.repository.count_missing()
3661 except RepositoryError as error:
3662 report_error(
3663 "Could check merge needed",
3664 project=self.project,
3665 skip_sentry=not settings.DEBUG,
3666 )
3667 self.add_alert("MergeFailure", error=self.error_text(error))
3668 return 0
3670 def _get_count_repo_outgoing(self, retry: bool = True):
3671 try:
3672 return self.repository.count_outgoing()
3673 except RepositoryError as error:
3674 error_text = self.error_text(error)
3675 if retry and "Host key verification failed" in error_text:
3676 self.add_ssh_host_key()
3677 return self._get_count_repo_outgoing(retry=False)
3678 report_error(
3679 "Could check push needed",
3680 project=self.project,
3681 skip_sentry=not settings.DEBUG,
3682 )
3683 self.add_alert("PushFailure", error=error_text)
3684 return 0
3686 @property
3687 def count_repo_outgoing(self):
3688 return self._get_count_repo_outgoing()
3690 @property
3691 def count_push_branch_outgoing(self):
3692 try:
3693 return self.repository.count_outgoing(self.push_branch)
3694 except RepositoryError:
3695 # We silently ignore this error as push branch might not be existing if not needed
3696 return self.count_repo_outgoing
3698 def needs_commit(self):
3699 """Check whether there are some not committed changes."""
3700 return self.count_pending_units > 0
3702 def repo_needs_merge(self):
3703 """Check for unmerged commits from remote repository."""
3704 return self.count_repo_missing > 0
3706 def repo_needs_push(self, retry: bool = True):
3707 """Check for something to push to remote repository."""
3708 return self.count_push_branch_outgoing > 0
3710 @property
3711 def file_format_name(self):
3712 return self.file_format_cls.name
3714 @property
3715 def file_format_create_style(self):
3716 return self.file_format_cls.create_style
3718 @cached_property
3719 def file_format_flags(self):
3720 return Flags(self.file_format_cls.check_flags)
3722 @cached_property
3723 def file_format_cls(self) -> type[TranslationFormat]:
3724 """Return file format object."""
3725 return FILE_FORMATS[self.file_format]
3727 def has_template(self) -> bool:
3728 """Return true if component is using template for translation."""
3729 monolingual = self.file_format_cls.monolingual
3730 return (monolingual or monolingual is None) and bool(self.template)
3732 def drop_template_store_cache(self) -> None:
3733 if "template_store" in self.__dict__:
3734 del self.__dict__["template_store"]
3735 if "intermediate_store" in self.__dict__: 3735 ↛ 3736line 3735 didn't jump to line 3736 because the condition on line 3735 was never true
3736 del self.__dict__["intermediate_store"]
3738 def drop_repository_cache(self) -> None:
3739 if "repository" in self.__dict__: 3739 ↛ 3740line 3739 didn't jump to line 3740 because the condition on line 3739 was never true
3740 del self.__dict__["repository"]
3742 def drop_addons_cache(self) -> None:
3743 if "addons_cache" in self.__dict__:
3744 del self.__dict__["addons_cache"]
3746 def drop_key_filter_cache(self) -> None:
3747 """Invalidate the cached value of key_filter."""
3748 if "key_filter_re" in self.__dict__:
3749 del self.__dict__["key_filter_re"]
3751 def drop_file_format_cache(self) -> None:
3752 self.__dict__.pop("file_format_cls", None)
3754 def load_intermediate_store(self):
3755 """Load translate-toolkit store for intermediate."""
3756 return self.file_format_cls( # pylint: disable=too-many-function-args,unexpected-keyword-arg
3757 self.get_intermediate_filename(),
3758 language_code=self.source_language.code,
3759 source_language=self.source_language.code,
3760 file_format_params=self.file_format_params,
3761 )
3763 @cached_property
3764 def intermediate_store(self):
3765 """Get translate-toolkit store for intermediate."""
3766 # Do we need template?
3767 if not self.has_template() or not self.intermediate:
3768 return None
3770 try:
3771 return self.load_intermediate_store()
3772 except Exception as exc:
3773 self.handle_parse_error(exc, filename=self.intermediate)
3775 def load_template_store(self, fileobj=None):
3776 """Load translate-toolkit store for template."""
3777 with self.start_sentry_span("load_template_store"):
3778 return self.file_format_cls( # pylint: disable=too-many-function-args,unexpected-keyword-arg
3779 fileobj or self.get_template_filename(),
3780 language_code=self.source_language.code,
3781 source_language=self.source_language.code,
3782 is_template=True,
3783 file_format_params=self.file_format_params,
3784 )
3786 @cached_property
3787 def template_store(self):
3788 """Get translate-toolkit store for template."""
3789 # Do we need template?
3790 if not self.has_template(): 3790 ↛ 3793line 3790 didn't jump to line 3793 because the condition on line 3790 was always true
3791 return None
3793 try:
3794 return self.load_template_store()
3795 except Exception as error:
3796 if not isinstance(error, FileNotFoundError):
3797 report_error("Template parse error", project=self.project)
3798 self.handle_parse_error(error, filename=self.template)
3800 @cached_property
3801 def all_flags(self):
3802 """Return parsed list of flags."""
3803 return Flags(self.project.check_flags, self.file_format_flags, self.check_flags)
3805 @property
3806 def is_multivalue(self):
3807 return self.file_format_cls.has_multiple_strings
3809 def can_add_new_language(self, user: User | None, fast: bool = False):
3810 """
3811 Check if a new language can be added.
3813 Generic users can add only if configured, in other situations it works if there
3814 is valid new base.
3815 """
3816 # Consistency and possibly other add-ons
3817 if user is not None and user.is_bot and user.username.startswith("addon:"): 3817 ↛ 3818line 3817 didn't jump to line 3818 because the condition on line 3817 was never true
3818 user = None
3819 # The user is None in case of consistency or cli invocation
3820 # The component.edit permission is intentional here as it allows overriding
3821 # of new_lang configuration for admins and add languages even if adding
3822 # for users is not configured.
3823 self.new_lang_error_message = gettext("Could not add new translation file.")
3824 if ( 3824 ↛ 3829line 3824 didn't jump to line 3829 because the condition on line 3824 was never true
3825 self.new_lang != "add"
3826 and user is not None
3827 and not user.has_perm("component.edit", self)
3828 ):
3829 self.new_lang_error_message = gettext(
3830 "You do not have permissions to add new translation file."
3831 )
3832 return False
3834 # Check if template can be parsed
3835 if self.has_template(): 3835 ↛ 3836line 3835 didn't jump to line 3836 because the condition on line 3835 was never true
3836 if not os.path.exists(self.get_template_filename()):
3837 self.new_lang_error_message = gettext(
3838 "The monolingual base language file is invalid."
3839 )
3840 return False
3841 if not fast:
3842 try:
3843 self.template_store.check_valid()
3844 except (FileParseError, ValueError):
3845 self.new_lang_error_message = gettext(
3846 "The monolingual base language file is invalid."
3847 )
3848 return False
3850 self.new_lang_error_message = gettext(
3851 "The template for new translations is invalid."
3852 )
3853 if self.new_base and not os.path.exists(self.get_new_base_filename()): 3853 ↛ 3854line 3853 didn't jump to line 3854 because the condition on line 3853 was never true
3854 return False
3855 return self.is_valid_base_for_new(fast=fast)
3857 def format_new_language_code(self, language):
3858 # Language code used for file
3859 code = self.file_format_cls.get_language_code(
3860 language.code, self.language_code_style
3861 )
3863 # Apply language aliases
3864 language_aliases = {v: k for k, v in self.project.language_aliases_dict.items()}
3865 if code in language_aliases: 3865 ↛ 3866line 3865 didn't jump to line 3866 because the condition on line 3865 was never true
3866 code = language_aliases[code]
3867 return code
3869 @transaction.atomic
3870 def add_new_language( # noqa: C901
3871 self,
3872 language: Language,
3873 request: AuthenticatedHttpRequest | None,
3874 send_signal: bool = True,
3875 create_translations: bool = True,
3876 show_messages: bool = True,
3877 ) -> Translation | None:
3878 """Create new language file."""
3879 if not self.can_add_new_language(request.user if request else None): 3879 ↛ 3880line 3879 didn't jump to line 3880 because the condition on line 3879 was never true
3880 if show_messages:
3881 messages.error(request, cast("str", self.new_lang_error_message))
3882 return None
3884 file_format = self.file_format_cls
3886 # Language code used for file
3887 code = self.format_new_language_code(language)
3889 # Check if resulting language is not present
3890 new_lang = Language.objects.fuzzy_get_strict(code=self.get_language_alias(code))
3891 if new_lang is not None: 3891 ↛ 3908line 3891 didn't jump to line 3908 because the condition on line 3891 was always true
3892 if new_lang == self.source_language: 3892 ↛ 3893line 3892 didn't jump to line 3893 because the condition on line 3892 was never true
3893 if show_messages:
3894 messages.error(
3895 request,
3896 gettext("The given language is used as a source language."),
3897 )
3898 return None
3900 if self.translation_set.filter(language=new_lang).exists(): 3900 ↛ 3901line 3900 didn't jump to line 3901 because the condition on line 3900 was never true
3901 if show_messages:
3902 messages.error(
3903 request, gettext("The given language already exists.")
3904 )
3905 return None
3907 # Check if language code is valid
3908 if re.match(self.language_regex, code) is None: 3908 ↛ 3909line 3908 didn't jump to line 3909 because the condition on line 3908 was never true
3909 if show_messages:
3910 messages.error(
3911 request,
3912 gettext("The given language is filtered by the language filter."),
3913 )
3914 return None
3916 base_filename = self.get_new_base_filename()
3918 filename = file_format.get_language_filename(self.filemask, code)
3919 fullname = os.path.join(self.full_path, filename)
3921 with self.repository.lock:
3922 if create_translations: 3922 ↛ 3923line 3922 didn't jump to line 3923 because the condition on line 3922 was never true
3923 self.commit_pending("add language", None)
3925 # Create or get translation object
3926 translation, created = self.translation_set.get_or_create(
3927 language=language,
3928 defaults={
3929 "plural": language.plural,
3930 "filename": filename,
3931 "language_code": code,
3932 },
3933 )
3934 # Make it clear that there is no change for the newly created translation
3935 # to avoid expensive last change lookup in stats while committing changes.
3936 if created: 3936 ↛ 3940line 3936 didn't jump to line 3940 because the condition on line 3936 was always true
3937 Change.store_last_change(translation, None)
3939 # Create the file
3940 if os.path.exists(fullname): 3940 ↛ 3944line 3940 didn't jump to line 3944 because the condition on line 3940 was never true
3941 # Ignore request if file exists (possibly race condition as
3942 # the processing of new language can take some time and user
3943 # can submit again)
3944 if show_messages:
3945 messages.error(request, gettext("Translation file already exists!"))
3946 else:
3947 file_format.add_language(
3948 fullname,
3949 language,
3950 base_filename,
3951 file_format_params=self.file_format_params,
3952 )
3953 if send_signal: 3953 ↛ 3957line 3953 didn't jump to line 3957 because the condition on line 3953 was always true
3954 translation_post_add.send(
3955 sender=self.__class__, translation=translation
3956 )
3957 translation.git_commit(
3958 request.user if request else None,
3959 request.user.get_author_name()
3960 if request
3961 else "Weblate <noreply@weblate.org>",
3962 template=self.add_message,
3963 store_hash=False,
3964 )
3966 # Trigger parsing of the newly added file
3967 if create_translations: 3967 ↛ 3970line 3967 didn't jump to line 3970 because the condition on line 3967 was never true
3968 # Forced scanning is needed in case adding file does not trigger commit,
3969 # for example when adding appstore metadata which only creates directory.
3970 self.create_translations(request=request, force_scan=True)
3971 if show_messages:
3972 messages.info(
3973 request,
3974 gettext("The translation will be updated in the background."),
3975 )
3977 # Delete no matches alert as we have just added the file
3978 self.delete_alert("NoMaskMatches")
3980 return translation
3982 def do_lock(self, user: User | None, lock: bool = True, auto: bool = False) -> None:
3983 """Lock or unlock component."""
3984 if self.locked == lock:
3985 return
3987 self.locked = lock
3988 # We avoid save here because it has unwanted side effects
3989 Component.objects.filter(pk=self.pk).update(locked=lock)
3990 change = self.get_lock_change(user=user, lock=lock, auto=auto)
3991 change.save()
3993 def get_lock_change(
3994 self, *, user: User | None, lock: bool = True, auto: bool = False
3995 ) -> Change:
3996 from weblate.trans.tasks import perform_commit
3998 change = Change(
3999 component=self,
4000 user=user,
4001 action=ActionEvents.LOCK if lock else ActionEvents.UNLOCK,
4002 details={"auto": auto},
4003 )
4004 if lock and not auto:
4005 perform_commit.delay_on_commit(
4006 self.pk, "lock", user_id=user.id if user else None
4007 )
4008 return change
4010 @cached_property
4011 def libre_license(self) -> bool:
4012 return is_libre(self.license)
4014 @cached_property
4015 def license_url(self) -> str:
4016 return get_license_url(self.license)
4018 def get_license_display(self) -> str: # type: ignore[no-redef]
4019 # Override Django implementation as that rebuilds the dict every time
4020 return get_license_name(self.license)
4022 def post_create(self, user: User, *, origin: str) -> None:
4023 self.change_set.create(
4024 action=ActionEvents.CREATE_COMPONENT,
4025 user=user,
4026 author=user,
4027 details={"origin": origin},
4028 )
4030 @property
4031 def context_label(self):
4032 if self.file_format in {"po", "po-mono", "tbx"}:
4033 # Translators: Translation context for Gettext
4034 return gettext("Context")
4035 # Translators: Translation key for monolingual translations
4036 return pgettext("Translation key", "Key")
4038 @cached_property
4039 def guidelines(self):
4040 from weblate.trans.guide import GUIDELINES
4042 return [guide(self) for guide in GUIDELINES]
4044 @cached_property
4045 def addons_cache(self):
4046 from weblate.addons.models import Addon
4048 result = defaultdict(list)
4049 result["__lookup__"] = {}
4050 for addon in Addon.objects.filter_for_execution(self): 4050 ↛ 4051line 4050 didn't jump to line 4051 because the loop on line 4050 never started
4051 for installed in addon.event_set.all():
4052 result[installed.event].append(addon)
4053 result["__all__"].append(addon)
4054 result["__names__"].append(addon.name)
4055 result["__lookup__"][addon.name] = addon
4056 return result
4058 def get_addon(self, name: str) -> Addon | None:
4059 return self.addons_cache["__lookup__"].get(name)
4061 def schedule_sync_terminology(self) -> None:
4062 """Trigger terminology sync in the background."""
4063 from weblate.glossary.tasks import sync_glossary_languages, sync_terminology
4065 if settings.CELERY_TASK_ALWAYS_EAGER: 4065 ↛ 4067line 4065 didn't jump to line 4067 because the condition on line 4065 was never true
4066 # Execute directly to avoid locking issues
4067 if self.is_glossary:
4068 sync_terminology(self.pk, component=self)
4069 else:
4070 for glossary in self.project.glossaries:
4071 sync_glossary_languages(glossary.pk, component=glossary)
4072 elif not self._glossary_sync_scheduled: 4072 ↛ exitline 4072 didn't return from function 'schedule_sync_terminology' because the condition on line 4072 was always true
4073 self._glossary_sync_scheduled = True
4074 transaction.on_commit(self._schedule_sync_terminology)
4076 def _schedule_sync_terminology(self) -> None:
4077 from weblate.glossary.tasks import sync_glossary_languages, sync_terminology
4079 if self.is_glossary:
4080 sync_terminology.delay_on_commit(self.pk)
4081 else:
4082 for glossary in self.project.glossaries:
4083 sync_glossary_languages.delay_on_commit(glossary.pk)
4084 self._glossary_sync_scheduled = False
4086 def get_unused_enforcements(self) -> Iterable[dict | BaseCheck]:
4087 from weblate.trans.models import Unit
4089 for current in self.enforced_checks: 4089 ↛ 4090line 4089 didn't jump to line 4090 because the loop on line 4089 never started
4090 try:
4091 check = CHECKS[current]
4092 except KeyError:
4093 yield {"name": current, "notsupported": True}
4094 continue
4095 # Check is always enabled
4096 if not check.default_disabled:
4097 continue
4098 flag = check.enable_string
4099 # Enabled on component level
4100 if flag in self.all_flags:
4101 continue
4102 # Enabled on translation level
4103 if self.translation_set.filter(check_flags__contains=flag).exists():
4104 continue
4105 # Enabled on unit level
4106 if Unit.objects.filter(
4107 Q(flags__contains=flag) | Q(extra_flags__contains=flag)
4108 ).exists():
4109 continue
4110 yield check
4112 def get_language_alias(self, code):
4113 if code in self.project.language_aliases_dict: 4113 ↛ 4114line 4113 didn't jump to line 4114 because the condition on line 4113 was never true
4114 return self.project.language_aliases_dict[code]
4115 if code in {"source", "src", "default"}: 4115 ↛ 4116line 4115 didn't jump to line 4116 because the condition on line 4115 was never true
4116 return self.source_language.code
4117 return code
4119 @property
4120 def get_add_label(self):
4121 if self.is_glossary:
4122 return gettext("Add term to glossary")
4123 return gettext("Add new translation string")
4125 def suggest_repo_link(self):
4126 if self.is_repo_link or self.is_repo_local: 4126 ↛ 4129line 4126 didn't jump to line 4129 because the condition on line 4126 was always true
4127 return None
4129 same_repo = self.project.component_set.filter(
4130 repo=self.repo, vcs=self.vcs, branch=self.branch
4131 )
4132 if self.push:
4133 same_repo = same_repo.filter(push=self.push)
4134 try:
4135 return same_repo[0].get_repo_link_url()
4136 except IndexError:
4137 return None
4139 @cached_property
4140 def enable_review(self):
4141 return self.project.enable_review
4143 @property
4144 def update_checks_key(self) -> str:
4145 return f"component-update-checks-{self.pk}"
4147 def schedule_update_checks(self, update_state: bool = False) -> None:
4148 from weblate.trans.tasks import update_checks
4150 update_token = get_random_identifier()
4151 cache.set(self.update_checks_key, update_token)
4152 update_checks.delay_on_commit(self.pk, update_token, update_state=update_state)
4154 @property
4155 def all_repo_components(self) -> list[Component]:
4156 if self.linked_component:
4157 return [self.linked_component]
4158 return [self]
4160 def start_sentry_span(self, op: str):
4161 return sentry_sdk.start_span(op=op, name=self.full_slug)
4163 @cached_property
4164 def key_filter_re(self) -> re.Pattern:
4165 """Provide the cached version of key_filter."""
4166 return re.compile(self.key_filter)
4168 def repository_status(self) -> str:
4169 try:
4170 return self.repository.status()
4171 except RepositoryError as error:
4172 return "{}\n\n{}".format(gettext("Could not get repository status!"), error)
4174 def update_enforced_checks(self) -> None:
4175 from weblate.trans.models import Unit
4177 units = Unit.objects.filter(
4178 check__name__in=self.enforced_checks,
4179 translation__component=self,
4180 state__in=(STATE_TRANSLATED, STATE_APPROVED),
4181 )
4183 for unit in units.select_for_update():
4184 unit.translate(
4185 None,
4186 unit.get_target_plurals(),
4187 STATE_FUZZY,
4188 change_action=ActionEvents.ENFORCED_CHECK,
4189 propagate=False,
4190 )
4192 @cached_property
4193 def api_slug(self):
4194 return "%252F".join(self.get_url_path()[1:])
4196 def get_all_available_languages(self) -> models.QuerySet[Language]:
4197 return Language.objects.exclude(
4198 Q(translation__component=self) | Q(component=self)
4199 )
4202@receiver(m2m_changed, sender=Component.links.through)
4203@disable_for_loaddata
4204def change_component_link(sender, instance, action, pk_set, **kwargs) -> None:
4205 from weblate.trans.models import Project
4207 if action not in {"post_add", "post_remove", "post_clear"}:
4208 return
4209 for project in Project.objects.filter(pk__in=pk_set):
4210 project.invalidate_glossary_cache()