Coverage for app/venv/lib/python3.14/site-packages/weblate/checks/fluent/references.py: 14%
275 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-10-07 07:15 +0000
1# Copyright © Henry Wilkes <henry@torproject.org>
2#
3# SPDX-License-Identifier: GPL-3.0-or-later
5from __future__ import annotations
7from typing import TYPE_CHECKING
9from django.utils.translation import gettext, gettext_lazy
11from weblate.checks.base import TargetCheck
12from weblate.checks.fluent.utils import (
13 FluentPatterns,
14 FluentUnitConverter,
15 format_html_code,
16 format_html_error_list,
17 translation_from_check,
18 variant_name,
19)
20from weblate.utils.html import format_html_join_comma, list_to_tuples
22if TYPE_CHECKING: 22 ↛ 23line 22 didn't jump to line 23 because the condition on line 22 was never true
23 from collections.abc import Iterable, Iterator
25 from django.utils.safestring import SafeString
26 from django_stubs_ext import StrOrPromise
27 from translate.storage.fluent import (
28 FluentPart,
29 FluentReference,
30 FluentSelectorBranch,
31 FluentSelectorNode,
32 )
34 from weblate.checks.fluent.utils import CheckModel, HighlightsType, TransUnitModel
37class _Reference:
38 """A class that wraps a FluentReference."""
40 def __init__(self, fluent_ref: FluentReference) -> None:
41 self.fluent_ref = fluent_ref
42 # A reference can be flagged as shared if it was borrowed from another
43 # branch.
44 self.shared = False
46 def share(self) -> _Reference:
47 copy = self.__class__(self.fluent_ref)
48 copy.shared = True
49 return copy
51 def present(self) -> str:
52 """Show the reference in a presentable form."""
53 ref_name = self.fluent_ref.name
54 if self.fluent_ref.type_name == "term":
55 ref_name = f"-{ref_name}"
56 elif self.fluent_ref.type_name == "variable":
57 ref_name = f"${ref_name}"
58 return f"{{\xa0{ref_name}\xa0}}"
60 def matches(self, other: _Reference) -> bool:
61 """Whether the two references match in name and type."""
62 return (
63 self.fluent_ref.type_name == other.fluent_ref.type_name
64 and self.fluent_ref.name == other.fluent_ref.name
65 )
68class _CountedReferences:
69 """
70 Tracks the number of matching _References in a list.
72 This will collect matching references together so they can be counted and
73 compared, without caring about the order.
74 """
76 def __init__(self, ref_list: Iterable[_Reference]) -> None:
77 self.matching_refs: list[list[_Reference]] = []
78 for ref in ref_list:
79 add = True
80 for other_refs in self.matching_refs:
81 if other_refs[0].matches(ref):
82 add = False
83 other_refs.append(ref)
84 break
85 if add:
86 self.matching_refs.append([ref])
88 def matches(self, other: _CountedReferences) -> bool:
89 """
90 Whether the two instances match.
92 This will match if the two instances have matching _References with the
93 same count in both.
94 """
95 if len(self.matching_refs) != len(other.matching_refs):
96 return False
97 for refs in self.matching_refs:
98 has_match = False
99 for _index, other_refs in enumerate(other.matching_refs):
100 if refs[0].matches(other_refs[0]):
101 if len(refs) != len(other_refs):
102 return False
103 has_match = True
104 break
105 if not has_match:
106 return False
107 # Both are the same length and each ref in self was matched, so there
108 # shouldn't be any refs in other that were not matched once.
109 return True
111 def count(self, ref: _Reference, include_shared: bool = True) -> int:
112 """
113 How many references match the given reference.
115 If `include_shared` is False, this will only count references that are
116 not flagged as "shared".
117 """
118 for other_refs in self.matching_refs:
119 if ref.matches(other_refs[0]):
120 if include_shared:
121 return len(other_refs)
122 count = 0
123 for equiv_ref in other_refs:
124 if not equiv_ref.shared:
125 count += 1
126 return count
127 return 0
130class _VariantReferences:
131 """Represents a variant string and the references it contains."""
133 def __init__(
134 self,
135 path: list[FluentSelectorBranch],
136 references: list[_Reference],
137 ) -> None:
138 self.path = path
139 self.references = references
140 self.counted_references = _CountedReferences(references)
142 def name(self) -> str:
143 """Generate the name for this variant."""
144 return variant_name(self.path)
147class _DifferentBranchCountError(Exception):
148 # Generic exception for when two branches under the same node have different
149 # counts for some reference.
150 pass
153class _SelectorReferences:
154 """Class for extracting the references found underneath some selector branch."""
156 def __init__(self, top_branch: FluentSelectorBranch) -> None:
157 # Map from the selector branches to a list of their direct references.
158 self._branch_references: dict[FluentSelectorBranch, list[_Reference]] = {}
159 # Map from the selector node to a list of references found in their
160 # SelectExpression selector expression.
161 self._selector_references: dict[FluentSelectorNode, list[_Reference]] = {}
163 # Populate the maps with references.
164 self._set_refs(top_branch)
166 # Try and share references between branches.
167 self._share_refs_between_branches(top_branch)
169 # Generate all possible variants and assign them the references that
170 # would appear in their flat form, plus any shared references.
171 self.variant_references = [
172 _VariantReferences(path, list(self._refs_for_path(top_branch, path)))
173 for path in top_branch.branch_paths()
174 ]
176 def _refs_for_path(
177 self,
178 top_branch: FluentSelectorBranch,
179 branches: list[FluentSelectorBranch],
180 ) -> Iterator[_Reference]:
181 """
182 Fetch all the references for a given branch path.
184 Each branch path represents a possible variant of the original fluent
185 entry.
186 """
187 yield from self._branch_references[top_branch]
188 for branch in branches:
189 yield from self._branch_references[branch]
191 def _set_refs(self, branch: FluentSelectorBranch) -> None:
192 self._branch_references[branch] = [
193 _Reference(fluent_ref) for fluent_ref in branch.top_references
194 ]
195 for node in branch.child_nodes:
196 # Only want unique refs for the selector references, otherwise this
197 # can lead to double-sharing of these references.
198 selector_refs: list[_Reference] = []
199 for fluent_ref in node.selector_references:
200 ref = _Reference(fluent_ref)
201 add = True
202 for other in selector_refs:
203 if other.matches(ref):
204 add = False
205 break
206 if add:
207 selector_refs.append(ref)
208 self._selector_references[node] = selector_refs
210 for child in node.child_branches:
211 self._set_refs(child)
213 def _branch_ref_count(
214 self,
215 branch: FluentSelectorBranch,
216 ref: _Reference,
217 ) -> int:
218 """
219 Get the number of times a reference appears in a branch.
221 If, for each node in the branch, each child branch of the node has the
222 same reference count, then this will be added to the returned count.
223 Otherwise, if they differ in count, this will raise the
224 _DifferentBranchCountError exception to indicate that there is no
225 consistent number to return.
226 """
227 count = 0
228 for node in branch.child_nodes:
229 same_count = None
230 for child in node.child_branches:
231 child_count = self._branch_ref_count(child, ref)
232 if same_count is None:
233 # same_count is uninitialized so set it using this first
234 # branch.
235 same_count = child_count
236 elif same_count != child_count:
237 # Two branches differ, so no consistent counting.
238 raise _DifferentBranchCountError
239 if same_count is None:
240 # Unexpected since each selector node should have at least one
241 # child.
242 raise _DifferentBranchCountError
243 count += same_count
244 # Add the count for the top references.
245 # NOTE: this includes shared references that have been added earlier.
246 for other in self._branch_references[branch]:
247 if other.matches(ref):
248 count += 1
249 return count
251 def _share_refs_between_branches(
252 self,
253 branch: FluentSelectorBranch,
254 ) -> None:
255 """
256 Try and share references between branches.
258 Each node below the given branch will represent a SelectExpression. If
259 the SelectExpression's selector contains a reference that matches one of
260 the references within one of its Variants, we want to share that
261 reference between all the Variants *as if* each variant contains the
262 reference.
264 This should help adjust for the fact that a Variant's key may make the
265 reference unnecessary. E.g. we might select over the number $num, but if
266 we match with the [zero] or [one] category, then we might not need to or
267 want to reference the { $num } value. Moreover, this could vary across
268 different locales.
269 """
270 # Share depth-first to try and equalise the reference count between
271 # sub-branches before moving up.
272 for node in branch.child_nodes:
273 for child in node.child_branches:
274 self._share_refs_between_branches(child)
276 for node in branch.child_nodes:
277 try:
278 # For each reference found in the selector, we want to count how
279 # many times it appears in each child branch.
280 ref_counts = {
281 ref: {
282 child: self._branch_ref_count(child, ref)
283 for child in node.child_branches
284 }
285 for ref in self._selector_references[node]
286 }
287 except _DifferentBranchCountError:
288 # For at least one of the references, one of the children does
289 # not have a consistent count.
290 # E.g.
291 # | { $num ->
292 # | [one] one
293 # | *[other] { $var ->
294 # | [a] { $num }
295 # | *[b] none
296 # | }
297 # | }
298 # Here the [other] branch does not have a consistent count for
299 # $num between its child branches [a] and [b], so we do not
300 # share it.
301 continue
302 for ref, child_counts in ref_counts.items():
303 # We want each child to have the same reference count for the
304 # purpose of comparison, so we find the maximum count and pad
305 # the other branches with shared references.
306 # NOTE: Each ref in selector_references should be unique to
307 # avoid double adding at this stage.
308 max_count = max(child_counts.values())
309 for child, count in child_counts.items():
310 self._branch_references[child].extend(
311 ref.share() for _num in range(max_count - count)
312 )
315class _VariantReferencesDifference:
316 """
317 The difference between the references found in the source and target.
319 Each variant in the source will be compared against each variant in the
320 target to see if they have a matching set of references with the same number
321 or appearances, but not necessarily in the same order.
323 If there is any source variant that does not have at least one match in the
324 target, it will be flagged as a missing variant. Similarly, if there is any
325 target variant with no matching source variant, it will be flagged as an
326 extra variant.
327 """
329 def __init__(
330 self,
331 source_part: FluentPart,
332 target_part: FluentPart,
333 ) -> None:
334 self._part_name = source_part.name
335 self._source_variants = _SelectorReferences(
336 source_part.top_branch
337 ).variant_references
338 self._target_variants = _SelectorReferences(
339 target_part.top_branch
340 ).variant_references
342 self._missing_variants = [
343 variant
344 for variant in self._source_variants
345 if not self._has_match(variant, self._target_variants)
346 ]
347 self._extra_variants = [
348 variant
349 for variant in self._target_variants
350 if not self._has_match(variant, self._source_variants)
351 ]
353 @classmethod
354 def _has_match(
355 cls, variant: _VariantReferences, search_list: list[_VariantReferences]
356 ) -> bool:
357 return any(
358 variant.counted_references.matches(other.counted_references)
359 for other in search_list
360 )
362 def __bool__(self) -> bool:
363 return bool(self._missing_variants or self._extra_variants)
365 def _missing_ref_message(
366 self,
367 ref: str,
368 variants: str,
369 ) -> SafeString:
370 if self._part_name:
371 if not variants:
372 return format_html_code(
373 gettext(
374 "Fluent {attribute} attribute is missing a "
375 "{reference} Fluent reference."
376 ),
377 attribute=self._part_name,
378 reference=ref,
379 )
380 return format_html_code(
381 gettext(
382 "Fluent {attribute} attribute is missing a {reference} "
383 "Fluent reference for the following variants: {variant_list}."
384 ),
385 attribute=self._part_name,
386 reference=ref,
387 variant_list=variants,
388 )
389 if not variants:
390 return format_html_code(
391 gettext("Fluent value is missing a {reference} Fluent reference."),
392 reference=ref,
393 )
394 return format_html_code(
395 gettext(
396 "Fluent value is missing a {reference} Fluent reference "
397 "for the following variants: {variant_list}."
398 ),
399 reference=ref,
400 variant_list=variants,
401 )
403 def _extra_ref_message(
404 self,
405 ref: str,
406 variants: str,
407 ) -> SafeString:
408 if self._part_name:
409 if not variants:
410 return format_html_code(
411 gettext(
412 "Fluent {attribute} attribute has an unexpected extra "
413 "{reference} Fluent reference."
414 ),
415 attribute=self._part_name,
416 reference=ref,
417 )
418 return format_html_code(
419 gettext(
420 "Fluent {attribute} attribute has an unexpected extra {reference} "
421 "Fluent reference for the following variants: {variant_list}."
422 ),
423 attribute=self._part_name,
424 reference=ref,
425 variant_list=variants,
426 )
427 if not variants:
428 return format_html_code(
429 gettext(
430 "Fluent value has an unexpected extra {reference} Fluent reference."
431 ),
432 reference=ref,
433 )
434 return format_html_code(
435 gettext(
436 "Fluent value has an unexpected extra {reference} Fluent "
437 "reference for the following variants: {variant_list}."
438 ),
439 reference=ref,
440 variant_list=variants,
441 )
443 @staticmethod
444 def _present_variant_list(
445 variant_list: list[_VariantReferences] | None,
446 ) -> str:
447 if not variant_list:
448 return ""
449 return format_html_join_comma(
450 "{}", list_to_tuples(variant.name() for variant in variant_list)
451 )
453 def _unique_target_refs(self) -> Iterator[_Reference]:
454 unique_refs: list[_Reference] = []
455 for variant in self._target_variants:
456 for ref in variant.references:
457 add = True
458 for other in unique_refs:
459 if other.matches(ref):
460 add = False
461 break
462 if add:
463 unique_refs.append(ref)
464 yield ref
466 def _errors_relative_to(
467 self,
468 source_counted_refs: _CountedReferences,
469 ) -> Iterator[SafeString]:
470 # NOTE: The source_counted_refs may contain shared references, but we
471 # ignore this property since at least one of the source variants
472 # contains the actual reference, and we expect it to appear in the
473 # targets as well.
474 for refs in source_counted_refs.matching_refs:
475 count = len(refs)
476 variants_missing_ref = []
477 all_variants = True
478 for variant in self._target_variants:
479 if variant.counted_references.count(refs[0]) < count:
480 variants_missing_ref.append(variant)
481 else:
482 all_variants = False
483 if not variants_missing_ref:
484 continue
485 yield self._missing_ref_message(
486 refs[0].present(),
487 self._present_variant_list(
488 None if all_variants else variants_missing_ref
489 ),
490 )
492 for ref in self._unique_target_refs():
493 count = source_counted_refs.count(ref)
494 variants_extra_ref = []
495 all_variants = True
496 for variant in self._target_variants:
497 # Here we are looking for extra references that shouldn't
498 # appear, we only want to highlight the ones that are not shared
499 # to avoid mentioning extra references for variants that do not
500 # contain them explicitly.
501 # NOTE: If there is a shared reference, then we expect at least
502 # one of the target variants will have the excessive count, so
503 # will be reported.
504 if variant.counted_references.count(ref, False) > count:
505 variants_extra_ref.append(variant)
506 else:
507 all_variants = False
508 if not variants_extra_ref:
509 continue
510 yield self._extra_ref_message(
511 ref.present(),
512 self._present_variant_list(
513 None if all_variants else variants_extra_ref
514 ),
515 )
517 def _missing_variants_message(
518 self,
519 variants: list[_VariantReferences],
520 ) -> SafeString:
521 # NOTE: variants should all have names since the source contains at
522 # least two variants in order to reach this step.
523 variant_list = self._present_variant_list(variants)
524 if self._part_name:
525 return format_html_code(
526 gettext(
527 "The following variants in the original Fluent {attribute} "
528 "attribute do not have at least one matching variant in the "
529 "translation with the same set of Fluent references: "
530 "{variant_list}."
531 ),
532 attribute=self._part_name,
533 variant_list=variant_list,
534 )
535 return format_html_code(
536 gettext(
537 "The following variants in the original Fluent value do not "
538 "have at least one matching variant in the translation with "
539 "the same set of Fluent references: {variant_list}."
540 ),
541 variant_list=variant_list,
542 )
544 def _extra_variants_message(
545 self,
546 variants: list[_VariantReferences] | None,
547 ) -> SafeString:
548 variant_list = self._present_variant_list(variants)
549 if self._part_name:
550 if not variant_list:
551 return format_html_code(
552 gettext(
553 "The translated Fluent {attribute} attribute does not "
554 "have a matching variant in the original with the same "
555 "set of Fluent references."
556 ),
557 attribute=self._part_name,
558 )
559 return format_html_code(
560 gettext(
561 "The following variants in the translated Fluent "
562 "{attribute} attribute do not have a matching variant in "
563 "the original with the same set of Fluent references: "
564 "{variant_list}."
565 ),
566 attribute=self._part_name,
567 variant_list=variant_list,
568 )
569 if not variant_list:
570 return format_html_code(
571 gettext(
572 "The translated Fluent value does not "
573 "have a matching variant in the original with the same "
574 "set of Fluent references."
575 ),
576 )
577 return format_html_code(
578 gettext(
579 "The following variants in the translated Fluent "
580 "value do not have a matching variant in "
581 "the original with the same set of Fluent references: "
582 "{variant_list}."
583 ),
584 variant_list=variant_list,
585 )
587 def _errors_for_unmatched_variants(
588 self,
589 ) -> Iterator[SafeString]:
590 if self._missing_variants:
591 yield self._missing_variants_message(self._missing_variants)
592 if self._extra_variants:
593 # Don't want to print a list of variants if we only have one in the
594 # original.
595 have_target_variants = len(self._target_variants) > 1
596 yield self._extra_variants_message(
597 self._extra_variants if have_target_variants else None
598 )
600 def description(self) -> SafeString:
601 # We want to be able to compare each target variant against some common
602 # set of expected references. This allows us to determine which specific
603 # references are missing or extra.
604 # This is only possible if each variant in the source has the same set
605 # of references (after sharing) to allow for this one-to-one comparison.
606 # But we expect this will happen in most cases.
607 common_refs: _CountedReferences | None = None
608 for variant in self._source_variants:
609 if common_refs is None:
610 common_refs = variant.counted_references
611 elif not common_refs.matches(variant.counted_references):
612 common_refs = None
613 break
615 if common_refs is not None:
616 return format_html_error_list(self._errors_relative_to(common_refs))
617 # The source contains multiple variants with different references.
618 return format_html_error_list(self._errors_for_unmatched_variants())
621class FluentReferencesCheck(TargetCheck):
622 r"""
623 Check that the target uses the same Fluent references as the source.
625 A Fluent Message or Term can reference another Message, Term, Attribute, or
626 a variable. For example:
628 | Here is a { message }, a { message.attribute } a { -term } and a { $variable }.
629 | Within a function { NUMBER($num, minimumFractionDigits: 2) }
631 Generally, translated Messages or Terms are expected to contain the same
632 references as the source, although not necessarily in the same order of
633 appearance. So this check ensures that translations use the same references
634 in their value as the source value, the same number of times, and with no
635 additions. For Messages, this will also check that each Attribute in the
636 translation uses the same references as the matching Attribute in the
637 source.
639 When the source or translation contains Fluent Select Expressions, then each
640 possible variant in the source must be matched with at least one variant in
641 the translation with the same references, and vice versa.
643 Moreover, if a variable reference appears both in the Select Expression's
644 selector and within one of its variants, then all variants may also be
645 considered as if they also contain that reference. The assumption being that
646 the variant's key may have made the reference redundant for that variant.
647 For example:
649 | { $num ->
650 | [one] an apple
651 | *[other] { $num } apples
652 | }
654 Here, for the purposes of this check, the ``[one]`` variant will also be
655 considered to contain the ``$num`` reference.
657 However, a reference within the Select Expression's selector, which can only
658 be a variable of a Term Attribute in Fluent's syntax, will not by itself
659 count as a required reference because they do not form the actual text
660 content of the string that the end-user will see, and the presence of a
661 Select Expression is considered locale-specific. For example:
663 | { -term.starts-with-vowel ->
664 | [yes] an { -term }
665 | *[no] a { -term }
666 | }
668 Here a reference to ``-term.starts-with-vowel`` is not expected to appear in
669 translations, but a reference to ``-term`` is.
670 """
672 check_id = "fluent-references"
673 name = gettext_lazy("Fluent references")
674 description = gettext_lazy("Fluent references should match.")
675 default_disabled = True
677 @classmethod
678 def _compare_references(
679 cls, unit: TransUnitModel, source: str, target: str
680 ) -> list[_VariantReferencesDifference]:
681 # NOTE: If the source or target contains a syntax error, then
682 # to_fluent_parts will return None. In this case, we just make it an
683 # empty list. Then the returned difference will naturally be empty.
684 source_unit = FluentUnitConverter(unit, source)
685 is_message = source_unit.fluent_type() == "Message"
686 source_parts = {
687 part.name: part
688 for part in (FluentUnitConverter(unit, source).to_fluent_parts() or [])
689 # Always compare references in the values, but only compare
690 # references in attributes for Messages (rather than Terms).
691 if is_message or not part.name
692 }
693 differences: list[_VariantReferencesDifference] = []
694 for part in FluentUnitConverter(unit, target).to_fluent_parts() or []:
695 if part.name not in source_parts:
696 # Ignore this part since we don't have anything to compare it
697 # against. The FluentParts checker will capture this.
698 continue
699 diff = _VariantReferencesDifference(source_parts[part.name], part)
700 if not diff:
701 continue
702 differences.append(diff)
703 return differences
705 def check_single(
706 self,
707 source: str,
708 target: str,
709 unit: TransUnitModel,
710 ) -> bool:
711 return bool(self._compare_references(unit, source, target))
713 @classmethod
714 def _get_all_references_in_branch(
715 cls,
716 branch: FluentSelectorBranch,
717 ) -> Iterator[FluentReference]:
718 """Get all references found across all variants as a single iterator."""
719 yield from branch.top_references
720 for node in branch.child_nodes:
721 for child in node.child_branches:
722 yield from cls._get_all_references_in_branch(child)
724 def check_highlight(
725 self,
726 source: str,
727 unit: TransUnitModel,
728 ) -> HighlightsType:
729 if self.should_skip(unit): 729 ↛ 732line 729 didn't jump to line 732 because the condition on line 729 was always true
730 return []
732 fluent_unit = FluentUnitConverter(unit, source)
733 is_message = fluent_unit.fluent_type() == "Message"
735 highlight_patterns: list[str] = []
736 # We simply match all references found in the source.
737 for part in fluent_unit.to_fluent_parts() or []:
738 if part.name and not is_message:
739 continue
740 highlight_patterns.extend(
741 FluentPatterns.reference(fluent_ref)
742 for fluent_ref in self._get_all_references_in_branch(part.top_branch)
743 )
744 return FluentPatterns.highlight_source(source, highlight_patterns)
746 def get_description(self, check_model: CheckModel) -> StrOrPromise:
747 (unit, source, target) = translation_from_check(check_model)
748 differences = self._compare_references(unit, source, target)
749 if not differences:
750 return super().get_description(check_model)
752 return format_html_error_list(diff.description() for diff in differences)