Coverage for app/venv/lib/python3.14/site-packages/weblate/lang/models.py: 39%
681 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-10-07 07:15 +0000
1# Copyright © Michal Čihař <michal@weblate.org>
2#
3# SPDX-License-Identifier: GPL-3.0-or-later
5from __future__ import annotations
7import re
8from collections import defaultdict
9from gettext import c2py
10from itertools import chain
11from typing import TYPE_CHECKING, Self
12from weakref import WeakValueDictionary
14from appconf import AppConf
15from django.conf import settings
16from django.contrib.admin.utils import NestedObjects
17from django.core.exceptions import ObjectDoesNotExist
18from django.db import models, transaction
19from django.db.models import Exists, OuterRef, Q
20from django.db.utils import OperationalError
21from django.urls import reverse
22from django.utils.functional import cached_property
23from django.utils.html import format_html
24from django.utils.translation import gettext, gettext_lazy, pgettext_lazy
25from django.utils.translation.trans_real import parse_accept_lang_header
26from weblate_language_data.aliases import ALIASES
27from weblate_language_data.case_insensitive import CASE_INSENSITIVE_LANGS
28from weblate_language_data.countries import DEFAULT_LANGS
29from weblate_language_data.plurals import CLDRPLURALS, EXTRAPLURALS, QTPLURALS
30from weblate_language_data.rtl import RTL_LANGS
32from weblate.checks.format import BaseFormatCheck
33from weblate.checks.models import CHECKS
34from weblate.lang import data
35from weblate.lang.data import BASIC_LANGUAGES, FORMULA_WITH_ZERO
36from weblate.logger import LOGGER
37from weblate.trans.defines import LANGUAGE_CODE_LENGTH, LANGUAGE_NAME_LENGTH
38from weblate.trans.mixins import CacheKeyMixin
39from weblate.trans.util import sort_objects, sort_unicode
40from weblate.utils.html import format_html_join_comma, list_to_tuples
41from weblate.utils.state import STATE_TRANSLATED
42from weblate.utils.validators import validate_plural_formula
44if TYPE_CHECKING: 44 ↛ 45line 44 didn't jump to line 45 because the condition on line 44 was never true
45 from collections.abc import Callable
47 from django_stubs_ext import StrOrPromise
49 from weblate.auth.models import AuthenticatedHttpRequest, User
50 from weblate.trans.models import Project, Unit
51 from weblate.trans.models.unit import UnitQuerySet
53PLURAL_RE = re.compile(
54 r"\s*nplurals\s*=\s*([0-9]+)\s*;\s*plural\s*=\s*([()n0-9!=|&<>+*/%\s?:-]+)"
55)
56PLURAL_TITLE = """
57{name} <span class="text-muted" title="{title}">({examples})</span>
58"""
59COPY_RE = re.compile(r"\([0-9]+\)")
60KNOWN_SUFFIXES = {"hant", "hans", "latn", "cyrl", "shaw"}
61GENERATED_SUFFIX = "(generated)"
64def get_plural_type(base_code, plural_formula):
65 """Get correct plural type for language."""
66 # Remove not needed parenthesis
67 if plural_formula[-1] == ";": 67 ↛ 68line 67 didn't jump to line 68 because the condition on line 67 was never true
68 plural_formula = plural_formula[:-1]
70 # No plural
71 if plural_formula == "0":
72 return data.PLURAL_NONE
74 # Remove whitespace
75 formula = plural_formula.replace(" ", "")
77 # Standard plural formulas
78 for mapping in data.PLURAL_MAPPINGS: 78 ↛ 83line 78 didn't jump to line 83 because the loop on line 78 didn't complete
79 if formula in mapping[0]:
80 return mapping[1]
82 # Arabic special case
83 if base_code == "ar":
84 return data.PLURAL_ARABIC
86 # Log error in case of unknown mapping
87 LOGGER.error("Can not guess type of plural for %s: %s", base_code, plural_formula)
89 # Try to calculate based on formula
90 for formulas, plural in data.PLURAL_MAPPINGS:
91 for data_formula in formulas:
92 if is_same_plural(-1, plural_formula, -1, data_formula):
93 return plural
95 return data.PLURAL_UNKNOWN
98def is_same_plural(
99 our_number: int,
100 our_formula: str,
101 number: int,
102 formula: str,
103 our_function: Callable | None = None,
104 plural_function: Callable | None = None,
105):
106 if our_function is None: 106 ↛ 107line 106 didn't jump to line 107 because the condition on line 106 was never true
107 try:
108 our_function = c2py(our_formula)
109 except ValueError:
110 return False
112 if plural_function is None: 112 ↛ 118line 112 didn't jump to line 118 because the condition on line 112 was always true
113 try:
114 plural_function = c2py(formula)
115 except ValueError:
116 return False
118 if number not in {-1, our_number}:
119 return False
120 if formula == our_formula:
121 return True
122 # Compare formula results
123 # It would be better to compare formulas,
124 # but this was easier to implement and the performance
125 # is still okay.
126 return all(
127 our_function(i) == plural_function(i)
128 for i in chain(range(-10, 200), [1000, 10000, 100000, 1000000, 10000000])
129 )
132def get_default_lang():
133 """Return object ID for English language."""
134 try:
135 return Language.objects.default_language.id
136 except (Language.DoesNotExist, OperationalError):
137 return -1
140class LanguageQuerySet(models.QuerySet):
141 def try_get(self, *args, **kwargs):
142 """Try to get language by code."""
143 result = self.filter(*args, **kwargs)[:2]
144 if len(result) != 1: 144 ↛ 145line 144 didn't jump to line 145 because the condition on line 144 was never true
145 return None
146 return result[0]
148 @staticmethod
149 def parse_lang_country(code):
150 """Parse language and country from locale code."""
151 # Parse private use subtag
152 subtag_pos = code.find("-x-")
153 if subtag_pos != -1:
154 subtag = code[subtag_pos:]
155 code = code[:subtag_pos]
156 else:
157 subtag = ""
158 # Parse the string
159 if "-" in code:
160 lang, country = code.split("-", 1)
161 # Android regional locales
162 if len(country) > 2 and country[0] == "r":
163 country = country[1:]
164 elif "_" in code:
165 lang, country = code.split("_", 1)
166 elif "+" in code:
167 lang, country = code.split("+", 1)
168 else:
169 lang = code
170 country = None
172 return lang, country, subtag
174 @staticmethod
175 def sanitize_code(code):
176 """Language code sanitization."""
177 # Strip b+ prefix from Android
178 code = code.removeprefix("b+")
180 # Replace -r from Android by _
181 if len(code) == 6 and "-r" in code: 181 ↛ 182line 181 didn't jump to line 182 because the condition on line 181 was never true
182 code = code.replace("-r", "_")
184 # Handle duplicate language files for example "cs (2)"
185 code = COPY_RE.sub("", code)
187 # Remove some unwanted characters
188 code = code.replace(" ", "").replace("(", "").replace(")", "")
190 # Strip leading and trailing .
191 return code.strip(".")
193 def aliases_get(
194 self, code: str, expanded_code: str | None = None
195 ) -> Language | None:
196 code = code.lower()
197 # Normalize script suffix
198 code = code.replace("_latin", "@latin").replace("_cyrillic", "@cyrillic")
199 codes = [code]
200 codes.extend(
201 code.replace(replacement, "_")
202 for replacement in ("+", "-", "-r", "_r")
203 if replacement in code
204 )
205 if expanded_code and expanded_code != code:
206 codes.append(expanded_code)
208 # Lookup in aliases
209 for newcode in codes:
210 if newcode in ALIASES:
211 testcode = ALIASES[newcode]
212 ret = self.try_get(code=testcode)
213 if ret is not None:
214 return ret
216 # Alias language code only
217 for newcode in codes:
218 language, _sep, country = newcode.partition("_")
219 if (
220 country
221 and len(language) > 2
222 and language in ALIASES
223 and "_" not in ALIASES[language]
224 ):
225 testcode = f"{ALIASES[language]}_{country}"
226 ret = self.fuzzy_get_strict(code=testcode)
227 if ret is not None:
228 return ret
230 return None
232 def fuzzy_get_strict(self, code: str) -> Language | None:
233 result = self.fuzzy_get(code)
234 if isinstance(result, str): 234 ↛ 235line 234 didn't jump to line 235 because the condition on line 234 was never true
235 return None
236 return result
238 def fuzzy_get(self, code: str) -> Language | str:
239 """
240 Get matching language for code.
242 The code does not have to be exactly same (cs_CZ is trteated same as
243 cs-CZ) or returns None.
245 It also handles Android special naming of regional locales like pt-rBR.
246 """
247 code = self.sanitize_code(code)
248 expanded_code = None
250 lookups = [
251 # First try getting language as is (case-sensitive)
252 Q(code=code),
253 # Then try getting language as is (case-insensitive)
254 Q(code__iexact=code),
255 # Replace dash with underscore (for things as zh_Hant)
256 Q(code__iexact=code.replace("-", "_")),
257 # Replace plus with underscore (for things as zh+Hant+HK on Android)
258 Q(code__iexact=code.replace("+", "_")),
259 ]
261 # Country codes used without underscore (ptbr insteat of pt_BR)
262 if len(code) == 4: 262 ↛ 263line 262 didn't jump to line 263 because the condition on line 262 was never true
263 expanded_code = f"{code[:2]}_{code[2:]}".lower()
264 lookups.append(Q(code__iexact=expanded_code))
266 for lookup in lookups: 266 ↛ 273line 266 didn't jump to line 273 because the loop on line 266 didn't complete
267 # First try getting language as is
268 ret = self.try_get(lookup)
269 if ret is not None: 269 ↛ 266line 269 didn't jump to line 266 because the condition on line 269 was always true
270 return ret
272 # Handle aliases
273 ret = self.aliases_get(code, expanded_code)
274 if ret is not None:
275 return ret
277 # Parse the string
278 lang, country, subtags = self.parse_lang_country(code)
280 # Try "corrected" code
281 if country is not None:
282 if "@" in country:
283 region, variant = country.split("@", 1)
284 country = f"{region.upper()}@{variant.lower()}"
285 elif "_" in country:
286 # Xliff way of defining variants
287 region, variant = country.split("_", 1)
288 country = f"{region.upper()}@{variant.lower()}"
289 elif country in KNOWN_SUFFIXES:
290 country = country.title()
291 else:
292 country = country.upper()
293 newcode = f"{lang.lower()}_{country}"
294 else:
295 newcode = lang.lower()
297 if subtags:
298 newcode += subtags
300 ret = self.try_get(code__iexact=newcode)
301 if ret is not None:
302 return ret
304 # Try canonical variant
305 if settings.SIMPLIFY_LANGUAGES:
306 if newcode.lower() in DEFAULT_LANGS:
307 ret = self.try_get(code=lang.lower())
308 elif expanded_code is not None and expanded_code in DEFAULT_LANGS:
309 ret = self.try_get(code=expanded_code[:2])
310 if ret is not None:
311 return ret
313 # Try using name
314 ret = self.try_get(Q(name__iexact=code) & Q(code__in=data.NO_CODE_LANGUAGES))
315 if ret is not None:
316 return ret
318 return newcode
320 def auto_get_or_create(
321 self,
322 code: str,
323 create: bool = True,
324 languages_cache: dict[str, Language] | None = None,
325 ) -> Language:
326 """Try to get language using fuzzy_get and create it if that fails."""
327 if languages_cache is not None and code in languages_cache: 327 ↛ 328line 327 didn't jump to line 328 because the condition on line 327 was never true
328 return languages_cache[code]
330 ret = self.fuzzy_get(code)
331 if isinstance(ret, Language): 331 ↛ 337line 331 didn't jump to line 337 because the condition on line 331 was always true
332 if languages_cache is not None:
333 languages_cache[code] = ret
334 return ret
336 # Create new one
337 language = self.auto_create(ret, create)
338 if languages_cache is not None:
339 languages_cache[code] = language
340 return language
342 def auto_create(self, code: str, create: bool = True) -> Language:
343 """
344 Automatically create new language.
346 It is based on code and best guess of parameters.
347 """
348 # Create standard language
349 name = f"{code} {GENERATED_SUFFIX}"
350 if create:
351 lang = self.get_or_create(code=code, defaults={"name": name})[0]
352 else:
353 lang = Language(code=code, name=name)
355 baselang = None
357 # Check for different variant
358 if baselang is None and "@" in code:
359 parts = code.split("@")
360 baselang = self.fuzzy_get_strict(code=parts[0])
362 # Check for different country
363 if baselang is None and ("_" in code or "-" in code):
364 parts = code.replace("-", "_").split("_")
365 baselang = self.fuzzy_get_strict(code=parts[0])
367 if baselang is not None:
368 lang.name = baselang.name
369 lang.direction = baselang.direction
370 if create:
371 lang.save()
372 baseplural = baselang.plural
373 lang.plural_set.create(
374 source=Plural.SOURCE_DEFAULT,
375 number=baseplural.number,
376 formula=baseplural.formula,
377 )
378 elif create:
379 lang.plural_set.create(
380 source=Plural.SOURCE_DEFAULT, number=2, formula="n != 1"
381 )
383 return lang
385 def have_translation(self):
386 """Return list of languages which have at least one translation."""
387 from weblate.trans.models import Translation
389 return self.filter(Exists(Translation.objects.filter(language=OuterRef("pk"))))
391 def order(self):
392 return self.order_by("name")
394 def order_translated(self):
395 return sort_objects(self)
397 def get_by_code(
398 self,
399 code: str,
400 cache: dict[str, Language],
401 langmap: dict[str, str] | None = None,
402 ) -> Language:
403 """
404 Get language by code.
406 Cached and aliases aware getter.
407 """
408 if code in cache:
409 return cache[code]
410 if langmap and code in langmap:
411 language = self.fuzzy_get_strict(code=langmap[code])
412 else:
413 language = self.fuzzy_get_strict(code=code)
414 if language is None:
415 raise Language.DoesNotExist(code)
416 cache[code] = language
417 return language
419 def as_choices(self, use_code: bool = True, user: User | None = None):
420 if user is not None:
421 key = user.profile.get_translation_orderer(request=None)
422 else:
423 key = str
425 languages = sort_unicode(self, key)
426 return (
427 (
428 language.code if use_code else language.pk,
429 f"{gettext(language.name)} ({language.code})",
430 )
431 for language in languages
432 )
434 def get(self, *args, **kwargs):
435 """Get language with caching of default language."""
436 if not args and not kwargs.pop("skip_cache", False):
437 default = Language.objects.default_language
438 if kwargs in (
439 {"code": settings.DEFAULT_LANGUAGE},
440 {"pk": default.pk},
441 {"id": default.id},
442 ):
443 return default
444 return super().get(*args, **kwargs)
446 def get_request_language(self, request: AuthenticatedHttpRequest):
447 """
448 Guess user language from a HTTP request.
450 Accept-Language HTTP header, for most browser it consists of browser
451 language with higher rank and OS language with lower rank so it still
452 might be usable guess.
453 """
454 accept = request.headers.get("accept-language", "")
455 for accept_lang, _unused in parse_accept_lang_header(accept):
456 if accept_lang == "en":
457 continue
458 try:
459 return self.get(code__iexact=accept_lang)
460 except Language.DoesNotExist:
461 try:
462 return self.filter(code__iexact=accept_lang.replace("-", "_"))[0]
463 except IndexError:
464 continue
465 return None
467 def search(self, query: str):
468 return self.filter(Q(name__icontains=query) | Q(code__icontains=query))
470 def prefetch(self) -> Self:
471 return self.prefetch_related("plural_set")
473 def filter_for_add(self, project: Project) -> Self:
474 codes = BASIC_LANGUAGES
475 if settings.BASIC_LANGUAGES is not None:
476 codes = settings.BASIC_LANGUAGES
477 return self.filter(
478 # Include basic languages
479 Q(code__in=codes)
480 # Include source languages in a project
481 | Q(component__project=project)
482 # Include translations in a project
483 | Q(translation__component__project=project)
484 ).distinct()
487def dummy_logger(message: str) -> None:
488 return
491class LanguageManager(models.Manager.from_queryset(LanguageQuerySet)):
492 use_in_migrations = True
494 def flush_object_cache(self) -> None:
495 if "default_language" in self.__dict__: 495 ↛ 496line 495 didn't jump to line 496 because the condition on line 495 was never true
496 del self.__dict__["default_language"]
498 @cached_property
499 def default_language(self):
500 """Return English language object."""
501 return self.get(code=settings.DEFAULT_LANGUAGE, skip_cache=True)
503 def setup( # noqa: C901
504 self,
505 *,
506 update: bool,
507 logger: Callable[[str], None] | None = None,
508 ) -> None:
509 """
510 Create basic set of languages.
512 It is based on languages defined in the languages-data repo.
513 """
514 from weblate_language_data.languages import LANGUAGES
515 from weblate_language_data.population import POPULATION
517 if logger is None: 517 ↛ 521line 517 didn't jump to line 521 because the condition on line 517 was always true
518 logger = dummy_logger
520 # Invalidate cache, we might change languages
521 self.flush_object_cache()
522 languages = {
523 language.code: language
524 for language in self.prefetch().iterator(chunk_size=1000)
525 }
526 plurals: dict[str, dict[int, list[Plural]]] = {}
527 # Create Weblate languages
528 for code, name, nplurals, plural_formula in LANGUAGES:
529 population = POPULATION[code]
531 if code in languages: 531 ↛ 532line 531 didn't jump to line 532 because the condition on line 531 was never true
532 lang = languages[code]
533 else:
534 languages[code] = lang = self.create(
535 code=code, name=name, population=population
536 )
537 logger(f"Created language {code}")
539 direction = lang.guess_direction()
540 # Should we update existing?
541 if update and ( 541 ↛ 546line 541 didn't jump to line 546 because the condition on line 541 was never true
542 lang.name != name
543 or lang.direction != direction
544 or lang.population != population
545 ):
546 lang.name = name
547 lang.direction = direction
548 lang.population = population
549 logger(f"Updated language {code}")
550 lang.save()
552 plural_data = {
553 "number": nplurals,
554 "formula": plural_formula,
555 }
557 # Fetch existing plurals
558 plurals[code] = defaultdict(list)
559 for plural in lang.plural_set.all(): 559 ↛ 560line 559 didn't jump to line 560 because the loop on line 559 never started
560 plurals[code][plural.source].append(plural)
562 if Plural.SOURCE_DEFAULT in plurals[code]: 562 ↛ 563line 562 didn't jump to line 563 because the condition on line 562 was never true
563 plural = plurals[code][Plural.SOURCE_DEFAULT][0]
564 modified = False
565 for item, value in plural_data.items():
566 if getattr(plural, item) != value:
567 modified = True
568 setattr(plural, item, value)
569 if modified:
570 logger(
571 f"Updated default plural {plural_formula} for language {code}"
572 )
573 plural.save()
574 else:
575 plural = lang.plural_set.create(
576 source=Plural.SOURCE_DEFAULT, language=lang, **plural_data
577 )
578 plurals[code][Plural.SOURCE_DEFAULT].append(plural)
579 logger(f"Created default plural {plural_formula} for language {code}")
581 # Create addditiona plurals
582 extra_plurals = (
583 (Plural.SOURCE_GETTEXT, EXTRAPLURALS),
584 (Plural.SOURCE_CLDR, CLDRPLURALS),
585 (Plural.SOURCE_QT, QTPLURALS),
586 )
587 for source, definitions in extra_plurals:
588 for code, _unused, nplurals, plural_formula in definitions:
589 lang = languages[code]
591 for plural in plurals[code][source]:
592 try:
593 if plural.same_plural(nplurals, plural_formula): 593 ↛ 594line 593 didn't jump to line 594 because the condition on line 593 was never true
594 break
595 except ValueError:
596 # Fall back to string compare if parsing failed
597 if (
598 plural.number == nplurals
599 and plural.formula == plural_formula
600 ):
601 break
602 else:
603 plural = lang.plural_set.create(
604 source=source,
605 number=nplurals,
606 formula=plural_formula,
607 type=get_plural_type(lang.base_code, plural_formula),
608 )
609 plurals[code][source].append(plural)
610 logger(f"Created plural {plural_formula} for language {code}")
612 # CLDR and QT plurals should have just a single of them
613 if source != Plural.SOURCE_GETTEXT and len(plurals[code][source]) > 1: 613 ↛ 614line 613 didn't jump to line 614 because the condition on line 613 was never true
614 logger(
615 f"Removing extra {source} plurals for language {code} ({len(plurals[code][source])})!"
616 )
617 for extra_plural in plurals[code][source]:
618 if extra_plural != plural:
619 extra_plural.translation_set.update(plural=plural)
620 extra_plural.delete()
621 plurals[code][source] = [plural]
623 # Sync FORMULA_WITH_ZERO
624 for code, language_plurals in plurals.items():
625 if Plural.SOURCE_CLDR_ZERO in language_plurals: 625 ↛ 626line 625 didn't jump to line 626 because the condition on line 625 was never true
626 if Plural.SOURCE_CLDR in language_plurals:
627 cldr_plural = language_plurals[Plural.SOURCE_CLDR][0]
628 else:
629 cldr_plural = language_plurals[Plural.SOURCE_DEFAULT][0]
630 zero_plural = language_plurals[Plural.SOURCE_CLDR_ZERO][0]
631 current_formula = FORMULA_WITH_ZERO[cldr_plural.formula]
632 if zero_plural.formula != current_formula:
633 logger(f"Updating CLDR plural with zero for {code}")
634 zero_plural.formula = current_formula
635 zero_plural.number = cldr_plural.number + 1
636 zero_plural.save(update_fields=["formula", "number"])
638 # Migrate content from aliased language to alias target
639 weblate_data_lang_codes = {lang[0] for lang in LANGUAGES}
640 for code, language in languages.items():
641 if (code in ALIASES) and (code not in weblate_data_lang_codes): 641 ↛ 642line 641 didn't jump to line 642 because the condition on line 641 was never true
642 alias_target = Language.objects.get(code=ALIASES[code])
643 self.move_language(language, alias_target, logger)
645 # delete alias language if blank
646 if language.has_no_children():
647 logger(f"Removing {language}")
648 language.delete()
649 else:
650 logger(
651 f"Skipping removal of {language} due to existing child objects"
652 )
654 self._fixup_plural_types(logger)
656 def move_language(
657 self,
658 source: Language,
659 target: Language,
660 logger: Callable[[str], None] | None = None,
661 ) -> None:
662 """Migrate all content from one language to anoother."""
663 if logger is None:
664 logger = dummy_logger
665 for translation in source.translation_set.iterator():
666 other = translation.component.translation_set.filter(language=target)
667 if other.exists():
668 logger(f"Already exists: {translation}")
669 continue
670 translation.language = target
671 translation.save()
672 source.announcement_set.update(language=target)
674 for profile in source.profile_set.iterator():
675 profile.languages.remove(source)
676 profile.languages.add(target)
678 for profile in source.secondary_profile_set.iterator():
679 profile.secondary_languages.remove(source)
680 profile.secondary_languages.add(target)
682 source.change_set.update(language=target)
684 source.component_set.update(source_language=target)
685 for group in source.group_set.iterator():
686 group.languages.remove(source)
687 group.languages.add(target)
689 for plural in source.plural_set.iterator():
690 formulas = target.plural_set.filter(
691 source=plural.source, formula=plural.formula
692 )
693 try:
694 # Use matching plural if it exists
695 new_plural = formulas[0]
696 except IndexError:
697 # Create new plural based on current one
698 new_plural = target.plural_set.create(
699 source=plural.source,
700 number=plural.number,
701 formula=plural.formula,
702 type=plural.type,
703 )
704 plural.save()
706 # Migrate all moved translations to the new plural
707 plural.translation_set.filter(language=target).update(plural=new_plural)
709 source.memory_source_set.update(source_language=target)
710 source.memory_target_set.update(target_language=target)
712 def _fixup_plural_types(self, logger) -> None:
713 """Fix plural types as they were changed in Weblate codebase."""
714 if not Plural.objects.filter(type=data.PLURAL_ONE_FEW_MANY).exists(): 714 ↛ 715line 714 didn't jump to line 715 because the condition on line 714 was never true
715 for plural in Plural.objects.filter(
716 type=data.PLURAL_ONE_FEW_OTHER
717 ).select_related("language"):
718 language = plural.language
719 newtype = get_plural_type(language.base_code, plural.formula)
720 if newtype == data.PLURAL_UNKNOWN:
721 msg = f"Invalid plural type of {plural.formula}"
722 raise ValueError(msg)
723 if newtype != plural.type:
724 plural.type = newtype
725 plural.save(update_fields=["type"])
726 logger(
727 f"Updated type of {plural.formula} for language {language.code}"
728 )
731def setup_lang(sender, **kwargs) -> None:
732 """Create basic set of languages on database migration."""
733 if settings.UPDATE_LANGUAGES: 733 ↛ exitline 733 didn't return from function 'setup_lang' because the condition on line 733 was always true
734 with transaction.atomic():
735 Language.objects.setup(update=True)
738class Language(models.Model, CacheKeyMixin):
739 code = models.SlugField(
740 max_length=LANGUAGE_CODE_LENGTH,
741 unique=True,
742 verbose_name=gettext_lazy("Language code"),
743 )
744 name = models.CharField(
745 max_length=LANGUAGE_NAME_LENGTH, verbose_name=gettext_lazy("Language name")
746 )
747 direction = models.CharField(
748 verbose_name=gettext_lazy("Text direction"),
749 max_length=3,
750 default="",
751 choices=(
752 ("", gettext_lazy("Automatically detect text direction")),
753 ("ltr", gettext_lazy("Left to right")),
754 ("rtl", gettext_lazy("Right to left")),
755 ),
756 )
757 population = models.BigIntegerField(
758 gettext_lazy("Number of speakers"),
759 help_text=gettext_lazy("Number of people speaking this language."),
760 default=0,
761 )
763 objects = LanguageManager()
765 class Meta:
766 verbose_name = "Language"
767 verbose_name_plural = "Languages"
768 # Use own manager to utilize caching of English
769 base_manager_name = "objects"
771 def __str__(self) -> str:
772 return self.format_full_name(self.get_localized_name())
774 def __init__(self, *args, **kwargs) -> None:
775 from weblate.utils.stats import LanguageStats
777 super().__init__(*args, **kwargs)
778 self.stats = LanguageStats(self)
780 def save(self, *args, **kwargs):
781 """Set default direction for language."""
782 if not self.direction:
783 self.direction = self.guess_direction()
784 return super().save(*args, **kwargs)
786 def get_absolute_url(self) -> str:
787 return reverse("show_language", kwargs={"lang": self.code})
789 def get_url_path(self):
790 return ("-", "-", self.code)
792 def get_name(self):
793 """Not localized version of __str__."""
794 return self.format_full_name(self.name)
796 def format_full_name(self, name: str):
797 if self.show_language_code:
798 return f"{name} ({self.code})"
799 return name
801 def get_localized_name(self):
802 if self.name.endswith(GENERATED_SUFFIX): 802 ↛ 803line 802 didn't jump to line 803 because the condition on line 802 was never true
803 return self.name
804 name = gettext(self.name)
805 return f"{name[0].title()}{name[1:]}"
807 def guess_direction(self) -> str:
808 if self.base_code in RTL_LANGS or self.code in RTL_LANGS:
809 return "rtl"
810 return "ltr"
812 @property
813 def show_language_code(self):
814 return self.code not in data.NO_CODE_LANGUAGES
816 def get_html(self):
817 """
818 Return html attributes for markup in this language.
820 Includes language and direction HTML.
821 """
822 return format_html('lang="{}" dir="{}"', self.code, self.direction)
824 @cached_property
825 def base_code(self) -> str:
826 return self.code.replace("_", "-").split("-")[0]
828 def uses_whitespace(self) -> bool:
829 return self.base_code not in data.NO_SPACE_LANGUAGES
831 @cached_property
832 def plural(self):
833 if not self.pk: 833 ↛ 835line 833 didn't jump to line 835 because the condition on line 833 was never true
834 # Not yet saved, used in tests
835 return Plural(language=self)
836 # Filter in Python if query is cached
837 if self.plural_set.all()._result_cache is not None: # noqa: SLF001
838 for plural in self.plural_set.all():
839 if plural.source == Plural.SOURCE_DEFAULT:
840 return plural
841 return self.plural_set.filter(source=Plural.SOURCE_DEFAULT)[0]
843 def get_aliases_names(self) -> list[str]:
844 aliases: list[str] = [
845 alias for alias, codename in ALIASES.items() if codename == self.code
846 ]
847 if settings.SIMPLIFY_LANGUAGES: 847 ↛ 853line 847 didn't jump to line 853 because the condition on line 847 was always true
848 aliases.extend(
849 default_lang
850 for default_lang in DEFAULT_LANGS
851 if default_lang.startswith(self.code)
852 )
853 return sorted(aliases)
855 def is_base(self, vals: set[str]) -> bool:
856 """Detect whether language is in given list, ignores variants."""
857 return self.base_code in vals
859 def is_cjk(self) -> bool:
860 """Detect whether language is CJK, ignores variants."""
861 return self.is_base({"ja", "zh", "ko"})
863 def is_case_sensitive(self) -> bool:
864 """Detect whether language is case sensitive."""
865 return (
866 self.code not in CASE_INSENSITIVE_LANGS
867 and self.base_code not in CASE_INSENSITIVE_LANGS
868 )
870 def get_case_sensitivity_display(self) -> StrOrPromise:
871 if self.is_case_sensitive():
872 return gettext("Case-sensitive")
873 return gettext("Case-insensitive")
875 def has_no_children(self) -> bool:
876 """
877 Check if language has no child objects.
879 Can be used to determine if language can be safely deleted.
880 """
881 # translations are most likely objects to exist for a language
882 if self.translation_set.exists():
883 return False
885 # Collect all objects that will be deleted along with the current instance.
886 # This can be expensive as it fetches all the objects from the database.
887 collector = NestedObjects(self.__class__.objects.db)
888 collector.collect([self])
890 for nodes in collector.edges.values():
891 for node in nodes:
892 if isinstance(node, Language) and node == self:
893 continue
894 if isinstance(node, Plural) and node.language == self:
895 continue
896 return False
897 return True
900class PluralQuerySet(models.QuerySet):
901 def order(self):
902 return self.order_by("source")
905class Plural(models.Model):
906 PLURAL_CHOICES = (
907 (
908 data.PLURAL_NONE,
909 pgettext_lazy("Plural type", "None"),
910 ),
911 (
912 data.PLURAL_ONE_OTHER,
913 pgettext_lazy("Plural type", "One/other"),
914 ),
915 (
916 data.PLURAL_ONE_FEW_OTHER,
917 pgettext_lazy("Plural type", "One/few/other"),
918 ),
919 (
920 data.PLURAL_ARABIC,
921 pgettext_lazy("Plural type", "Arabic languages"),
922 ),
923 (
924 data.PLURAL_ZERO_ONE_OTHER,
925 pgettext_lazy("Plural type", "Zero/one/other"),
926 ),
927 (
928 data.PLURAL_ONE_TWO_OTHER,
929 pgettext_lazy("Plural type", "One/two/other"),
930 ),
931 (
932 data.PLURAL_ONE_OTHER_TWO,
933 pgettext_lazy("Plural type", "One/other/two"),
934 ),
935 (
936 data.PLURAL_ONE_TWO_FEW_OTHER,
937 pgettext_lazy("Plural type", "One/two/few/other"),
938 ),
939 (
940 data.PLURAL_OTHER_ONE_TWO_FEW,
941 pgettext_lazy("Plural type", "Other/one/two/few"),
942 ),
943 (
944 data.PLURAL_ONE_TWO_THREE_OTHER,
945 pgettext_lazy("Plural type", "One/two/three/other"),
946 ),
947 (
948 data.PLURAL_ONE_OTHER_ZERO,
949 pgettext_lazy("Plural type", "One/other/zero"),
950 ),
951 (
952 data.PLURAL_ONE_FEW_MANY_OTHER,
953 pgettext_lazy("Plural type", "One/few/many/other"),
954 ),
955 (
956 data.PLURAL_TWO_OTHER,
957 pgettext_lazy("Plural type", "Two/other"),
958 ),
959 (
960 data.PLURAL_ONE_TWO_FEW_MANY_OTHER,
961 pgettext_lazy("Plural type", "One/two/few/many/other"),
962 ),
963 (
964 data.PLURAL_ZERO_ONE_TWO_FEW_MANY_OTHER,
965 pgettext_lazy("Plural type", "Zero/one/two/few/many/other"),
966 ),
967 (
968 data.PLURAL_ZERO_OTHER,
969 pgettext_lazy("Plural type", "Zero/other"),
970 ),
971 (
972 data.PLURAL_ZERO_ONE_FEW_OTHER,
973 pgettext_lazy("Plural type", "Zero/one/few/other"),
974 ),
975 (
976 data.PLURAL_ZERO_ONE_TWO_FEW_OTHER,
977 pgettext_lazy("Plural type", "Zero/one/two/few/other"),
978 ),
979 (
980 data.PLURAL_ZERO_ONE_TWO_OTHER,
981 pgettext_lazy("Plural type", "Zero/one/two/other"),
982 ),
983 (
984 data.PLURAL_ZERO_ONE_FEW_MANY_OTHER,
985 pgettext_lazy("Plural type", "Zero/one/few/many/other"),
986 ),
987 (
988 data.PLURAL_ONE_MANY_OTHER,
989 pgettext_lazy("Plural type", "One/many/other"),
990 ),
991 (
992 data.PLURAL_ZERO_ONE_MANY_OTHER,
993 pgettext_lazy("Plural type", "Zero/one/many/other"),
994 ),
995 (
996 data.PLURAL_ONE_FEW_MANY,
997 pgettext_lazy("Plural type", "One/few/many"),
998 ),
999 (
1000 data.PLURAL_ONE_ZERO_FEW_OTHER,
1001 pgettext_lazy("Plural type", "One/zero/few/other"),
1002 ),
1003 (
1004 data.PLURAL_UNKNOWN,
1005 pgettext_lazy("Plural type", "Unknown"),
1006 ),
1007 )
1008 SOURCE_DEFAULT = 0
1009 SOURCE_GETTEXT = 1
1010 SOURCE_MANUAL = 2
1011 SOURCE_CLDR_ZERO = 3
1012 SOURCE_CLDR = 4
1013 SOURCE_ANDROID = 5
1014 SOURCE_QT = 6
1015 source = models.SmallIntegerField(
1016 default=SOURCE_DEFAULT,
1017 verbose_name=gettext_lazy("Plural definition source"),
1018 choices=(
1019 (SOURCE_DEFAULT, gettext_lazy("Default plural")),
1020 (SOURCE_GETTEXT, gettext_lazy("gettext plural formula")),
1021 (SOURCE_CLDR_ZERO, gettext_lazy("CLDR plural with zero")),
1022 (SOURCE_CLDR, gettext_lazy("CLDR v38+ plural")),
1023 (SOURCE_ANDROID, gettext_lazy("Android plural")),
1024 (SOURCE_QT, gettext_lazy("Qt Linguist plural")),
1025 (SOURCE_MANUAL, gettext_lazy("Manually entered formula")),
1026 ),
1027 )
1028 number = models.SmallIntegerField(
1029 default=2, verbose_name=gettext_lazy("Number of plurals")
1030 )
1031 formula = models.TextField(
1032 default="n != 1",
1033 validators=[validate_plural_formula],
1034 blank=False,
1035 verbose_name=gettext_lazy("Plural formula"),
1036 )
1037 type = models.IntegerField(
1038 choices=PLURAL_CHOICES,
1039 default=data.PLURAL_UNKNOWN,
1040 verbose_name=gettext_lazy("Plural type"),
1041 editable=False,
1042 )
1043 language = models.ForeignKey(Language, on_delete=models.deletion.CASCADE)
1045 objects = PluralQuerySet.as_manager()
1047 class Meta:
1048 verbose_name = "Plural form"
1049 verbose_name_plural = "Plural forms"
1051 def __str__(self) -> str:
1052 return self.get_type_display()
1054 def save(self, *args, **kwargs) -> None:
1055 self.type = get_plural_type(self.language.base_code, self.formula)
1056 super().save(*args, **kwargs)
1058 def get_absolute_url(self) -> str:
1059 return "{}#information".format(
1060 reverse("show_language", kwargs={"lang": self.language.code})
1061 )
1063 @cached_property
1064 def plural_form(self) -> str:
1065 return f"nplurals={self.number:d}; plural={self.formula};"
1067 @cached_property
1068 def plural_function(self):
1069 try:
1070 return c2py(self.formula or "0")
1071 except ValueError as error:
1072 msg = f"Could not compile formula {self.formula!r}: {error}"
1073 raise ValueError(msg) from error
1075 @cached_property
1076 def examples(self) -> dict[int, list[str]]:
1077 result: dict[int, list[str]] = defaultdict(list)
1078 func = self.plural_function
1079 for i in chain(range(10000), range(10000, 2000001, 1000)):
1080 ret = func(i) # pylint: disable=too-many-function-args
1081 if len(result[ret]) >= 10:
1082 continue
1083 result[ret].append(str(i))
1084 for example in result.values():
1085 if len(example) >= 10:
1086 example.append("…")
1087 return result
1089 @staticmethod
1090 def parse_plural_forms(plurals):
1091 matches = PLURAL_RE.match(plurals)
1092 if matches is None: 1092 ↛ 1093line 1092 didn't jump to line 1093 because the condition on line 1092 was never true
1093 msg = "Could not parse plural forms"
1094 raise ValueError(msg)
1096 number = int(matches.group(1))
1097 formula = matches.group(2)
1098 if not formula: 1098 ↛ 1099line 1098 didn't jump to line 1099 because the condition on line 1098 was never true
1099 formula = "0"
1100 # Try to parse the formula
1101 c2py(formula)
1103 return number, formula
1105 def same_as(self, other):
1106 """Check whether the given plurals are equivalent."""
1107 return is_same_plural(
1108 self.number,
1109 self.formula,
1110 other.number,
1111 other.formula,
1112 our_function=self.plural_function,
1113 plural_function=other.plural_function,
1114 )
1116 def same_plural(self, number: int, formula: str):
1117 """Compare whether given plurals formula matches."""
1118 return is_same_plural(
1119 self.number,
1120 self.formula,
1121 number,
1122 formula,
1123 our_function=self.plural_function,
1124 )
1126 def get_plural_label(self, idx):
1127 """Return label for plural form."""
1128 return format_html(
1129 PLURAL_TITLE,
1130 name=self.get_plural_name(idx),
1131 examples=format_html_join_comma(
1132 "{}", list_to_tuples(self.examples.get(idx, []))
1133 ),
1134 title=gettext("Example counts for this plural form."),
1135 )
1137 def get_plural_name(self, idx):
1138 """Return name for plural form."""
1139 try:
1140 return str(data.PLURAL_NAMES[self.type][idx])
1141 except (IndexError, KeyError):
1142 if idx == 0:
1143 return gettext("Singular")
1144 if idx == 1:
1145 return gettext("Plural")
1146 return gettext("Plural form %d") % idx
1148 def list_plurals(self):
1149 for i in range(self.number):
1150 yield {
1151 "index": i,
1152 "name": self.get_plural_name(i),
1153 "examples": format_html_join_comma(
1154 "{}", list_to_tuples(self.examples.get(i, []))
1155 ),
1156 }
1159class PluralMapper:
1160 instances: WeakValueDictionary[tuple[str, str], PluralMapper] = (
1161 WeakValueDictionary()
1162 )
1164 def __new__(cls, source_plural: Plural, target_plural: Plural):
1165 key = (source_plural.formula, target_plural.formula)
1166 obj = cls.instances.get(key)
1167 if obj is None:
1168 obj = cls.instances[key] = super().__new__(cls)
1169 return obj
1171 def __init__(self, source_plural: Plural, target_plural: Plural) -> None:
1172 self.source_plural = source_plural
1173 self.target_plural = target_plural
1174 self.same_plurals = source_plural.same_as(target_plural)
1176 def __str__(self):
1177 return f"<PluralMapper '{self.source_plural}' -> '{self.target_plural}'>"
1179 @cached_property
1180 def target_map(self) -> tuple[tuple[int | None, int | None], ...]:
1181 exact_source_map: dict[int, int] = {}
1182 all_source_map: dict[int, int] = {}
1183 for i, examples in self.source_plural.examples.items():
1184 if len(examples) == 1:
1185 exact_source_map[int(examples[0])] = i
1186 else:
1187 for example in examples:
1188 try:
1189 value = int(example)
1190 except ValueError:
1191 continue
1192 all_source_map[value] = i
1194 target_plural = self.target_plural
1195 result: list[tuple[int | None, int | None]] = []
1196 last = target_plural.number - 1
1197 for i in range(target_plural.number):
1198 examples = target_plural.examples.get(i, [])
1199 if len(examples) == 1:
1200 # Map plurals 1:1
1201 number = int(examples[0])
1202 if number in exact_source_map:
1203 result.append((exact_source_map[number], None))
1204 elif number in all_source_map:
1205 result.append((all_source_map[number], number))
1206 else:
1207 result.append((-1, number))
1208 elif i == last:
1209 # Map last plural
1210 result.append((-1, None))
1211 else:
1212 # Look for examples subset
1213 values = []
1214 for example in examples:
1215 try:
1216 value = int(example)
1217 except ValueError:
1218 continue
1219 values.append(value)
1220 mapped = {
1221 all_source_map[value] for value in values if value in all_source_map
1222 }
1223 if len(mapped) == 1:
1224 result.append((mapped.pop(), None))
1225 else:
1226 # Fall back to not mapping
1227 result.append((None, None))
1228 return tuple(result)
1230 def map(self, unit: Unit, other_unit: Unit | None = None) -> list[str]:
1231 if other_unit is not None:
1232 source_strings = other_unit.get_target_plurals()
1233 else:
1234 source_strings = unit.get_source_plurals()
1235 if self.same_plurals or len(source_strings) == 1:
1236 strings_to_translate = source_strings
1237 elif self.target_plural.number == 1:
1238 strings_to_translate = [source_strings[-1]]
1239 else:
1240 strings_to_translate = []
1241 format_check = next(
1242 (
1243 check
1244 for check in CHECKS.values()
1245 if (
1246 isinstance(check, BaseFormatCheck)
1247 and check.enable_string in unit.all_flags
1248 and check.plural_parameter_regexp
1249 )
1250 ),
1251 None,
1252 )
1253 for source_index, number_to_interpolate in self.target_map:
1254 s = "" if source_index is None else source_strings[source_index]
1255 if s and number_to_interpolate is not None and format_check:
1256 s = format_check.interpolate_number(s, number_to_interpolate)
1257 strings_to_translate.append(s)
1258 return strings_to_translate
1260 @staticmethod
1261 def get_other_units(
1262 units: list[Unit] | UnitQuerySet, language: Language
1263 ) -> dict[int, Unit]:
1264 if not units:
1265 return {}
1266 component = units[0].translation.component
1267 try:
1268 translation = component.translation_set.get(language=language)
1269 except ObjectDoesNotExist:
1270 return {}
1271 return {
1272 other.id_hash: other
1273 for other in translation.unit_set.filter(
1274 state__gte=STATE_TRANSLATED,
1275 id_hash__in={unit.id_hash for unit in units},
1276 )
1277 }
1279 def map_units(
1280 self, units: list[Unit] | UnitQuerySet, other_units: dict[int, Unit] | None
1281 ) -> None:
1282 other: Unit | None
1283 for unit in units:
1284 if other_units is None:
1285 other = None
1286 elif unit.id_hash not in other_units:
1287 # Unit strings not available
1288 unit.plural_map = []
1289 continue
1290 else:
1291 other = other_units[unit.id_hash]
1292 unit.plural_map = self.map(unit, other)
1294 def zip(self, sources: list[str], targets: list[str], unit: Unit):
1295 if len(sources) != self.source_plural.number:
1296 msg = "length of `sources` doesn't match the number of source plurals"
1297 raise ValueError(msg)
1298 if len(targets) != self.target_plural.number:
1299 msg = "length of `targets` doesn't match the number of target plurals"
1300 raise ValueError(msg)
1301 if self.same_plurals:
1302 return zip(sources, targets, strict=True)
1303 return [
1304 (sources[-1 if i is None else i], targets[j])
1305 for (i, _), j in zip(self.target_map, range(len(targets)), strict=True)
1306 ]
1309class WeblateLanguagesConf(AppConf):
1310 """Languages settings."""
1312 # Update languages on migration
1313 UPDATE_LANGUAGES = True
1315 # Use simple language codes for default language/country combinations
1316 SIMPLIFY_LANGUAGES = True
1318 # Default source languaage
1319 DEFAULT_LANGUAGE = "en"
1321 # List of basic languages to show for user when adding new translation
1322 BASIC_LANGUAGES = None
1324 class Meta:
1325 prefix = ""