Coverage for app/venv/lib/python3.14/site-packages/weblate/trans/discovery.py: 13%
200 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-10-07 07:15 +0000
1# Copyright © Michal Čihař <michal@weblate.org>
2#
3# SPDX-License-Identifier: GPL-3.0-or-later
4from __future__ import annotations
6import os
7import re
8from itertools import chain
9from typing import TYPE_CHECKING
11from django.core.exceptions import ValidationError
12from django.utils.functional import cached_property
13from django.utils.text import slugify
14from django.utils.translation import gettext
16from weblate.formats.models import FILE_FORMATS
17from weblate.logger import LOGGER
18from weblate.trans.defines import COMPONENT_NAME_LENGTH
19from weblate.trans.models import Component
20from weblate.trans.tasks import create_component
21from weblate.trans.util import path_separator
22from weblate.utils.render import render_template
24if TYPE_CHECKING: 24 ↛ 25line 24 didn't jump to line 25 because the condition on line 24 was never true
25 from weblate.formats.base import TranslationFormat
27# Attributes to copy from main component
28COPY_ATTRIBUTES = (
29 "project",
30 "vcs",
31 "license",
32 "agreement",
33 "source_language",
34 "report_source_bugs",
35 "allow_translation_propagation",
36 "enable_suggestions",
37 "suggestion_voting",
38 "suggestion_autoaccept",
39 "check_flags",
40 "new_lang",
41 "language_code_style",
42 "commit_message",
43 "add_message",
44 "delete_message",
45 "merge_message",
46 "addon_message",
47 "pull_message",
48 "push_on_commit",
49 "commit_pending_age",
50 "edit_template",
51 "manage_units",
52 "variant_regex",
53 "category_id",
54 "key_filter",
55 "secondary_language",
56)
59class ComponentDiscovery:
60 def __init__(
61 self,
62 component,
63 *,
64 match: str,
65 name_template: str,
66 file_format: str,
67 language_regex: str = "^[^.]+$",
68 base_file_template: str = "",
69 new_base_template: str = "",
70 intermediate_template: str = "",
71 path=None,
72 copy_addons=True,
73 ) -> None:
74 self.component = component
75 if path is None:
76 self.path = self.component.full_path
77 else:
78 self.path = path
79 self.path_match = self.compile_match(match)
80 self.name_template = name_template
81 self.base_file_template = base_file_template
82 self.new_base_template = new_base_template
83 self.intermediate_template = intermediate_template
84 self.language_re = language_regex
85 self.language_match = re.compile(language_regex)
86 self.file_format = file_format
87 self.copy_addons = copy_addons
89 @cached_property
90 def file_format_cls(self) -> type[TranslationFormat]:
91 return FILE_FORMATS[self.file_format]
93 @staticmethod
94 def extract_kwargs(params):
95 """Extract kwargs for discovery from wider dict."""
96 attrs = (
97 "match",
98 "name_template",
99 "language_regex",
100 "base_file_template",
101 "new_base_template",
102 "intermediate_template",
103 "file_format",
104 "copy_addons",
105 )
106 return {k: v for k, v in params.items() if k in attrs}
108 def compile_match(self, match):
109 parts = match.split("(?P=language)")
110 offset = 1
111 while len(parts) > 1:
112 parts[0:2] = [f"{parts[0]}(?P<_language_{offset}>(?P=language)){parts[1]}"]
113 offset += 1
114 return re.compile(f"^{parts[0]}$")
116 @cached_property
117 def matches(self):
118 """Return matched files together with match groups and mask."""
119 result = []
120 base = os.path.realpath(self.path)
121 for root, dirnames, filenames in os.walk(self.path, followlinks=True):
122 for filename in chain(filenames, dirnames):
123 fullname = os.path.join(root, filename)
125 # Skip files outside our root
126 if not os.path.realpath(fullname).startswith(base):
127 continue
129 # Calculate relative path
130 path = path_separator(os.path.relpath(fullname, self.path))
132 # Check match against our regexp
133 matches = self.path_match.match(path)
134 if not matches:
135 continue
137 # Check language regexp
138 language_part = matches.group("language")
139 if language_part is None or not self.language_match.match(
140 language_part
141 ):
142 continue
144 # Calculate file mask for match
145 replacements = [(matches.start("language"), matches.end("language"))]
146 replacements.extend(
147 (matches.start(group), matches.end(group))
148 for group in matches.groupdict()
149 if group.startswith("_language_")
150 )
151 maskparts = []
152 maskpath = path
153 for start, end in sorted(replacements, reverse=True):
154 maskparts.append(maskpath[end:])
155 maskpath = maskpath[:start]
156 maskparts.append(maskpath)
158 mask = "*".join(reversed(maskparts))
160 result.append((path, matches.groupdict(), mask))
162 return result
164 @cached_property
165 def matched_files(self):
166 """Return list of matched files."""
167 return [x[0] for x in self.matches]
169 @cached_property
170 def matched_components(self):
171 """Return list of matched components."""
172 result = {}
173 for path, groups, mask in self.matches:
174 if mask not in result:
175 name = render_template(self.name_template, **groups)
176 result[mask] = {
177 "files": {path},
178 "languages": {groups["language"]},
179 "files_langs": {(path, groups["language"])},
180 "base_file": render_template(self.base_file_template, **groups),
181 "new_base": render_template(self.new_base_template, **groups),
182 "intermediate": render_template(
183 self.intermediate_template, **groups
184 ),
185 "mask": mask,
186 "name": name,
187 "slug": slugify(name),
188 }
189 else:
190 result[mask]["files"].add(path)
191 result[mask]["languages"].add(groups["language"])
192 result[mask]["files_langs"].add((path, groups["language"]))
193 return result
195 def log(self, *args) -> None:
196 if self.component:
197 self.component.log_info(*args)
198 else:
199 LOGGER.info(*args)
201 def create_component(
202 self,
203 main: Component | None,
204 match,
205 *,
206 background: bool = False,
207 preview: bool = False,
208 existing_slugs: set[str],
209 existing_names: set[str],
210 **kwargs,
211 ):
212 max_length = COMPONENT_NAME_LENGTH
214 def get_val(key, extra=0):
215 result = match[key]
216 if len(result) > max_length - extra:
217 result = result[: max_length - extra]
218 return result
220 # Get name and slug
221 name = get_val("name") or "Component"
222 slug = get_val("slug") or "component"
224 # Copy attributes from main component
225 for key in COPY_ATTRIBUTES:
226 if key not in kwargs and main is not None:
227 kwargs[key] = getattr(main, key)
229 # Disable template editing if not supported by format
230 if not self.file_format_cls.can_edit_base:
231 kwargs["edit_template"] = False
233 # Fill in repository
234 if "repo" not in kwargs:
235 if main is None:
236 raise ValueError
237 kwargs["repo"] = main.get_repo_link_url()
239 # Deal with duplicate name or slug in the same (or none) category
240 if slug.lower() in existing_slugs or name.lower() in existing_names:
241 base_name = get_val("name", 4)
242 base_slug = get_val("slug", 4)
244 for i in range(1, 1000):
245 name = f"{base_name} {i}"
246 slug = f"{base_slug}-{i}"
248 if slug.lower() in existing_slugs or name.lower() in existing_names:
249 continue
250 break
251 existing_slugs.add(slug.lower())
252 existing_names.add(name.lower())
254 # Fill in remaining attributes
255 kwargs.update(
256 {
257 "name": name,
258 "slug": slug,
259 "template": match["base_file"],
260 "filemask": match["mask"],
261 "new_base": match["new_base"],
262 "intermediate": match["intermediate"],
263 "file_format": self.file_format,
264 "language_regex": self.language_re,
265 "copy_from": main.pk if main else None,
266 "copy_addons": self.copy_addons,
267 }
268 )
270 # Create non-saved object for validation
271 component_kwargs = kwargs.copy()
272 component_kwargs.pop("copy_from")
273 component_kwargs.pop("copy_addons")
274 # main can be None as well
275 component = Component(**component_kwargs, linked_component=main)
277 # Special handling for new_lang
278 try:
279 component.clean_new_lang()
280 except ValidationError as error:
281 component.new_lang = kwargs["new_lang"] = "none"
282 self.log("Disabling adding new languages for %s because of %s", name, error)
284 # This might raise an exception
285 component.full_clean()
287 if preview:
288 return component
290 self.log("Creating component %s", name)
292 # Can't pass objects, pass only IDs
293 kwargs["project"] = kwargs["project"].pk
294 kwargs["source_language"] = kwargs["source_language"].pk
295 if background:
296 create_component.delay(**kwargs, in_task=True)
297 return None
298 return create_component(**kwargs)
300 def cleanup(self, main, processed, preview=False):
301 deleted = []
302 for component in main.linked_children.exclude(pk__in=processed):
303 if component.has_template():
304 # Valid template?
305 if os.path.exists(component.get_template_filename()):
306 continue
307 elif component.new_base:
308 # Valid new base?
309 if os.path.exists(component.get_new_base_filename()):
310 continue
311 elif component.get_mask_matches():
312 continue
314 # Delete as needed files seem to be missing
315 deleted.append((None, component))
316 if not preview:
317 component.delete()
319 return deleted
321 def get_skip_reason(self, match):
322 # Skip matches to main component
323 if match["mask"] == self.component.filemask:
324 return gettext("File mask matches the main component.")
326 for param in ("base_file", "new_base", "intermediate"):
327 name = match[param]
328 if not name:
329 continue
330 fullname = os.path.join(self.component.full_path, name)
331 if not os.path.exists(fullname):
332 return gettext("{filename} ({parameter}) does not exist.").format(
333 filename=name,
334 parameter=param,
335 )
337 return None
339 def perform(self, preview=False, remove=False, background=False):
340 created = []
341 matched = []
342 deleted = []
343 skipped = []
344 processed = set()
346 main = self.component
347 category_components = main.project.component_set.filter(category=main.category)
348 existing_children: dict[str, Component] = {
349 component.filemask: component for component in main.linked_children.all()
350 }
351 existing_slugs: set[str] = {
352 slug.lower() for slug in category_components.values_list("slug", flat=True)
353 }
354 existing_names: set[str] = {
355 name.lower() for name in category_components.values_list("name", flat=True)
356 }
358 for match in self.matched_components.values():
359 # Skip invalid matches
360 reason = self.get_skip_reason(match)
361 if reason is not None:
362 skipped.append((match, reason))
363 continue
365 try:
366 found = existing_children[match["mask"]]
367 except KeyError:
368 # Create new component
369 try:
370 component = self.create_component(
371 main,
372 match,
373 background=background,
374 preview=preview,
375 existing_slugs=existing_slugs,
376 existing_names=existing_names,
377 )
378 except ValidationError as error:
379 skipped.append((match, str(error)))
380 else:
381 # Correctly created
382 if component is not None and component.id:
383 processed.add(component.id)
384 created.append((match, component))
385 existing_children[match["mask"]] = component
386 else:
387 # Component already exists
388 matched.append((match, found))
389 processed.add(found.id)
391 if remove:
392 deleted = self.cleanup(main, processed, preview)
394 return created, matched, deleted, skipped