Coverage for app/venv/lib/python3.14/site-packages/weblate/metrics/models.py: 57%
245 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-10-07 07:15 +0000
1# Copyright © Michal Čihař <michal@weblate.org>
2#
3# SPDX-License-Identifier: GPL-3.0-or-later
5from __future__ import annotations
7import datetime
8from itertools import zip_longest
9from typing import TYPE_CHECKING, cast
11from django.core.cache import cache
12from django.db import models, transaction
13from django.db.models import Count, Q
14from django.db.models.signals import post_delete, post_save
15from django.dispatch import receiver
16from django.utils import timezone
17from django.utils.functional import cached_property
19from weblate.auth.models import User
20from weblate.lang.models import Language
21from weblate.memory.models import Memory
22from weblate.screenshots.models import Screenshot
23from weblate.trans.actions import ActionEvents
24from weblate.trans.models import (
25 Category,
26 Component,
27 ComponentList,
28 Project,
29 Translation,
30)
31from weblate.trans.models.change import Change
32from weblate.utils.decorators import disable_for_loaddata
33from weblate.utils.stats import (
34 CategoryLanguage,
35 GlobalStats,
36 ProjectLanguage,
37 prefetch_stats,
38)
40if TYPE_CHECKING: 40 ↛ 41line 40 didn't jump to line 41 because the condition on line 40 was never true
41 from weblate.trans.models.change import ChangeQuerySet
42 from weblate.utils.stats import (
43 BaseStats,
44 )
46BASIC_KEYS = {
47 "all",
48 "all_words",
49 "translated",
50 "translated_words",
51 "approved",
52 "approved_words",
53 "allchecks",
54 "allchecks_words",
55 "dismissed_checks",
56 "dismissed_checks_words",
57 "suggestions",
58 "suggestions_words",
59 "comments",
60 "comments_words",
61 "languages",
62}
63SOURCE_KEYS = BASIC_KEYS | {
64 "source_strings",
65 "source_words",
66}
68METRIC_ORDER = [
69 "all",
70 "all_words",
71 "translated",
72 "translated_words",
73 "approved",
74 "approved_words",
75 "allchecks",
76 "allchecks_words",
77 "dismissed_checks",
78 "dismissed_checks_words",
79 "suggestions",
80 "suggestions_words",
81 "comments",
82 "comments_words",
83 "languages",
84 "source_strings",
85 "source_words",
86 "changes",
87 "memory",
88 "users",
89 "contributors",
90 "projects",
91 "screenshots",
92 "components",
93 "translations",
94 "machinery:internal",
95 "machinery:external",
96 "public_projects",
97 "contributors_total",
98]
101class MetricQuerySet(models.QuerySet["Metric"]):
102 def filter_metric(
103 self, scope: int, relation: int, secondary: int = 0
104 ) -> MetricQuerySet:
105 # Include secondary in the query as it is part of unique index
106 # and makes subsequent date filtering more effective.
107 return self.filter(scope=scope, relation=relation, secondary=secondary)
109 def get_current_metric(
110 self, obj, scope: int, relation: int, secondary: int = 0
111 ) -> Metric:
112 today = timezone.now().date()
113 yesterday = today - datetime.timedelta(days=1)
115 base = self.filter_metric(scope, relation, secondary)
117 # Get metrics
118 try:
119 metric = base.get(date=today)
120 except Metric.DoesNotExist:
121 # Fallback to day before in case they are not yet calculated
122 try:
123 metric = base.get(date=yesterday)
124 except Metric.DoesNotExist:
125 metric = Metric()
127 # Trigger collection in case no data is present or when only
128 # changes are counted - when there is a single key.
129 if metric.data is None:
130 metric = Metric.objects.collect_auto(obj)
132 return metric
135class MetricManager(models.Manager["Metric"]):
136 def create_metrics(
137 self,
138 data: dict,
139 stats: BaseStats | None,
140 keys: set,
141 scope: int,
142 relation: int,
143 secondary: int = 0,
144 date=None,
145 ):
146 if stats is not None:
147 for key in keys:
148 data[key] = getattr(stats, key)
149 if date is None: 149 ↛ 153line 149 didn't jump to line 153 because the condition on line 149 was always true
150 date = timezone.now().date()
152 # Prepare data for database
153 db_data = None
154 changes = data.pop("changes")
155 if data: 155 ↛ 161line 155 didn't jump to line 161 because the condition on line 155 was always true
156 db_data = [data.pop(name, 0) for name in METRIC_ORDER]
157 if data: 157 ↛ 158line 157 didn't jump to line 158 because the condition on line 157 was never true
158 msg = f"Unsupported data: {data}"
159 raise ValueError(msg)
161 metric, created = self.get_or_create(
162 scope=scope,
163 relation=relation,
164 secondary=secondary,
165 date=date,
166 defaults={
167 "changes": changes,
168 "data": db_data,
169 },
170 )
171 if not created and not metric.data and db_data:
172 metric.data = db_data
173 metric.save(update_fields=["data"])
174 return metric
176 def initialize_metrics(self, scope: int, relation: int, secondary: int = 0) -> None:
177 today = timezone.now().date()
178 self.bulk_create(
179 [
180 Metric(
181 scope=scope,
182 relation=relation,
183 secondary=secondary,
184 changes=0,
185 date=today,
186 )
187 ],
188 ignore_conflicts=True,
189 )
191 def calculate_changes(
192 self, date, obj, scope: int, relation: int, secondary: int = 0
193 ):
194 """
195 Calculate changes for given scope and date.
197 This is used to fill in blanks in a history.
198 """
199 changes: ChangeQuerySet
200 if obj is None:
201 changes = Change.objects.all()
202 elif isinstance(
203 obj,
204 Translation
205 | Component
206 | Project
207 | User
208 | Language
209 | ProjectLanguage
210 | CategoryLanguage,
211 ):
212 changes = cast("ChangeQuerySet", obj.change_set.all()) # type: ignore[misc]
213 elif isinstance(obj, ComponentList):
214 changes = Change.objects.filter(component__in=obj.components.all())
215 elif isinstance(obj, Category):
216 changes = Change.objects.for_category(obj)
217 else:
218 msg = f"Unsupported type for metrics: {obj!r}"
219 raise TypeError(msg)
221 count = changes.filter_by_day(date - datetime.timedelta(days=1)).count()
222 self.create_metrics(
223 {"changes": count}, None, set(), scope, relation, secondary, date=date
224 )
225 return count
227 def collect_auto(self, obj):
228 if obj is None:
229 return self.collect_global()
230 if isinstance(obj, Translation):
231 return self.collect_translation(obj)
232 if isinstance(obj, Component):
233 return self.collect_component(obj)
234 if isinstance(obj, Project):
235 return self.collect_project(obj)
236 if isinstance(obj, Category):
237 return self.collect_category(obj)
238 if isinstance(obj, ComponentList):
239 return self.collect_component_list(obj)
240 if isinstance(obj, ProjectLanguage):
241 return self.collect_project_language(obj)
242 if isinstance(obj, CategoryLanguage):
243 return self.collect_category_language(obj)
244 if isinstance(obj, Language):
245 return self.collect_language(obj)
246 msg = f"Unsupported type for metrics: {obj!r}"
247 raise ValueError(msg)
249 @transaction.atomic
250 def collect_global(self):
251 stats = GlobalStats()
252 data = {
253 "projects": Project.objects.count(),
254 "public_projects": Project.objects.filter(
255 access_control__in={Project.ACCESS_PUBLIC, Project.ACCESS_PROTECTED}
256 ).count(),
257 "components": Component.objects.count(),
258 "translations": Translation.objects.count(),
259 "memory": Memory.objects.count(),
260 "screenshots": Screenshot.objects.count(),
261 "changes": Change.objects.filter_by_day(
262 timezone.now().date() - datetime.timedelta(days=1)
263 ).count(),
264 "contributors": Change.objects.since_day(
265 timezone.now().date() - datetime.timedelta(days=30)
266 ).count_users(),
267 "contributors_total": Change.objects.count_users(),
268 "users": User.objects.count(),
269 }
270 return self.create_metrics(data, stats, SOURCE_KEYS, Metric.SCOPE_GLOBAL, 0)
272 @transaction.atomic
273 def collect_project_language(self, project_language: ProjectLanguage):
274 project = project_language.project
275 changes = project.change_set.filter(
276 translation__language=project_language.language
277 )
279 data = {
280 "changes": changes.filter_by_day(
281 timezone.now().date() - datetime.timedelta(days=1),
282 ).count(),
283 "contributors": changes.since_day(
284 timezone.now().date() - datetime.timedelta(days=30)
285 ).count_users(),
286 "contributors_total": changes.count_users(),
287 }
289 return self.create_metrics(
290 data,
291 project_language.stats,
292 SOURCE_KEYS,
293 Metric.SCOPE_PROJECT_LANGUAGE,
294 project.pk,
295 project_language.language.pk,
296 )
298 @transaction.atomic
299 def collect_category_language(self, category_language: CategoryLanguage):
300 category = category_language.category
301 changes = category.project.change_set.for_category(category).filter(
302 translation__language=category_language.language
303 )
305 data = {
306 "changes": changes.filter_by_day(
307 timezone.now().date() - datetime.timedelta(days=1),
308 ).count(),
309 "contributors": changes.since_day(
310 timezone.now().date() - datetime.timedelta(days=30)
311 ).count_users(),
312 "contributors_total": changes.count_users(),
313 }
315 return self.create_metrics(
316 data,
317 category_language.stats,
318 SOURCE_KEYS,
319 Metric.SCOPE_CATEGORY_LANGUAGE,
320 category.project.pk,
321 category_language.language.pk,
322 )
324 @transaction.atomic
325 def collect_category(self, category: Category):
326 languages = prefetch_stats(
327 [CategoryLanguage(category, language) for language in category.languages]
328 )
329 for category_language in languages:
330 self.collect_category_language(category_language)
331 changes = Change.objects.for_category(category)
332 data = {
333 "components": category.component_set.count(),
334 "translations": Translation.objects.filter(
335 component__category=category
336 ).count(),
337 "changes": changes.filter_by_day(
338 timezone.now().date() - datetime.timedelta(days=1)
339 ).count(),
340 "contributors": changes.since_day(
341 timezone.now().date() - datetime.timedelta(days=30)
342 ).count_users(),
343 "contributors_total": changes.count_users(),
344 }
346 return self.create_metrics(
347 data, category.stats, SOURCE_KEYS, Metric.SCOPE_CATEGORY, category.pk
348 )
350 @transaction.atomic
351 def collect_project(self, project: Project):
352 languages = prefetch_stats(
353 [ProjectLanguage(project, language) for language in project.languages]
354 )
355 for project_language in languages:
356 self.collect_project_language(project_language)
357 data = {
358 "components": project.component_set.count(),
359 "translations": Translation.objects.filter(
360 component__project=project
361 ).count(),
362 "memory": project.memory_set.count(),
363 "screenshots": Screenshot.objects.filter(
364 translation__component__project=project
365 ).count(),
366 "changes": project.change_set.filter_by_day(
367 timezone.now().date() - datetime.timedelta(days=1)
368 ).count(),
369 "contributors": project.change_set.since_day(
370 timezone.now().date() - datetime.timedelta(days=30)
371 ).count_users(),
372 "contributors_total": project.change_set.count_users(),
373 }
374 keys = [
375 f"machinery-accounting:internal:{project.id}",
376 f"machinery-accounting:external:{project.id}",
377 ]
378 for key, value in cache.get_many(keys).items():
379 if ":internal:" in key:
380 data["machinery:internal"] = value
381 else:
382 data["machinery:external"] = value
383 cache.delete_many(keys)
385 return self.create_metrics(
386 data, project.stats, SOURCE_KEYS, Metric.SCOPE_PROJECT, project.pk
387 )
389 @transaction.atomic
390 def collect_component(self, component: Component):
391 data = {
392 "translations": component.translation_set.count(),
393 "screenshots": Screenshot.objects.filter(
394 translation__component=component
395 ).count(),
396 "changes": component.change_set.filter_by_day(
397 timezone.now().date() - datetime.timedelta(days=1)
398 ).count(),
399 "contributors": component.change_set.since_day(
400 timezone.now().date() - datetime.timedelta(days=30)
401 ).count_users(),
402 "contributors_total": component.change_set.count_users(),
403 }
404 return self.create_metrics(
405 data, component.stats, SOURCE_KEYS, Metric.SCOPE_COMPONENT, component.pk
406 )
408 @transaction.atomic
409 def collect_component_list(self, clist: ComponentList):
410 changes = Change.objects.filter(component__in=clist.components.all())
411 data = {
412 "changes": changes.filter_by_day(
413 timezone.now().date() - datetime.timedelta(days=1)
414 ).count(),
415 "contributors": changes.since_day(
416 timezone.now().date() - datetime.timedelta(days=30)
417 ).count_users(),
418 "contributors_total": changes.count_users(),
419 }
420 return self.create_metrics(
421 data,
422 clist.stats,
423 SOURCE_KEYS,
424 Metric.SCOPE_COMPONENT_LIST,
425 clist.pk,
426 )
428 @transaction.atomic
429 def collect_translation(self, translation: Translation):
430 data = {
431 "screenshots": translation.screenshot_set.count(),
432 "changes": translation.change_set.filter_by_day(
433 timezone.now().date() - datetime.timedelta(days=1)
434 ).count(),
435 "contributors": translation.change_set.since_day(
436 timezone.now().date() - datetime.timedelta(days=30)
437 ).count_users(),
438 "contributors_total": translation.change_set.count_users(),
439 }
440 return self.create_metrics(
441 data,
442 translation.stats,
443 BASIC_KEYS,
444 Metric.SCOPE_TRANSLATION,
445 translation.pk,
446 )
448 @transaction.atomic
449 def collect_user(self, user: User):
450 data = user.change_set.filter_by_day(
451 timezone.now().date() - datetime.timedelta(days=1)
452 ).aggregate(
453 changes=Count("id"),
454 comments=Count("id", filter=Q(action=ActionEvents.COMMENT)),
455 suggestions=Count("id", filter=Q(action=ActionEvents.SUGGESTION)),
456 translations=Count("id", filter=Q(action__in=Change.ACTIONS_CONTENT)),
457 screenshots=Count(
458 "id",
459 filter=Q(
460 action__in=(
461 ActionEvents.SCREENSHOT_ADDED,
462 ActionEvents.SCREENSHOT_UPLOADED,
463 )
464 ),
465 ),
466 )
467 return self.create_metrics(data, None, set(), Metric.SCOPE_USER, user.pk)
469 @transaction.atomic
470 def collect_language(self, language: Language):
471 changes = language.change_set.all()
472 data = {
473 "changes": changes.filter_by_day(
474 timezone.now().date() - datetime.timedelta(days=1),
475 ).count(),
476 "contributors": changes.since_day(
477 timezone.now().date() - datetime.timedelta(days=30)
478 ).count_users(),
479 "contributors_total": changes.count_users(),
480 "users": language.profile_set.count(),
481 }
482 return self.create_metrics(
483 data,
484 language.stats,
485 SOURCE_KEYS,
486 Metric.SCOPE_LANGUAGE,
487 language.pk,
488 )
491class Metric(models.Model):
492 SCOPE_GLOBAL = 0
493 SCOPE_PROJECT = 1
494 SCOPE_COMPONENT = 2
495 SCOPE_TRANSLATION = 3
496 SCOPE_USER = 4
497 SCOPE_COMPONENT_LIST = 5
498 SCOPE_PROJECT_LANGUAGE = 6
499 SCOPE_LANGUAGE = 7
500 SCOPE_CATEGORY = 8
501 SCOPE_CATEGORY_LANGUAGE = 9
503 id = models.BigAutoField(primary_key=True)
504 date = models.DateField(default=datetime.date.today)
505 scope = models.SmallIntegerField()
506 relation = models.IntegerField()
507 secondary = models.IntegerField(default=0)
508 changes = models.IntegerField()
509 data = models.JSONField(null=True)
511 objects = MetricManager.from_queryset(MetricQuerySet)()
513 class Meta:
514 unique_together = (("scope", "relation", "secondary", "date"),)
515 verbose_name = "Metric"
516 verbose_name_plural = "Metrics"
518 def __str__(self) -> str:
519 return f"<{self.scope}.{self.relation}>:{self.date}:{self.changes} {self.data}"
521 def __getitem__(self, item: str):
522 return self.dict_data[item]
524 @cached_property
525 def dict_data(self) -> dict:
526 return dict(zip_longest(METRIC_ORDER, self.data or [], fillvalue=0))
528 def get(self, item: str, default=None):
529 return self.dict_data.get(item, default)
532@receiver(post_save, sender=Project)
533@disable_for_loaddata
534def create_metrics_project(sender, instance, created=False, **kwargs) -> None:
535 if created:
536 Metric.objects.initialize_metrics(
537 scope=Metric.SCOPE_PROJECT, relation=instance.pk
538 )
541@receiver(post_save, sender=Category)
542@disable_for_loaddata
543def create_metrics_category(sender, instance, created=False, **kwargs) -> None:
544 if created:
545 Metric.objects.initialize_metrics(
546 scope=Metric.SCOPE_CATEGORY, relation=instance.pk
547 )
550@receiver(post_save, sender=Component)
551@disable_for_loaddata
552def create_metrics_component(sender, instance, created=False, **kwargs) -> None:
553 if created:
554 Metric.objects.initialize_metrics(
555 scope=Metric.SCOPE_COMPONENT, relation=instance.pk
556 )
559@receiver(post_save, sender=Translation)
560@disable_for_loaddata
561def create_metrics_translation(sender, instance, created=False, **kwargs) -> None:
562 if created:
563 Metric.objects.initialize_metrics(
564 scope=Metric.SCOPE_TRANSLATION, relation=instance.pk
565 )
568@receiver(post_save, sender=User)
569@disable_for_loaddata
570def create_metrics_user(sender, instance, created=False, **kwargs) -> None:
571 if created:
572 Metric.objects.initialize_metrics(scope=Metric.SCOPE_USER, relation=instance.pk)
575@receiver(post_delete, sender=Category)
576@disable_for_loaddata
577def delete_metrics_category(sender, instance, **kwargs) -> None:
578 Metric.objects.filter(
579 scope__in=(Metric.SCOPE_CATEGORY_LANGUAGE, Metric.SCOPE_CATEGORY),
580 relation=instance.pk,
581 ).delete()
584@receiver(post_delete, sender=Project)
585@disable_for_loaddata
586def delete_metrics_project(sender, instance, **kwargs) -> None:
587 Metric.objects.filter(
588 scope__in=(Metric.SCOPE_PROJECT_LANGUAGE, Metric.SCOPE_PROJECT),
589 relation=instance.pk,
590 ).delete()
593@receiver(post_delete, sender=Component)
594@disable_for_loaddata
595def delete_metrics_component(sender, instance, **kwargs) -> None:
596 Metric.objects.filter(scope=Metric.SCOPE_COMPONENT, relation=instance.pk).delete()
599@receiver(post_delete, sender=ComponentList)
600@disable_for_loaddata
601def delete_metrics_component_list(sender, instance, **kwargs) -> None:
602 Metric.objects.filter(
603 scope=Metric.SCOPE_COMPONENT_LIST, relation=instance.pk
604 ).delete()
607@receiver(post_delete, sender=Translation)
608@disable_for_loaddata
609def delete_metrics_translation(sender, instance, **kwargs) -> None:
610 Metric.objects.filter(scope=Metric.SCOPE_TRANSLATION, relation=instance.pk).delete()
613@receiver(post_delete, sender=User)
614@disable_for_loaddata
615def delete_metrics_user(sender, instance, **kwargs) -> None:
616 Metric.objects.filter(scope=Metric.SCOPE_USER, relation=instance.pk).delete()
619@receiver(post_delete, sender=Language)
620@disable_for_loaddata
621def delete_metrics_language(sender, instance, **kwargs) -> None:
622 Metric.objects.filter(scope=Metric.SCOPE_LANGUAGE, relation=instance.pk).delete()