Coverage for app/venv/lib/python3.14/site-packages/weblate/utils/tasks.py: 40%

1# Copyright © Michal Čihař <michal@weblate.org> 

2# 

3# SPDX-License-Identifier: GPL-3.0-or-later 

4 

5from __future__ import annotations 

6 

7import gzip 

8import os 

9import shutil 

10import subprocess 

11import sys 

12import time 

13from importlib import import_module 

14from shutil import copyfile 

15from typing import cast 

16 

17from celery.schedules import crontab 

18from django.conf import settings 

19from django.core.cache import cache 

20from django.core.management.commands import diffsettings 

21from ruamel.yaml import YAML 

22 

23import weblate.utils.version 

24from weblate.formats.models import FILE_FORMATS 

25from weblate.logger import LOGGER 

26from weblate.machinery.models import MACHINERY 

27from weblate.trans.models import Component, Project, Translation 

28from weblate.trans.util import get_clean_env 

29from weblate.utils.backup import backup_lock 

30from weblate.utils.celery import app 

31from weblate.utils.data import data_dir 

32from weblate.utils.db import using_postgresql 

33from weblate.utils.errors import add_breadcrumb, report_error 

34from weblate.utils.lock import WeblateLockTimeoutError 

35from weblate.vcs.models import VCS_REGISTRY 

36 

37from .const import HEARTBEAT_FREQUENCY 

38 

39 

40@app.task(trail=False) 

41def ping(): 

42 return { 

43 "version": weblate.utils.version.GIT_VERSION, 

44 "vcs": sorted(VCS_REGISTRY.keys()), 

45 "formats": sorted(FILE_FORMATS.keys()), 

46 "mt_services": sorted(MACHINERY.keys()), 

47 "encoding": [sys.getfilesystemencoding(), sys.getdefaultencoding()], 

48 "uid": os.getuid(), 

49 "data_dir": settings.DATA_DIR, 

50 } 

51 

52 

53@app.task(trail=False) 

54def heartbeat() -> None: 

55 cache.set("celery_loaded", time.time()) 

56 cache.set("celery_heartbeat", time.time()) 

57 cache.set( 

58 "celery_encoding", [sys.getfilesystemencoding(), sys.getdefaultencoding()] 

59 ) 

60 

61 

62@app.task(trail=False, autoretry_for=(WeblateLockTimeoutError,)) 

63def settings_backup() -> None: 

64 with backup_lock(): 

65 # Expand settings in case it contains non-trivial code 

66 command = diffsettings.Command() 

67 kwargs = {"default": None, "all": False, "output": "hash"} 

68 with open(data_dir("backups", "settings-expanded.py"), "w") as handle: 

69 handle.write(command.handle(**kwargs)) 

70 

71 # Backup original settings 

72 if settings.SETTINGS_MODULE: 

73 settings_mod = import_module(settings.SETTINGS_MODULE) 

74 if settings_mod.__file__ is not None: 

75 copyfile(settings_mod.__file__, data_dir("backups", "settings.py")) 

76 

77 # Backup environment (to make restoring Docker easier) 

78 with open(data_dir("backups", "environment.yml"), "w") as handle: 

79 yaml = YAML() 

80 yaml.dump(dict(os.environ), handle) 

81 

82 

83@app.task(trail=False) 

84def update_translation_stats_parents(pk: int) -> None: 

85 translation = Translation.objects.get(pk=pk) 

86 translation.stats.update_parents() 

87 

88 

89@app.task(trail=False) 

90def update_language_stats_parents(pk: int) -> None: 

91 component = Component.objects.get(pk=pk) 

92 component.stats.update_language_stats_parents() 

93 

94 

95@app.task(trail=False) 

96def update_project_stats_link(pk: int) -> None: 

97 project = Project.objects.get(pk=pk) 

98 for language in project.stats.get_language_stats(): 

99 language.update_stats(update_parents=False) 

100 project.stats.update_stats() 

101 

102 

103@app.task(trail=False, autoretry_for=(WeblateLockTimeoutError,)) 

104def database_backup() -> None: 

105 if settings.DATABASE_BACKUP == "none": 

106 return 

107 with backup_lock(): 

108 database = settings.DATABASES["default"] 

109 env = get_clean_env() 

110 compress = settings.DATABASE_BACKUP == "compressed" 

111 

112 out_compressed = data_dir("backups", "database.sql.gz") 

113 out_text = data_dir("backups", "database.sql") 

114 

115 if using_postgresql(): 

116 cmd = [ 

117 "pg_dump", 

118 # Superuser only, crashes on Alibaba Cloud Database PolarDB 

119 "--no-subscriptions", 

120 "--clean", 

121 "--if-exists", 

122 "--dbname", 

123 database["NAME"], 

124 ] 

125 

126 if database["HOST"]: 

127 cmd.extend(["--host", database["HOST"]]) 

128 if database["PORT"]: 

129 cmd.extend(["--port", database["PORT"]]) 

130 if database["USER"]: 

131 cmd.extend(["--username", database["USER"]]) 

132 if settings.DATABASE_BACKUP == "compressed": 

133 cmd.extend(["--file", out_compressed]) 

134 cmd.extend(["--compress", "6"]) 

135 compress = False 

136 else: 

137 cmd.extend(["--file", out_text]) 

138 

139 env["PGPASSWORD"] = cast("str", database["PASSWORD"]) 

140 else: 

141 cmd = [ 

142 "mysqldump", 

143 "--result-file", 

144 out_text, 

145 "--single-transaction", 

146 "--skip-lock-tables", 

147 ] 

148 

149 if database["HOST"]: 

150 cmd.extend(["--host", database["HOST"]]) 

151 if database["PORT"]: 

152 cmd.extend(["--port", database["PORT"]]) 

153 if database["USER"]: 

154 cmd.extend(["--user", database["USER"]]) 

155 

156 cmd.extend(["--databases", database["NAME"]]) 

157 

158 env["MYSQL_PWD"] = cast("str", database["PASSWORD"]) 

159 

160 try: 

161 subprocess.run( 

162 cmd, # type: ignore[arg-type] 

163 env=env, 

164 capture_output=True, 

165 stdin=subprocess.DEVNULL, 

166 check=True, 

167 text=True, 

168 ) 

169 except subprocess.CalledProcessError as error: 

170 add_breadcrumb( 

171 category="backup", 

172 message="database dump output", 

173 stdout=error.stdout, 

174 stderr=error.stderr, 

175 ) 

176 LOGGER.error("failed database backup: %s", error.stderr) 

177 report_error("Database backup failed") 

178 raise 

179 

180 if compress: 

181 with open(out_text, "rb") as f_in, gzip.open(out_compressed, "wb") as f_out: 

182 shutil.copyfileobj(f_in, f_out) 

183 os.unlink(out_text) 

184 

185 

186@app.on_after_finalize.connect 

187def setup_periodic_tasks(sender, **kwargs) -> None: 

188 cache.set("celery_loaded", time.time()) 

189 sender.add_periodic_task( 

190 crontab(hour=1, minute=0), settings_backup.s(), name="settings-backup" 

191 ) 

192 sender.add_periodic_task( 

193 crontab(hour=1, minute=30), database_backup.s(), name="database-backup" 

194 ) 

195 sender.add_periodic_task(HEARTBEAT_FREQUENCY, heartbeat.s(), name="heartbeat")