Coverage for app/venv/lib/python3.14/site-packages/weblate/trans/management/commands/import_project.py: 0%

1# Copyright © Michal Čihař <michal@weblate.org> 

2# 

3# SPDX-License-Identifier: GPL-3.0-or-later 

4from __future__ import annotations 

5 

6import os 

7import re 

8import tempfile 

9from typing import TYPE_CHECKING 

10 

11from django.conf import settings 

12from django.core.management.base import CommandError 

13 

14from weblate.formats.models import FILE_FORMATS 

15from weblate.lang.models import Language 

16from weblate.logger import LOGGER 

17from weblate.trans.discovery import ComponentDiscovery 

18from weblate.trans.models import Component, Project 

19from weblate.trans.util import is_repo_link 

20from weblate.utils.files import remove_tree 

21from weblate.utils.management.base import BaseCommand 

22from weblate.vcs.base import RepositoryError 

23from weblate.vcs.models import VCS_REGISTRY 

24 

25if TYPE_CHECKING: 

26 from django.core.management.base import CommandParser 

27 

28 

29class Command(BaseCommand): 

30 """Command for mass importing of repositories into Weblate.""" 

31 

32 help = "imports projects with more components" 

33 

34 def add_arguments(self, parser: CommandParser) -> None: 

35 super().add_arguments(parser) 

36 parser.add_argument( 

37 "--name-template", 

38 default="{{ component }}", 

39 help="Template string, transforming the file mask match to a project name", 

40 ) 

41 parser.add_argument( 

42 "--base-file-template", 

43 default="", 

44 help=( 

45 "Template string, transforming the file mask " 

46 "match to a monolingual base filename" 

47 ), 

48 ) 

49 parser.add_argument( 

50 "--new-base-template", 

51 default="", 

52 help=( 

53 "Template string, transforming the file mask " 

54 "match to a base filename for new translations" 

55 ), 

56 ) 

57 parser.add_argument( 

58 "--file-format", 

59 default="po", 

60 help="File format type, defaults to Gettext PO", 

61 ) 

62 parser.add_argument( 

63 "--language-regex", 

64 default="^[^.]+$", 

65 help=( 

66 "Language filter regular expression to be used for created components" 

67 ), 

68 ) 

69 parser.add_argument( 

70 "--license", default="", help="License of imported components" 

71 ) 

72 parser.add_argument( 

73 "--vcs", default=settings.DEFAULT_VCS, help="Version control system to use" 

74 ) 

75 parser.add_argument( 

76 "--push-url", default="", help="Set push URL for the project" 

77 ) 

78 parser.add_argument( 

79 "--push-url-same", 

80 action="store_true", 

81 default=False, 

82 help="Set push URL for the project to same as pull", 

83 ) 

84 parser.add_argument( 

85 "--disable-push-on-commit", 

86 action="store_false", 

87 default=settings.DEFAULT_PUSH_ON_COMMIT, 

88 dest="push_on_commit", 

89 help="Disable push on commit for created components", 

90 ) 

91 parser.add_argument( 

92 "--push-on-commit", 

93 action="store_true", 

94 default=settings.DEFAULT_PUSH_ON_COMMIT, 

95 dest="push_on_commit", 

96 help="Enable push on commit for created components", 

97 ) 

98 parser.add_argument( 

99 "--main-component", 

100 default=None, 

101 help=( 

102 "Define which component will be used as main - including full" 

103 " VCS repository" 

104 ), 

105 ) 

106 parser.add_argument( 

107 "--source-language", 

108 default=settings.DEFAULT_LANGUAGE, 

109 help="Source language code", 

110 ) 

111 parser.add_argument("project", help="Existing project slug") 

112 parser.add_argument("repo", help="VCS repository URL") 

113 parser.add_argument("branch", help="VCS repository branch") 

114 parser.add_argument("filemask", help="File mask") 

115 

116 def __init__(self, *args, **kwargs) -> None: 

117 super().__init__(*args, **kwargs) 

118 self.filemask: str 

119 self.file_format: str 

120 self.language_regex: str 

121 self.license: str 

122 self.main_component: str | None = None 

123 self.name_template: str 

124 self.source_language: str 

125 self.base_file_template: str 

126 self.new_base_template: str 

127 self.vcs: str 

128 self.push_url: str 

129 self.discovery: ComponentDiscovery | None = None 

130 self.logger = LOGGER 

131 self.push_on_commit = True 

132 

133 def checkout_tmp(self, project, repo, branch): 

134 """Checkout project to temporary location.""" 

135 # Create temporary working dir 

136 workdir = tempfile.mkdtemp(dir=project.full_path) 

137 # Make the temporary directory readable by others 

138 os.chmod(workdir, 0o755) # noqa: S103, nosec 

139 

140 # Initialize git repository 

141 self.logger.info("Cloning git repository...") 

142 try: 

143 gitrepo = VCS_REGISTRY[self.vcs].clone(repo, workdir, branch) 

144 except RepositoryError as error: 

145 msg = f"Failed clone: {error}" 

146 raise CommandError(msg) from error 

147 self.logger.info("Updating working copy in git repository...") 

148 with gitrepo.lock: 

149 gitrepo.configure_branch(branch) 

150 

151 return workdir 

152 

153 def parse_options(self, repo, options) -> None: 

154 """Parse parameters.""" 

155 self.filemask = options["filemask"] 

156 self.vcs = options["vcs"] 

157 if options["push_url_same"]: 

158 self.push_url = repo 

159 else: 

160 self.push_url = options["push_url"] 

161 self.file_format = options["file_format"] 

162 self.language_regex = options["language_regex"] 

163 self.main_component = options["main_component"] 

164 self.name_template = options["name_template"] 

165 self.source_language = Language.objects.get(code=options["source_language"]) 

166 if "%s" in self.name_template: 

167 self.name_template = self.name_template.replace("%s", "{{ component }}") 

168 self.license = options["license"] 

169 self.push_on_commit = options["push_on_commit"] 

170 self.base_file_template = options["base_file_template"] 

171 self.new_base_template = options["new_base_template"] 

172 if "%s" in self.base_file_template: 

173 self.base_file_template = self.base_file_template.replace( 

174 "%s", "{{ component }}" 

175 ) 

176 

177 # Is file format supported? 

178 if self.file_format not in FILE_FORMATS: 

179 msg = "Invalid file format: {}".format(options["file_format"]) 

180 raise CommandError(msg) 

181 

182 # Is vcs supported? 

183 if self.vcs not in VCS_REGISTRY: 

184 msg = "Invalid vcs: {}".format(options["vcs"]) 

185 raise CommandError(msg) 

186 

187 # Do we have correct mask? 

188 # - if there is **, then it's simple mask (it's invalid in regexp) 

189 # - validate regexp otherwise 

190 if "**" in self.filemask and "*" in self.filemask.replace("**", ""): 

191 match = re.escape(self.filemask) 

192 match = match.replace(r"\*\*", "(?P<component>[[WILDCARD]])", 1) 

193 match = match.replace(r"\*\*", "(?P=component)") 

194 match = match.replace(r"\*", "(?P<language>[[WILDCARD]])", 1) 

195 match = match.replace(r"\*", "(?P=language)") 

196 match = match.replace("[[WILDCARD]]", "[^/]*") 

197 self.filemask = match 

198 else: 

199 try: 

200 compiled = re.compile(self.filemask) 

201 except re.error as error: 

202 msg = f"Could not compile regular expression {self.filemask!r}: {error}" 

203 raise CommandError(msg) from error 

204 if ( 

205 "component" not in compiled.groupindex 

206 or "language" not in compiled.groupindex 

207 ): 

208 msg = ( 

209 "Component regular expression lacks named group " 

210 '"component" and/or "language"' 

211 ) 

212 raise CommandError(msg) 

213 

214 def handle(self, *args, **options) -> None: 

215 """Automatic import of project.""" 

216 # Read params 

217 repo = options["repo"] 

218 branch = options["branch"] 

219 self.parse_options(repo, options) 

220 

221 # Try to get project 

222 try: 

223 project = Project.objects.get(slug=options["project"]) 

224 except Project.DoesNotExist as error: 

225 msg = 'Project "{}" not found, please create it first!'.format( 

226 options["project"] 

227 ) 

228 raise CommandError(msg) from error 

229 

230 # Get or create main component 

231 if is_repo_link(repo): 

232 try: 

233 component = Component.objects.get_linked(repo) 

234 # Avoid operating on link 

235 if component.is_repo_link: 

236 component = component.linked_component 

237 except Component.DoesNotExist as error: 

238 msg = f"Component {repo!r} not found, please create it first!" 

239 raise CommandError(msg) from error 

240 else: 

241 component = self.import_initial(project, repo, branch) 

242 

243 discovery = self.get_discovery(component) 

244 discovery.perform() 

245 

246 def get_discovery(self, component, path=None): 

247 """Return discovery object after doing basic sanity check.""" 

248 if self.discovery is not None: 

249 self.discovery.component = component 

250 else: 

251 self.discovery = ComponentDiscovery( 

252 component, 

253 match=self.filemask, 

254 name_template=self.name_template, 

255 language_regex=self.language_regex, 

256 base_file_template=self.base_file_template, 

257 new_base_template=self.new_base_template, 

258 file_format=self.file_format, 

259 path=path, 

260 ) 

261 self.logger.info( 

262 "Found %d matching files", len(self.discovery.matched_files) 

263 ) 

264 

265 if not self.discovery.matched_files: 

266 msg = "Your mask did not match any files!" 

267 raise CommandError(msg) 

268 

269 self.logger.info( 

270 "Found %d components", len(self.discovery.matched_components) 

271 ) 

272 langs = set() 

273 for match in self.discovery.matched_components.values(): 

274 langs.update(match["languages"]) 

275 self.logger.info("Found %d languages", len(langs)) 

276 

277 # Do some basic sanity check on languages 

278 if not Language.objects.filter(code__in=langs).exists(): 

279 msg = ( 

280 "None of matched languages exists, maybe you have " 

281 "mixed * and ** in the mask?" 

282 ) 

283 raise CommandError(msg) 

284 return self.discovery 

285 

286 def import_initial(self, project, repo, branch): 

287 """Import the first repository of a project.""" 

288 # Checkout git to temporary dir 

289 workdir = self.checkout_tmp(project, repo, branch) 

290 # Create fake discovery without existing component 

291 discovery = self.get_discovery(None, workdir) 

292 

293 components = project.component_set.all() 

294 

295 component = None 

296 

297 # Create first component (this one will get full git repo) 

298 if self.main_component: 

299 match = None 

300 for match in discovery.matched_components.values(): 

301 if match["slug"] == self.main_component: 

302 break 

303 if match is None or match["slug"] != self.main_component: 

304 msg = "Specified --main-component was not found in matches!" 

305 raise CommandError(msg) 

306 else: 

307 # Try if one is already there 

308 for match in discovery.matched_components.values(): 

309 try: 

310 component = components.get(repo=repo, filemask=match["mask"]) 

311 except Component.DoesNotExist: 

312 continue 

313 # Pick random 

314 if component is None: 

315 match = next(iter(discovery.matched_components.values())) 

316 

317 try: 

318 if component is None: 

319 component = components.get(slug=match["slug"]) 

320 self.logger.warning( 

321 "Component %s already exists, skipping and using it " 

322 "as a main component", 

323 match["slug"], 

324 ) 

325 remove_tree(workdir) 

326 except Component.DoesNotExist: 

327 self.logger.info("Creating component %s as main one", match["slug"]) 

328 

329 # Rename gitrepository to new name 

330 os.rename(workdir, os.path.join(project.full_path, match["slug"])) 

331 

332 # Create new component 

333 component = discovery.create_component( 

334 None, 

335 match, 

336 existing_slugs=set(), 

337 existing_names=set(), 

338 project=project, 

339 source_language=self.source_language, 

340 repo=repo, 

341 branch=branch, 

342 vcs=self.vcs, 

343 push_on_commit=self.push_on_commit, 

344 license=self.license, 

345 ) 

346 

347 return component