Coverage for app/venv/lib/python3.14/site-packages/weblate/formats/convert.py: 36%

1# Copyright © Michal Čihař <michal@weblate.org> 

2# 

3# SPDX-License-Identifier: GPL-3.0-or-later 

4 

5"""Translate Toolkit converter based file format wrappers.""" 

6 

7from __future__ import annotations 

8 

9import codecs 

10import os 

11import shutil 

12from collections import defaultdict 

13from io import BytesIO 

14from typing import TYPE_CHECKING, Any, BinaryIO, ClassVar, NoReturn 

15from zipfile import ZipFile 

16 

17from django.utils.functional import cached_property 

18from django.utils.translation import gettext_lazy 

19from translate.convert.po2html import po2html 

20from translate.convert.po2idml import translate_idml, write_idml 

21from translate.convert.po2rc import rerc 

22from translate.convert.po2txt import po2txt 

23from translate.convert.rc2po import rc2po 

24from translate.convert.xliff2odf import translate_odf, write_odf 

25from translate.storage.html import htmlfile 

26from translate.storage.idml import INLINE_ELEMENTS, NO_TRANSLATE_ELEMENTS, open_idml 

27from translate.storage.odf_io import open_odf 

28from translate.storage.odf_shared import inline_elements, no_translate_content_elements 

29from translate.storage.pypo import pofile 

30from translate.storage.rc import rcfile 

31from translate.storage.txt import TxtFile 

32from translate.storage.xliff import xlifffile 

33from translate.storage.xml_extract.extract import ( 

34 IdMaker, 

35 ParseState, 

36 build_idml_store, 

37 build_store, 

38 make_postore_adder, 

39) 

40 

41from weblate.formats.base import ( 

42 TranslationFormat, 

43) 

44from weblate.formats.helpers import NamedBytesIO 

45from weblate.formats.ttkit import PoUnit, XliffUnit 

46from weblate.trans.util import get_string 

47from weblate.utils.errors import report_error 

48from weblate.utils.state import STATE_APPROVED 

49 

50if TYPE_CHECKING: 50 ↛ 51line 50 didn't jump to line 51 because the condition on line 50 was never true

51 from collections.abc import Callable 

52 

53 from translate.storage.base import TranslationStore 

54 from translate.storage.base import TranslationUnit as TranslateToolkitUnit 

55 

56 from weblate.formats.base import ( 

57 TranslationUnit, 

58 ) 

59 from weblate.trans.models import Unit 

60 

61 

62class ConvertPoUnit(PoUnit): 

63 id_hash_with_source: bool = True 

64 

65 def is_translated(self): 

66 """Check whether unit is translated.""" 

67 if self.parent.is_template: 

68 return self.has_translation() 

69 return self.unit is not None and self.has_translation() 

70 

71 def is_fuzzy(self, fallback=False): 

72 """Check whether unit needs editing.""" 

73 return fallback 

74 

75 def is_approved(self, fallback=False): 

76 """Check whether unit is approved.""" 

77 return fallback 

78 

79 @cached_property 

80 def source(self): 

81 """Return source string from a Translate Toolkit unit.""" 

82 if self.template: 

83 return get_string(self.template.source) 

84 return get_string(self.unit.source) 

85 

86 

87class ConvertXliffUnit(XliffUnit): 

88 def is_fuzzy(self, fallback=False): 

89 """Check whether unit needs editing.""" 

90 return fallback 

91 

92 def is_approved(self, fallback=False): 

93 """Check whether unit is approved.""" 

94 return fallback 

95 

96 def is_translated(self): 

97 """Check whether unit is translated.""" 

98 if self.parent.is_template: 

99 return self.has_translation() 

100 return self.unit is not None 

101 

102 @cached_property 

103 def flags(self): 

104 flags = super().flags 

105 flags.remove("xml-text") 

106 return flags 

107 

108 

109class ConvertFormat(TranslationFormat): 

110 """ 

111 Base class for convert based formats. 

112 

113 This always uses intermediate representation. 

114 """ 

115 

116 monolingual = True 

117 can_add_unit = False 

118 can_delete_unit = False 

119 can_edit_base: bool = False 

120 unit_class: type[TranslationUnit] = ConvertPoUnit 

121 autoaddon: ClassVar[dict[str, dict[str, Any]]] = { 

122 "weblate.flags.same_edit": {}, 

123 "weblate.cleanup.generic": {}, 

124 } 

125 create_style = "copy" 

126 units: list[TranslateToolkitUnit] 

127 store: TranslationStore 

128 

129 def save_content(self, handle) -> None: 

130 """Store content to file.""" 

131 raise NotImplementedError 

132 

133 def save(self) -> None: 

134 """Save underlying store to disk.""" 

135 self.save_atomic(self.storefile, self.save_content) 

136 

137 def convertfile(self, storefile, template_store) -> NoReturn: 

138 raise NotImplementedError 

139 

140 @staticmethod 

141 def needs_target_sync(template_store) -> bool: # noqa: ARG004 

142 return False 

143 

144 def load( 

145 self, 

146 storefile: str | BinaryIO, 

147 template_store: TranslationFormat | None, 

148 ) -> TranslationStore: 

149 # Did we get file or filename? 

150 if not hasattr(storefile, "read"): 

151 with open(storefile, "rb") as handle: 

152 store = self.convertfile(handle, template_store) 

153 else: 

154 store = self.convertfile(storefile, template_store) 

155 # Adjust store to have translations 

156 if self.needs_target_sync(template_store): 

157 for unit in store.units: 

158 if unit.isheader(): 

159 continue 

160 # HTML does this properly on loading, others need it 

161 unit.target = unit.source 

162 unit.rich_target = unit.rich_source 

163 return store 

164 

165 @classmethod 

166 def create_new_file( 

167 cls, 

168 filename: str, 

169 language: str, # noqa: ARG003 

170 base: str, 

171 callback: Callable | None = None, # noqa: ARG003 

172 file_format_params: dict[str, Any] | None = None, # noqa: ARG003 

173 ) -> None: 

174 """Handle creation of new translation file.""" 

175 if not base: 

176 msg = "Not supported" 

177 raise ValueError(msg) 

178 # Copy file 

179 shutil.copy(base, filename) 

180 

181 @classmethod 

182 def is_valid_base_for_new( 

183 cls, 

184 base: str, 

185 monolingual: bool, # noqa: ARG003 

186 errors: list[Exception] | None = None, 

187 fast: bool = False, 

188 file_format_params: dict[str, Any] | None = None, # noqa: ARG003 

189 ) -> bool: 

190 """Check whether base is valid.""" 

191 if not base: 

192 return False 

193 try: 

194 if not fast: 

195 cls(base, None) 

196 except Exception as exception: 

197 if errors is not None: 

198 errors.append(exception) 

199 report_error("File-parsing error") 

200 return False 

201 return True 

202 

203 def add_unit(self, unit: TranslationUnit) -> None: 

204 self.store.addunit(unit.unit) 

205 

206 @classmethod 

207 def get_class(cls) -> None: 

208 return None 

209 

210 def create_unit( 

211 self, 

212 key: str, 

213 source: str | list[str], 

214 target: str | list[str] | None = None, 

215 ) -> NoReturn: 

216 msg = "Not supported" 

217 raise ValueError(msg) 

218 

219 def cleanup_unused(self) -> list[str]: 

220 """ 

221 Bring target in sync with the source. 

222 

223 This is done automatically on save as it reshapes translations 

224 based on the template. 

225 """ 

226 self.save() 

227 return [] 

228 

229 def convert_to_po(self, parser, template_store, use_location: bool = True): 

230 store = pofile() 

231 # Prepare index of existing translations 

232 unitindex: dict[str, list[Unit]] = defaultdict(list) 

233 for existing_unit in self.existing_units: 

234 for source in existing_unit.get_source_plurals(): 

235 unitindex[source].append(existing_unit) 

236 

237 # Convert store 

238 if template_store: 

239 parser.makeindex() 

240 for unit in template_store.content_units: 

241 thepo = store.addsourceunit(unit.source) 

242 locations = unit.unit.getlocations() 

243 thepo.addlocations(locations) 

244 thepo.addnote(unit.unit.getnotes(), "developer") 

245 if self.is_template: 

246 thepo.target = unit.source 

247 elif use_location and not unitindex: 

248 # Try to import initial translation from the file 

249 for location in locations: 

250 try: 

251 translation = parser.locationindex[location] 

252 thepo.target = translation.source 

253 break 

254 except KeyError: 

255 continue 

256 else: 

257 for htmlunit in parser.units: 

258 # Source file 

259 thepo = store.addsourceunit(htmlunit.source) 

260 thepo.target = htmlunit.source 

261 thepo.addlocations(htmlunit.getlocations()) 

262 thepo.addnote(htmlunit.getnotes(), "developer") 

263 

264 # Handle duplicate strings (use context to differentiate them) 

265 store.removeduplicates("msgctxt") 

266 

267 # Merge existing translations 

268 if unitindex and not self.is_template: 

269 for unit in store.units: 

270 possible_translations = unitindex[unit.source] 

271 # Single match 

272 if len(possible_translations) == 1: 

273 unit.target = possible_translations[0].target 

274 continue 

275 # None match 

276 if not possible_translations: 

277 continue 

278 # Multiple matches 

279 for translation in possible_translations: 

280 if translation.context == unit.getcontext(): 

281 unit.target = translation.target 

282 break 

283 

284 return store 

285 

286 

287class HTMLFormat(ConvertFormat): 

288 # Translators: File format name 

289 name = gettext_lazy("HTML file") 

290 autoload = ("*.htm", "*.html") 

291 format_id = "html" 

292 check_flags = ("safe-html", "strict-same") 

293 

294 def convertfile(self, storefile, template_store): 

295 # Fake input file with a blank filename 

296 htmlparser = htmlfile(inputfile=NamedBytesIO("", storefile.read())) 

297 return self.convert_to_po(htmlparser, template_store) 

298 

299 def save_content(self, handle) -> None: 

300 """Store content to file.""" 

301 converter = po2html() 

302 templatename = self.template_store.storefile 

303 if hasattr(templatename, "name"): 

304 templatename = templatename.name 

305 with open(templatename, "rb") as templatefile: 

306 outputstring = converter.mergestore( 

307 self.store, templatefile, includefuzzy=True 

308 ) 

309 handle.write(outputstring.encode("utf-8")) 

310 

311 @staticmethod 

312 def mimetype() -> str: 

313 """Return most common mime type for format.""" 

314 return "text/html" 

315 

316 @staticmethod 

317 def extension() -> str: 

318 """Return most common file extension for format.""" 

319 return "html" 

320 

321 

322class MarkdownFormat(ConvertFormat): 

323 # Translators: File format name 

324 name = gettext_lazy("Markdown file") 

325 autoload = ("*.md", "*.markdown") 

326 format_id = "markdown" 

327 check_flags = ("safe-html", "strict-same", "md-text") 

328 

329 def convertfile(self, storefile, template_store): 

330 # Lazy import as mistletoe is expensive 

331 from translate.storage.markdown import MarkdownFile 

332 

333 # Fake input file with a blank filename 

334 mdparser = MarkdownFile(inputfile=NamedBytesIO("", storefile.read())) 

335 return self.convert_to_po(mdparser, template_store, use_location=False) 

336 

337 def save_content(self, handle) -> None: 

338 """Store content to file.""" 

339 # Lazy import as mistletoe is expensive 

340 from translate.convert.po2md import MarkdownTranslator 

341 

342 converter = MarkdownTranslator( 

343 inputstore=self.store, includefuzzy=True, outputthreshold=None, maxlength=80 

344 ) 

345 templatename = self.template_store.storefile 

346 if hasattr(templatename, "name"): 

347 templatename = templatename.name 

348 with open(templatename, "rb") as templatefile: 

349 converter.translate(templatefile, handle) 

350 

351 @staticmethod 

352 def mimetype() -> str: 

353 """Return most common mime type for format.""" 

354 return "text/markdown" 

355 

356 @staticmethod 

357 def extension() -> str: 

358 """Return most common file extension for format.""" 

359 return "md" 

360 

361 

362class OpenDocumentFormat(ConvertFormat): 

363 # Translators: File format name 

364 name = gettext_lazy("OpenDocument file") 

365 autoload = ( 

366 "*.sxw", 

367 "*.odt", 

368 "*.ods", 

369 "*.odp", 

370 "*.odg", 

371 "*.odc", 

372 "*.odf", 

373 "*.odi", 

374 "*.odm", 

375 "*.ott", 

376 "*.ots", 

377 "*.otp", 

378 "*.otg", 

379 "*.otc", 

380 "*.otf", 

381 "*.oti", 

382 "*.oth", 

383 ) 

384 format_id = "odf" 

385 check_flags = ("strict-same",) 

386 unit_class = ConvertXliffUnit 

387 

388 @staticmethod 

389 def convertfile(storefile, template_store): # noqa: ARG004 

390 store = xlifffile() 

391 store.setfilename(store.getfilenode("NoName"), "odf") 

392 contents = open_odf(storefile) 

393 for data in contents.values(): 

394 parse_state = ParseState(no_translate_content_elements, inline_elements) 

395 build_store(BytesIO(data), store, parse_state) 

396 return store 

397 

398 def save_content(self, handle) -> None: 

399 """Store content to file.""" 

400 templatename = self.template_store.storefile 

401 if hasattr(templatename, "name"): 

402 templatename = templatename.name 

403 # This is workaround for weird fuzzy handling in translate-toolkit 

404 for unit in self.all_units: 

405 if any(state == "translated" for state in unit.get_xliff_states()): 

406 unit.set_state(STATE_APPROVED) 

407 

408 with open(templatename, "rb") as templatefile: 

409 dom_trees = translate_odf(templatefile, self.store) 

410 write_odf(templatefile, handle, dom_trees) 

411 

412 @staticmethod 

413 def mimetype() -> str: 

414 """Return most common mime type for format.""" 

415 return "application/vnd.oasis.opendocument.text" 

416 

417 @staticmethod 

418 def extension() -> str: 

419 """Return most common file extension for format.""" 

420 return "odt" 

421 

422 @staticmethod 

423 def needs_target_sync(template_store) -> bool: # noqa: ARG004 

424 return True 

425 

426 

427class IDMLFormat(ConvertFormat): 

428 # Translators: File format name 

429 name = gettext_lazy("IDML file") 

430 autoload = ("*.idml", "*.idms") 

431 format_id = "idml" 

432 check_flags = ("strict-same",) 

433 

434 @staticmethod 

435 def convertfile(storefile, template_store): # noqa: ARG004 

436 store = pofile() 

437 

438 contents = open_idml(storefile) 

439 

440 # Create it here to avoid having repeated ids. 

441 id_maker = IdMaker() 

442 

443 for filename, translatable_file in contents.items(): 

444 parse_state = ParseState(NO_TRANSLATE_ELEMENTS, INLINE_ELEMENTS) 

445 po_store_adder = make_postore_adder(store, id_maker, filename) 

446 build_idml_store( 

447 BytesIO(translatable_file), 

448 store, 

449 parse_state, 

450 store_adder=po_store_adder, 

451 ) 

452 

453 return store 

454 

455 def save_content(self, handle) -> None: 

456 """Store content to file.""" 

457 templatename = self.template_store.storefile 

458 if hasattr(templatename, "name"): 

459 templatename = templatename.name 

460 with ZipFile(templatename, "r") as template_zip: 

461 translatable_files = [ 

462 filename 

463 for filename in template_zip.namelist() 

464 if filename.startswith("Stories/") 

465 ] 

466 

467 dom_trees = translate_idml(templatename, self.store, translatable_files) 

468 

469 write_idml(template_zip, handle, dom_trees) 

470 

471 @staticmethod 

472 def mimetype() -> str: 

473 """Return most common mime type for format.""" 

474 return "application/octet-stream" 

475 

476 @staticmethod 

477 def extension() -> str: 

478 """Return most common file extension for format.""" 

479 return "idml" 

480 

481 @staticmethod 

482 def needs_target_sync(template_store) -> bool: # noqa: ARG004 

483 return True 

484 

485 

486class WindowsRCFormat(ConvertFormat): 

487 # Translators: File format name 

488 name = gettext_lazy("RC file") 

489 format_id = "rc" 

490 autoload = ("*.rc",) 

491 language_format = "bcp" 

492 

493 @staticmethod 

494 def needs_target_sync(template_store): 

495 return template_store is None 

496 

497 @staticmethod 

498 def mimetype() -> str: 

499 """Return most common media type for format.""" 

500 return "text/plain" 

501 

502 @staticmethod 

503 def extension() -> str: 

504 """Return most common file extension for format.""" 

505 return "rc" 

506 

507 def convertfile(self, storefile, template_store): 

508 input_store = rcfile() 

509 input_store.parse(storefile.read()) 

510 converter = rc2po() 

511 if template_store: 

512 store = converter.merge_store(template_store.store.rcfile, input_store) 

513 else: 

514 store = converter.convert_store(input_store) 

515 store.rcfile = input_store 

516 return store 

517 

518 def save_content(self, handle) -> None: 

519 """Store content to file.""" 

520 # Fallback language 

521 lang = "LANG_ENGLISH" 

522 sublang = "SUBLANG_DEFAULT" 

523 

524 # Keep existing language tags 

525 storage = self.store.rcfile 

526 if storage.lang: 

527 lang = storage.lang 

528 if storage.sublang: 

529 sublang = storage.sublang 

530 

531 templatename = self.template_store.storefile 

532 if hasattr(templatename, "name"): 

533 templatename = templatename.name 

534 encoding = "utf-8" 

535 with open(templatename, "rb") as templatefile: 

536 bom = templatefile.read(2) 

537 if bom == codecs.BOM_UTF16_LE or b"\000" in bom: 

538 encoding = "utf-16-le" 

539 templatefile.seek(0) 

540 converter = rerc( 

541 templatefile, 

542 lang=lang, 

543 sublang=sublang, 

544 charset=encoding, 

545 ) 

546 outputrclines = converter.convertstore(self.store, includefuzzy=True) 

547 try: 

548 handle.write(outputrclines.encode(encoding)) 

549 except UnicodeEncodeError: 

550 handle.write(codecs.BOM_UTF16_LE) 

551 handle.write(outputrclines.encode("utf-16-le")) 

552 

553 

554class PlainTextFormat(ConvertFormat): 

555 # Translators: File format name 

556 name = gettext_lazy("Plain text file") 

557 format_id = "txt" 

558 autoload = ("*.txt",) 

559 flavour = "plain" 

560 

561 @staticmethod 

562 def mimetype() -> str: 

563 """Return most common media type for format.""" 

564 return "text/plain" 

565 

566 @staticmethod 

567 def extension() -> str: 

568 """Return most common file extension for format.""" 

569 return "txt" 

570 

571 def convertfile(self, storefile, template_store): 

572 input_store = TxtFile(encoding="utf-8", flavour=self.flavour) 

573 input_store.parse(storefile.readlines()) 

574 input_store.filename = os.path.basename(storefile.name) 

575 return self.convert_to_po(input_store, template_store) 

576 

577 def save_content(self, handle) -> None: 

578 """Store content to file.""" 

579 templatename = self.template_store.storefile 

580 if hasattr(templatename, "name"): 

581 templatename = templatename.name 

582 with open(templatename, "rb") as templatefile: 

583 converter = po2txt( 

584 input_file=self.store, 

585 output_file=None, 

586 template_file=templatefile, 

587 ) 

588 outputstring = converter.merge_stores() 

589 handle.write(outputstring.encode("utf-8")) 

590 

591 

592class DokuWikiFormat(PlainTextFormat): 

593 # Translators: File format name 

594 name = gettext_lazy("DokuWiki text file") 

595 format_id = "dokuwiki" 

596 autoload = ("*.dw",) 

597 flavour = "dokuwiki" 

598 

599 

600class MediaWikiFormat(PlainTextFormat): 

601 # Translators: File format name 

602 name = gettext_lazy("MediaWiki text file") 

603 format_id = "mediawiki" 

604 autoload = ("*.mw",) 

605 flavour = "mediawiki"