Coverage for app/venv/lib/python3.14/site-packages/weblate/formats/base.py: 55%

3# SPDX-License-Identifier: GPL-3.0-or-later

5"""Base classes for file formats."""

7from __future__ import annotations

9import os

10import tempfile

11from copy import copy

12from pathlib import Path

13from typing import TYPE_CHECKING, Any, BinaryIO, ClassVar, TypeAlias

15from django.http import HttpResponse

16from django.utils.functional import cached_property

17from django.utils.translation import gettext

18from translate.misc.multistring import multistring

19from translate.storage.base import TranslationStore as TranslateToolkitStore

20from translate.storage.base import TranslationUnit as TranslateToolkitUnit

21from weblate_language_data.countries import DEFAULT_LANGS

23from weblate.checks.flags import Flags

24from weblate.trans.util import get_string, join_plural, split_plural

25from weblate.utils.errors import add_breadcrumb

26from weblate.utils.hash import calculate_hash

27from weblate.utils.site import get_site_url

28from weblate.utils.state import STATE_EMPTY, STATE_TRANSLATED

30if TYPE_CHECKING: 30 ↛ 31line 30 didn't jump to line 31 because the condition on line 30 was never true

31 from collections.abc import Callable, Iterator, Sequence

33 from django_stubs_ext import StrOrPromise

35 from weblate.trans.models import Translation, Unit

38EXPAND_LANGS = {code[:2]: f"{code[:2]}_{code[3:].upper()}" for code in DEFAULT_LANGS}

40ANDROID_CODES = {

41 "he": "iw",

42 "id": "in",

43 "yi": "ji",

44}

45LEGACY_CODES = {

46 "zh_Hans": "zh_CN",

47 "zh_Hant": "zh_TW",

48 "zh_Hans_SG": "zh_SG",

49 "zh_Hant_HK": "zh_HK",

50}

51APPSTORE_CODES = {

52 "ar": "ar-SA",

53 "de": "de-DE",

54 "fr": "fr-FR",

55 "nl": "nl-NL",

56 "pt": "pt-PT",

57}

59# Based on https://support.google.com/googleplay/android-developer/answer/9844778

60GOOGLEPLAY_CODES = {

61 "hy": "hy-AM",

62 "az": "az-AZ",

63 "eu": "eu-ES",

64 "my": "my-MM",

65 "zh_Hant_HK": "zh-HK",

66 "zh_Hans": "zh-CN",

67 "zh_Hant": "zh-TW",

68 "cs": "cs-CZ",

69 "da": "da-DK",

70 "nl": "nl-NL",

71 "en": "en-SG",

72 "fi": "fi-FI",

73 "fr": "fr-FR",

74 "gl": "gl-ES",

75 "ka": "ka-GE",

76 "de": "de-DE",

77 "el": "el-GR",

78 "he": "iw-IL",

79 "hi": "hi-IN",

80 "hu": "hu-HU",

81 "is": "is-IS",

82 "it": "it-IT",

83 "ja": "ja-JP",

84 "kn": "kn-IN",

85 "km": "km-KH",

86 "ko": "ko-KR",

87 "ky": "ky-KG",

88 "lo": "lo-LA",

89 "mk": "mk-MK",

90 "ms": "ms-MY",

91 "ml": "ml-IN",

92 "mr": "mr-IN",

93 "mn": "mn-MN",

94 "ne": "ne-NP",

95 "nb_NO": "no-NO",

96 "fa": "fa-IR",

97 "pl": "pl-PL",

98 "ru": "ru-RU",

99 "si": "si-LK",

100 "es": "es-ES",

101 "sv": "sv-SE",

102 "ta": "ta-IN",

103 "te": "te-IN", # codespell:ignore te

104 "tr": "tr-TR",

105}

106

107

108class UnitNotFoundError(Exception):

109 def __str__(self) -> str:

110 args = list(self.args)

111 if "" in args:

112 args.remove("")

113 return "Unit not found: {}".format(", ".join(args))

114

115

116class UpdateError(Exception):

117 def __init__(self, cmd, output) -> None:

118 super().__init__(output)

119 self.cmd = cmd

120 self.output = output

121

122

123class MissingTemplateError(TypeError):

124 pass

125

126

127class BaseItem:

128 pass

129

130

131InnerUnit: TypeAlias = TranslateToolkitUnit | BaseItem

132

133

134class BaseStore:

135 units: Sequence[InnerUnit]

136

137

138InnerStore: TypeAlias = TranslateToolkitStore | BaseStore

139

140

141class TranslationUnit:

142 """

143 Wrapper for translate-toolkit unit.

144

145 It handles ID/template based translations and other API differences.

146 """

147

148 id_hash_with_source: bool = False

149 template: InnerUnit | None

150 unit: InnerUnit | None

151 parent: TranslationFormat

152 mainunit: InnerUnit

153 empty_unit_ok: ClassVar[bool] = False

154

155 def __init__(

156 self,

157 parent: TranslationFormat,

158 unit: InnerUnit | None,

159 template: InnerUnit | None = None,

160 ) -> None:

161 """Create wrapper object."""

162 self.unit = unit

163 self.template = template

164 self.parent = parent

165 if template is not None: 165 ↛ 166line 165 didn't jump to line 166 because the condition on line 165 was never true

166 self.mainunit = template

167 elif unit is None and not self.empty_unit_ok: 167 ↛ 170line 167 didn't jump to line 170 because the condition on line 167 was never true

168 # MultiUnit is just wrapper around more unit objects, does not have

169 # actual main unit

170 raise MissingTemplateError

171 else:

172 self.mainunit = unit

173

174 def _invalidate_target(self) -> None:

175 """Invalidate target cache."""

176 if "target" in self.__dict__:

177 del self.__dict__["target"]

178

179 def invalidate_all_caches(self) -> None:

180 """Invalidate attributes cache."""

181 for attr in (

182 "context",

183 "source",

184 "locations",

185 "flags",

186 "notes",

187 "explanation",

188 "source_explanation",

189 "previous_source",

190 ):

191 if attr in self.__dict__:

192 del self.__dict__[attr]

193 self._invalidate_target()

194

195 @cached_property

196 def locations(self) -> str:

197 """Return comma separated list of locations."""

198 return ""

199

200 @cached_property

201 def flags(self) -> Flags:

202 """Return flags or typecomments from units."""

203 flags = Flags()

204 # add default flags based location extensions

205 for location in self.locations.split(","):

206 _, extension = os.path.splitext(location.split(":")[0].strip())

207 if extension == ".rst": 207 ↛ 208line 207 didn't jump to line 208 because the condition on line 207 was never true

208 flags.merge("rst-text")

209 elif extension in {".md", ".markdown"}: 209 ↛ 210line 209 didn't jump to line 210 because the condition on line 209 was never true

210 flags.merge("md-text")

211 return flags

212

213 @cached_property

214 def notes(self) -> str:

215 """Return notes from units."""

216 return ""

217

218 @cached_property

219 def source(self) -> str:

220 """Return source string from a ttkit unit."""

221 raise NotImplementedError

222

223 @cached_property

224 def target(self) -> str:

225 """Return target string from a ttkit unit."""

226 raise NotImplementedError

227

228 @cached_property

229 def explanation(self) -> str:

230 """Return explanation from a ttkit unit."""

231 return ""

232

233 @cached_property

234 def source_explanation(self) -> str:

235 """Return source explanation from a ttkit unit."""

236 return ""

237

238 @cached_property

239 def context(self) -> str:

240 """

241 Return context of message.

242

243 In some cases we have to use ID here to make all backends consistent.

244 """

245 raise NotImplementedError

246

247 @cached_property

248 def previous_source(self) -> str:

249 """Return previous message source if there was any."""

250 return ""

251

252 @classmethod

253 def calculate_id_hash(cls, has_template: bool, source: str, context: str) -> int:

254 """

255 Return hash of source string, used for quick lookup.

256

257 We use siphash as it is fast and works well for our purpose.

258 """

259 if not has_template or cls.id_hash_with_source: 259 ↛ 261line 259 didn't jump to line 261 because the condition on line 259 was always true

260 return calculate_hash(source, context)

261 return calculate_hash(context)

262

263 @cached_property

264 def id_hash(self) -> int:

265 return self.calculate_id_hash(

266 self.template is not None,

267 self.source,

268 self.context,

269 )

270

271 def has_translation(self) -> bool:

272 """Check whether unit has translation."""

273 return any(split_plural(self.target))

274

275 def is_translated(self) -> bool:

276 """Check whether unit is translated."""

277 return self.has_translation()

278

279 def is_approved(self, fallback=False) -> bool:

280 """Check whether unit is approved."""

281 return fallback

282

283 def is_fuzzy(self, fallback=False) -> bool:

284 """Check whether unit needs edit."""

285 return fallback

286

287 def has_content(self) -> bool:

288 """Check whether unit has content."""

289 return True

290

291 def is_readonly(self) -> bool:

292 """Check whether unit is read-only."""

293 return False

294

295 def set_target(self, target: str | list[str]) -> None:

296 """Set translation unit target."""

297 raise NotImplementedError

298

299 def set_explanation(self, explanation: str) -> None:

300 return

301

302 def set_source_explanation(self, explanation: str) -> None:

303 return

304

305 def set_state(self, state) -> None:

306 """Set fuzzy /approved flag on translated unit."""

307 raise NotImplementedError

308

309 def has_unit(self) -> bool:

310 return self.unit is not None

311

312 def clone_template(self) -> None:

313 if self.template is None:

314 raise MissingTemplateError

315 self.mainunit = self.unit = copy(self.template)

316 self._invalidate_target()

317

318 def untranslate(self, language) -> None:

319 self.set_target("")

320 self.set_state(STATE_EMPTY)

321

322

323class TranslationFormat:

324 """Generic object defining file format loader."""

325

326 name: StrOrPromise = ""

327 format_id: str = ""

328 monolingual: bool | None = None

329 check_flags: tuple[str, ...] = ()

330 unit_class: type[TranslationUnit] = TranslationUnit

331 autoload: tuple[str, ...] = ()

332 can_add_unit: bool = True

333 can_delete_unit: bool = True

334 language_format: str = "posix"

335 simple_filename: bool = True

336 new_translation: str | bytes | None = None

337 autoaddon: ClassVar[dict[str, dict[str, Any]]] = {}

338 create_empty_bilingual: bool = False

339 bilingual_class: type[TranslationFormat] | None = None

340 create_style = "create"

341 has_multiple_strings: bool = False

342 supports_explanation: bool = False

343 supports_plural: bool = False

344 can_edit_base: bool = True

345 strict_format_plurals: bool = False

346 plural_preference: tuple[int, ...] | None = None

347 store: InnerStore

348

349 @classmethod

350 def get_identifier(cls):

351 return cls.format_id

352

353 def __init__(

354 self,

355 storefile: Path | str | BinaryIO,

356 template_store: TranslationFormat | None = None,

357 language_code: str | None = None,

358 source_language: str | None = None,

359 is_template: bool = False,

360 existing_units: list[Unit] | None = None,

361 file_format_params: dict[str, Any] | None = None,

362 ) -> None:

363 """Create file format object, wrapping up translate-toolkit's store."""

364 if isinstance(storefile, Path): 364 ↛ 365line 364 didn't jump to line 365 because the condition on line 364 was never true

365 storefile = str(storefile.as_posix())

366 if not isinstance(storefile, str) and not hasattr(storefile, "mode"): 366 ↛ 368line 366 didn't jump to line 368 because the condition on line 366 was never true

367 # This is BinaryIO like but without a mode

368 storefile.mode = "r" # type: ignore[misc]

369

370 self.storefile = storefile

371 self.language_code = language_code

372 self.source_language = source_language

373 # Remember template

374 self.template_store = template_store

375 self.is_template = is_template

376 self.existing_units = [] if existing_units is None else existing_units

377

378 # Load store

379 self.file_format_params = file_format_params or {}

380 self.store = self.load(storefile, template_store)

381

382 self.add_breadcrumb(

383 "Loaded translation file {}".format(

384 getattr(storefile, "filename", storefile)

385 ),

386 template_store=str(template_store),

387 is_template=is_template,

388 )

389

390 def _invalidate_units(self) -> None:

391 for key in ("all_units", "template_units", "_unit_index", "_template_index"):

392 if key in self.__dict__:

393 del self.__dict__[key]

394

395 def check_valid(self) -> None:

396 """Check store validity."""

397 if not self.is_valid(): 397 ↛ 398line 397 didn't jump to line 398 because the condition on line 397 was never true

398 raise ValueError(

399 gettext(

400 "Could not load strings from the file, try choosing other format."

401 )

402 )

403 self.ensure_index()

404

405 def get_filenames(self):

406 if isinstance(self.storefile, str):

407 return [self.storefile]

408 return [self.storefile.name]

409

410 def load(

411 self,

412 storefile: str | BinaryIO,

413 template_store: TranslationFormat | None,

414 ) -> InnerStore:

415 raise NotImplementedError

416

417 @classmethod

418 def get_plural(cls, language, store=None): # noqa: ARG003

419 """Return matching plural object."""

420 if cls.plural_preference is not None:

421 # Fetch all matching plurals

422 plurals = language.plural_set.filter(source__in=cls.plural_preference)

423

424 # Use first matching in the order of preference

425 for source in cls.plural_preference: 425 ↛ 431line 425 didn't jump to line 431 because the loop on line 425 didn't complete

426 for plural in plurals:

427 if plural.source == source:

428 return plural

429

430 # Fall back to default one

431 return language.plural

432

433 @cached_property

434 def has_template(self):

435 """Check whether class is using template."""

436 return (

437 self.monolingual or self.monolingual is None

438 ) and self.template_store is not None

439

440 @cached_property

441 def _template_index(self):

442 """ID based index for units."""

443 return {unit.id_hash: unit for unit in self.template_units}

444

445 def find_unit_template(

446 self, context: str, source: str, id_hash: int | None = None

447 ) -> InnerUnit | None:

448 if id_hash is None:

449 id_hash = self._calculate_string_hash(context, source)

450 try:

451 # The mono units always have only template set

452 return self._template_index[id_hash].template

453 except KeyError:

454 return None

455

456 def _find_unit_monolingual(

457 self, context: str, source: str

458 ) -> tuple[TranslationUnit, bool]:

459 # We search by ID when using template

460 id_hash = self._calculate_string_hash(context, source)

461 try:

462 result = self._unit_index[id_hash]

463 except KeyError as error:

464 raise UnitNotFoundError(context, source) from error

465

466 add = False

467 if not result.has_unit():

468 # We always need copy of template unit to translate

469 result.clone_template()

470 add = True

471 return result, add

472

473 @cached_property

474 def _unit_index(self) -> dict[int, TranslationUnit]:

475 """Context and source based index for units."""

476 return {unit.id_hash: unit for unit in self.content_units}

477

478 def _calculate_string_hash(self, context: str, source: str) -> int:

479 """Calculate id hash for a string."""

480 return self.unit_class.calculate_id_hash(

481 self.has_template or self.is_template, get_string(source), context

482 )

483

484 def _find_unit_bilingual(

485 self, context: str, source: str

486 ) -> tuple[TranslationUnit, bool]:

487 id_hash = self._calculate_string_hash(context, source)

488 try:

489 return (self._unit_index[id_hash], False)

490 except KeyError as error:

491 raise UnitNotFoundError(context, source) from error

492

493 def find_unit(self, context: str, source: str) -> tuple[TranslationUnit, bool]:

494 """

495 Find unit by context and source.

496

497 Returns tuple (ttkit_unit, created) indicating whether returned unit is new one.

498 """

499 if self.has_template: 499 ↛ 500line 499 didn't jump to line 500 because the condition on line 499 was never true

500 return self._find_unit_monolingual(context, source)

501 return self._find_unit_bilingual(context, source)

502

503 def ensure_index(self):

504 return self._unit_index

505

506 def add_unit(self, unit: TranslationUnit) -> None:

507 """Add new unit to underlying store."""

508 raise NotImplementedError

509

510 def update_header(self, **kwargs) -> None:

511 """Update store header if available."""

512 return

513

514 @staticmethod

515 def save_atomic(filename, callback) -> None:

516 dirname, basename = os.path.split(filename)

517 if dirname and not os.path.exists(dirname): 517 ↛ 518line 517 didn't jump to line 518 because the condition on line 517 was never true

518 os.makedirs(dirname)

519 try:

520 with tempfile.NamedTemporaryFile(

521 prefix=basename, dir=dirname, delete=False

522 ) as temp:

523 callback(temp)

524 os.replace(temp.name, filename)

525 finally:

526 if os.path.exists(temp.name): 526 ↛ 527line 526 didn't jump to line 527 because the condition on line 526 was never true

527 os.unlink(temp.name)

528

529 def save(self) -> None:

530 """Save underlying store to disk."""

531 raise NotImplementedError

532

533 @property

534 def all_store_units(self) -> list[InnerUnit]:

535 """Wrapper for all store units for possible filtering."""

536 return self.store.units # type: ignore[return-value]

537

538 @cached_property

539 def template_units(self) -> list[TranslationUnit]:

540 return [self.unit_class(self, None, unit) for unit in self.all_store_units]

541

542 def _get_all_bilingual_units(self) -> list[TranslationUnit]:

543 return [self.unit_class(self, unit) for unit in self.all_store_units]

544

545 def _build_monolingual_unit(self, unit: TranslationUnit) -> TranslationUnit:

546 return self.unit_class(

547 self,

548 self.find_unit_template(unit.context, unit.source, unit.id_hash),

549 unit.template,

550 )

551

552 def _get_all_monolingual_units(self) -> list[TranslationUnit]:

553 if self.template_store is None:

554 raise MissingTemplateError

555 return [

556 self._build_monolingual_unit(unit)

557 for unit in self.template_store.template_units

558 ]

559

560 @cached_property

561 def all_units(self) -> list[TranslationUnit]:

562 """List of all units."""

563 if not self.has_template: 563 ↛ 565line 563 didn't jump to line 565 because the condition on line 563 was always true

564 return self._get_all_bilingual_units()

565 return self._get_all_monolingual_units()

566

567 @property

568 def content_units(self) -> list[TranslationUnit]:

569 return [unit for unit in self.all_units if unit.has_content()]

570

571 @staticmethod

572 def mimetype() -> str:

573 """Return most common mime type for format."""

574 return "text/plain"

575

576 @staticmethod

577 def extension() -> str:

578 """Return most common file extension for format."""

579 return "txt"

580

581 def is_valid(self) -> bool:

582 """Check whether store seems to be valid."""

583 for unit in self.content_units:

584 # Just ensure that id_hash can be calculated

585 unit.id_hash # noqa: B018

586 return True

587

588 @classmethod

589 def is_valid_base_for_new(

590 cls,

591 base: str,

592 monolingual: bool,

593 errors: list[Exception] | None = None,

594 fast: bool = False,

595 file_format_params: dict[str, Any] | None = None,

596 ) -> bool:

597 """Check whether base is valid."""

598 raise NotImplementedError

599

600 @classmethod

601 def get_language_code(cls, code: str, language_format: str | None = None) -> str:

602 """Do any possible formatting needed for language code."""

603 if not language_format: 603 ↛ 605line 603 didn't jump to line 605 because the condition on line 603 was always true

604 language_format = cls.language_format

605 return getattr(cls, f"get_language_{language_format}")(code)

606

607 @staticmethod

608 def get_language_posix(code: str) -> str:

609 return code.replace("-", "_")

610

611 @classmethod

612 def get_language_posix_lowercase(cls, code: str) -> str:

613 return cls.get_language_posix(code).lower()

614

615 @staticmethod

616 def get_language_bcp(code: str) -> str:

617 return code.replace("_", "-")

618

619 @classmethod

620 def get_language_bcp_lower(cls, code: str) -> str:

621 return cls.get_language_bcp(code).lower()

622

623 @classmethod

624 def get_language_posix_long(cls, code: str) -> str:

625 return EXPAND_LANGS.get(code, cls.get_language_posix(code))

626

627 @classmethod

628 def get_language_posix_long_lowercase(cls, code: str) -> str:

629 return EXPAND_LANGS.get(code, cls.get_language_posix(code)).lower()

630

631 @classmethod

632 def get_language_linux(cls, code: str) -> str:

633 """Linux doesn't use Hans/Hant, but rather TW/CN variants."""

634 return LEGACY_CODES.get(code, cls.get_language_posix(code))

635

636 @classmethod

637 def get_language_linux_lowercase(cls, code: str) -> str:

638 return cls.get_language_linux(code).lower()

639

640 @classmethod

641 def get_language_bcp_long(cls, code: str) -> str:

642 return cls.get_language_bcp(cls.get_language_posix_long(code))

643

644 @classmethod

645 def get_language_android(cls, code: str) -> str:

646 """Android doesn't use Hans/Hant, but rather TW/CN variants."""

647 # Exceptions

648 if code in ANDROID_CODES:

649 return ANDROID_CODES[code]

650

651 # Base on Java

652 sanitized = cls.get_language_linux(code)

653

654 # Handle variants

655 if "_" in sanitized and len(sanitized.split("_")[1]) > 2:

656 return "b+{}".format(sanitized.replace("_", "+"))

657

658 # Handle countries

659 return sanitized.replace("_", "-r")

660

661 @classmethod

662 def get_language_bcp_legacy(cls, code: str) -> str:

663 """BCP, but doesn't use Hans/Hant, but rather TW/CN variants."""

664 return cls.get_language_bcp(cls.get_language_linux(code))

665

666 @classmethod

667 def get_language_appstore(cls, code: str) -> str:

668 """Apple App Store language codes."""

669 return cls.get_language_bcp(APPSTORE_CODES.get(code, code))

670

671 @classmethod

672 def get_language_googleplay(cls, code: str) -> str:

673 """Google Play language codes."""

674 return cls.get_language_bcp(GOOGLEPLAY_CODES.get(code, code))

675

676 @classmethod

677 def get_language_filename(cls, mask: str, code: str) -> str:

678 """

679 Return full filename of a language file.

680

681 Calculated for given path, filemask and language code.

682 """

683 return mask.replace("*", code)

684

685 @classmethod

686 def add_language(

687 cls,

688 filename: str | Path,

689 language: str,

690 base: str,

691 callback: Callable | None = None,

692 file_format_params: dict[str, Any] | None = None,

693 ) -> None:

694 """Add new language file."""

695 # Create directory for a translation

696 if not isinstance(filename, Path): 696 ↛ 698line 696 didn't jump to line 698 because the condition on line 696 was always true

697 filename = Path(filename)

698 dirname = filename.parent

699 if not dirname.exists(): 699 ↛ 700line 699 didn't jump to line 700 because the condition on line 699 was never true

700 dirname.mkdir(parents=True)

701

702 cls.create_new_file(

703 str(filename.as_posix()), language, base, callback, file_format_params

704 )

705

706 @classmethod

707 def get_new_file_content(cls) -> bytes:

708 return b""

709

710 @classmethod

711 def create_new_file(

712 cls,

713 filename: str,

714 language: str,

715 base: str,

716 callback: Callable | None = None,

717 file_format_params: dict[str, Any] | None = None,

718 ) -> None:

719 """Handle creation of new translation file."""

720 raise NotImplementedError

721

722 def iterate_merge(

723 self, fuzzy: str, only_translated: bool = True

724 ) -> Iterator[tuple[bool, TranslationUnit]]:

725 """

726 Iterate over units for merging.

727

728 Note: This can change fuzzy state of units!

729 """

730 for unit in self.content_units:

731 # Skip fuzzy (if asked for that)

732 if unit.is_fuzzy():

733 if not fuzzy:

734 continue

735 elif only_translated and not unit.is_translated():

736 continue

737

738 # Unmark unit as fuzzy (to allow merge)

739 set_fuzzy = False

740 if fuzzy and unit.is_fuzzy():

741 unit.set_state(STATE_TRANSLATED)

742 if fuzzy != "approve":

743 set_fuzzy = True

744

745 yield set_fuzzy, unit

746

747 def create_unit(

748 self,

749 key: str,

750 source: str | list[str],

751 target: str | list[str] | None = None,

752 ) -> InnerUnit:

753 raise NotImplementedError

754

755 def new_unit(

756 self,

757 key: str,

758 source: str | list[str],

759 target: str | list[str] | None = None,

760 ) -> TranslationUnit:

761 """Add new unit to monolingual store."""

762 # Create backend unit object

763 unit = self.create_unit(key, source, target)

764

765 # Build an unit object

766 template_unit: InnerUnit | None

767 if self.has_template:

768 if self.is_template:

769 template_unit = unit

770 else:

771 template_unit = self._find_unit_monolingual(

772 key, join_plural(source) if isinstance(source, list) else source

773 )[0].unit

774 else:

775 template_unit = None

776 result = self.unit_class(self, unit, template_unit)

777 mono_unit = self.unit_class(self, None, unit)

778

779 # Update cached lookups

780 if "all_units" in self.__dict__:

781 self.all_units.append(result)

782 if "template_units" in self.__dict__:

783 self.template_units.append(mono_unit)

784 if "_unit_index" in self.__dict__:

785 self._unit_index[result.id_hash] = result

786 if "_template_index" in self.__dict__:

787 self._template_index[mono_unit.id_hash] = mono_unit

788

789 # Add it to the file

790 self.add_unit(result)

791

792 # Invalidate all attributes

793 result.invalidate_all_caches()

794

795 return result

796

797 @classmethod

798 def get_class(cls) -> InnerStore | None:

799 raise NotImplementedError

800

801 @classmethod

802 def add_breadcrumb(cls, message, **data) -> None:

803 add_breadcrumb(category="storage", message=message, **data)

804

805 def delete_unit(self, ttkit_unit: InnerUnit) -> str | None:

806 raise NotImplementedError

807

808 def cleanup_unused(self) -> list[str] | None:

809 """Remove unused strings, returning list of additional changed files."""

810 if not self.template_store or not self.can_delete_unit:

811 return None

812 existing = {template.context for template in self.template_store.template_units}

813

814 changed = False

815 needs_save = False

816 result = []

817

818 # Iterate over copy of a list as we are changing it when removing units

819 for unit in list(self.all_store_units):

820 if self.unit_class(self, None, unit).context not in existing:

821 changed = True

822 item = self.delete_unit(unit)

823 if item is not None:

824 result.append(item)

825 else:

826 needs_save = True

827

828 if not changed:

829 return None

830

831 if needs_save:

832 self.save()

833 self._invalidate_units()

834 return result

835

836 def cleanup_blank(self) -> list[str] | None:

837 """

838 Remove strings without translations.

839

840 Returning list of additional changed files.

841 """

842 if not self.can_delete_unit:

843 return None

844 changed = False

845 needs_save = False

846 result = []

847

848 # Iterate over copy of a list as we are changing it when removing units

849 for ttkit_unit in list(self.all_store_units):

850 target = split_plural(self.unit_class(self, ttkit_unit, ttkit_unit).target)

851 if not any(target):

852 changed = True

853 item = self.delete_unit(ttkit_unit)

854 if item is not None:

855 result.append(item)

856 else:

857 needs_save = True

858

859 if not changed:

860 return None

861

862 if needs_save:

863 self.save()

864 self._invalidate_units()

865 return result

866

867 def remove_unit(self, ttkit_unit: InnerUnit) -> list[str]:

868 """High level wrapper for unit removal."""

869 changed = False

870

871 result = []

872

873 item = self.delete_unit(ttkit_unit)

874 if item is not None:

875 result.append(item)

876 else:

877 changed = True

878

879 if changed:

880 self.save()

881 self._invalidate_units()

882 return result

883

884 @staticmethod

885 def validate_context(context: str) -> None: # noqa: ARG004

886 return

887

888

889class EmptyFormat(TranslationFormat):

890 """For testing purposes."""

891

892 @classmethod

893 def load(cls, storefile, template_store): # noqa: ARG003

894 return type("", (object,), {"units": []})()

895

896 def save(self) -> None:

897 return

898

899

900class BilingualUpdateMixin:

901 @classmethod

902 def do_bilingual_update(

903 cls, in_file: str, out_file: str, template: str, **kwargs

904 ) -> None:

905 raise NotImplementedError

906

907 @classmethod

908 def update_bilingual(cls, filename: str, template: str, **kwargs) -> None:

909 with tempfile.NamedTemporaryFile(

910 prefix=filename, dir=os.path.dirname(filename), delete=False

911 ) as temp:

912 # We want file to be created only here

913 pass

914 try:

915 cls.do_bilingual_update(filename, temp.name, template, **kwargs)

916 os.replace(temp.name, filename)

917 finally:

918 if os.path.exists(temp.name):

919 os.unlink(temp.name)

920

921 @classmethod

922 def get_msgmerge_args(cls, component) -> list[str]:

923 """

924 Return list of arguments for update command.

925

926 This is used to pass additional arguments to update command.

927 """

928 from weblate.addons.gettext import MsgmergeAddon

929

930 params = component.file_format_params

931 args: list[str] = []

932 if component.get_addon(MsgmergeAddon.name):

933 if not params.get("po_fuzzy_matching", True):

934 args.append("--no-fuzzy-matching")

935 if params.get("po_keep_previous", True):

936 args.append("--previous")

937 if params.get("po_no_location", False):

938 args.append("--no-location")

939 else:

940 args.append("--previous")

941

942 if int(params.get("po_line_wrap", 77)) != 77:

943 args.append("--no-wrap")

944 return args

945

946

947class BaseExporter:

948 content_type = "text/plain"

949 extension = "txt"

950 name = ""

951 verbose: StrOrPromise = ""

952 set_id = False

953 storage_class: ClassVar[type[TranslateToolkitStore]]

954

955 def __init__(

956 self,

957 project=None,

958 source_language=None,

959 language=None,

960 url=None,

961 translation=None,

962 fieldnames=None,

963 ) -> None:

964 self.translation = translation

965 if translation is not None: 965 ↛ 972line 965 didn't jump to line 972 because the condition on line 965 was always true

966 self.plural = translation.plural

967 self.project = translation.component.project

968 self.source_language = translation.component.source_language

969 self.language = translation.language

970 self.url = get_site_url(translation.get_absolute_url())

971 else:

972 self.project = project

973 self.language = language

974 self.source_language = source_language

975 self.plural = language.plural

976 self.url = url

977 self.fieldnames = fieldnames

978

979 @staticmethod

980 def supports(translation: Translation) -> bool: # noqa: ARG004

981 return True

982

983 @cached_property

984 def storage(self):

985 storage = self.get_storage()

986 storage.setsourcelanguage(self.source_language.code)

987 storage.settargetlanguage(self.language.code)

988 return storage

989

990 def string_filter(self, text):

991 return text

992

993 def handle_plurals(self, plurals):

994 if len(plurals) == 1: 994 ↛ 996line 994 didn't jump to line 996 because the condition on line 994 was always true

995 return self.string_filter(plurals[0])

996 return multistring([self.string_filter(plural) for plural in plurals])

997

998 @classmethod

999 def get_identifier(cls):

1000 return cls.name

1001

1002 def get_storage(self):

1003 return self.storage_class()

1004

1005 def add(self, unit: TranslateToolkitUnit, word: str) -> None:

1006 unit.target = word

1007

1008 def create_unit(self, source: str) -> TranslateToolkitUnit:

1009 return self.storage.UnitClass(source)

1010

1011 def add_units(self, units: list[Unit]) -> None:

1012 for unit in units:

1013 self.add_unit(unit)

1014

1015 def build_unit(self, unit: Unit) -> TranslateToolkitUnit:

1016 output = self.create_unit(self.handle_plurals(unit.get_source_plurals()))

1017 # Propagate source language

1018 if hasattr(output, "setsource"): 1018 ↛ 1019line 1018 didn't jump to line 1019 because the condition on line 1018 was never true

1019 output.setsource(output.source, sourcelang=self.source_language.code)

1020 self.add(output, self.handle_plurals(unit.get_target_plurals()))

1021 return output

1022

1023 def add_note(self, output, note: str, origin: str) -> None:

1024 output.addnote(note, origin=origin)

1025

1026 def add_unit(self, unit: Unit) -> None:

1027 output = self.build_unit(unit)

1028 # Location needs to be set prior to ID to avoid overwrite

1029 # on some formats (for example xliff)

1030 for location in self.string_filter(unit.location).split(","):

1031 location = location.strip()

1032 if location:

1033 output.addlocation(location)

1034

1035 # Store context as context and ID

1036 context = self.string_filter(unit.context)

1037 if context: 1037 ↛ 1038line 1037 didn't jump to line 1038 because the condition on line 1037 was never true

1038 output.setcontext(context)

1039 if self.set_id:

1040 output.setid(context)

1041 elif self.set_id: 1041 ↛ 1043line 1041 didn't jump to line 1043 because the condition on line 1041 was never true

1042 # Use checksum based ID on formats requiring it

1043 output.setid(unit.checksum)

1044

1045 # Store note

1046 note = self.string_filter(unit.note)

1047 if note: 1047 ↛ 1048line 1047 didn't jump to line 1048 because the condition on line 1047 was never true

1048 self.add_note(output, note, origin="developer")

1049 # In Weblate explanation

1050 note = self.string_filter(unit.source_unit.explanation)

1051 if note:

1052 self.add_note(output, note, origin="developer")

1053 # Comments

1054 for comment in unit.unresolved_comments:

1055 self.add_note(

1056 output, self.string_filter(comment.comment), origin="translator"

1057 )

1058 # Suggestions

1059 for suggestion in unit.suggestions: 1059 ↛ 1060line 1059 didn't jump to line 1060 because the loop on line 1059 never started

1060 self.add_note(

1061 output,

1062 self.string_filter(

1063 "Suggested in Weblate: {}".format(

1064 ", ".join(split_plural(suggestion.target))

1065 )

1066 ),

1067 origin="translator",

1068 )

1069

1070 # Store flags

1071 if unit.all_flags:

1072 self.store_flags(output, unit.all_flags)

1073

1074 # Store fuzzy flag

1075 self.store_unit_state(output, unit)

1076

1077 self.storage.addunit(output)

1078

1079 def store_unit_state(self, output, unit) -> None:

1080 if unit.fuzzy: 1080 ↛ 1081line 1080 didn't jump to line 1081 because the condition on line 1080 was never true

1081 output.markfuzzy(True)

1082 if hasattr(output, "markapproved"): 1082 ↛ 1083line 1082 didn't jump to line 1083 because the condition on line 1082 was never true

1083 output.markapproved(unit.approved)

1084

1085 def get_filename(self, filetemplate: str = "{path}.{extension}"):

1086 return filetemplate.format(

1087 project=self.project.slug,

1088 language=self.language.code,

1089 extension=self.extension,

1090 path="-".join(

1091 self.translation.get_url_path()

1092 if self.translation

1093 else (self.project.slug, self.language.code)

1094 ),

1095 )

1096

1097 def get_response(self, filetemplate: str = "{path}.{extension}"):

1098 filename = self.get_filename(filetemplate)

1099

1100 response = HttpResponse(content_type=f"{self.content_type}; charset=utf-8")

1101 response["Content-Disposition"] = f"attachment; filename={filename}"

1102

1103 # Save to response

1104 response.write(self.serialize())

1105

1106 return response

1107

1108 def serialize(self) -> bytes:

1109 """Return storage content."""

1110 from weblate.formats.ttkit import TTKitFormat

1111

1112 return TTKitFormat.serialize(self.storage)

1113

1114 def store_flags(self, output, flags) -> None:

1115 return