Coverage for documents/models.py: 83%

1import datetime

2import uuid

3from pathlib import Path

4from typing import Final

6import pathvalidate

7from django.conf import settings

8from django.contrib.auth.models import Group

9from django.contrib.auth.models import User

10from django.core.exceptions import ValidationError

11from django.core.validators import MaxValueValidator

12from django.core.validators import MinValueValidator

13from django.db import models

14from django.utils import timezone

15from django.utils.translation import gettext_lazy as _

16from multiselectfield import MultiSelectField

17from treenode.models import TreeNodeModel

19if settings.AUDIT_LOG_ENABLED: 19 ↛ 22line 19 didn't jump to line 22 because the condition on line 19 was always true

20 from auditlog.registry import auditlog

22from django.db.models import Case

23from django.db.models import PositiveIntegerField

24from django.db.models.functions import Cast

25from django.db.models.functions import Length

26from django.db.models.functions import Substr

27from django_softdelete.models import SoftDeleteModel

29from documents.data_models import DocumentSource

30from documents.parsers import get_default_file_extension

33class ModelWithOwner(models.Model):

34 owner = models.ForeignKey(

35 User,

36 blank=True,

37 null=True,

38 default=None,

39 on_delete=models.SET_NULL,

40 verbose_name=_("owner"),

41 )

43 class Meta:

44 abstract = True

47class MatchingModel(ModelWithOwner):

48 MATCH_NONE = 0

49 MATCH_ANY = 1

50 MATCH_ALL = 2

51 MATCH_LITERAL = 3

52 MATCH_REGEX = 4

53 MATCH_FUZZY = 5

54 MATCH_AUTO = 6

56 MATCHING_ALGORITHMS = (

57 (MATCH_NONE, _("None")),

58 (MATCH_ANY, _("Any word")),

59 (MATCH_ALL, _("All words")),

60 (MATCH_LITERAL, _("Exact match")),

61 (MATCH_REGEX, _("Regular expression")),

62 (MATCH_FUZZY, _("Fuzzy word")),

63 (MATCH_AUTO, _("Automatic")),

64 )

66 name = models.CharField(_("name"), max_length=128)

68 match = models.CharField(_("match"), max_length=256, blank=True)

70 matching_algorithm = models.PositiveSmallIntegerField(

71 _("matching algorithm"),

72 choices=MATCHING_ALGORITHMS,

73 default=MATCH_ANY,

74 )

76 is_insensitive = models.BooleanField(_("is insensitive"), default=True)

78 class Meta(ModelWithOwner.Meta):

79 abstract = True

80 ordering = ("name",)

81 constraints = [

82 models.UniqueConstraint(

83 fields=["name", "owner"],

84 name="%(app_label)s_%(class)s_unique_name_owner",

85 ),

86 models.UniqueConstraint(

87 name="%(app_label)s_%(class)s_name_uniq",

88 fields=["name"],

89 condition=models.Q(owner__isnull=True),

90 ),

91 ]

93 def __str__(self):

94 return self.name

97class Correspondent(MatchingModel):

98 class Meta(MatchingModel.Meta):

99 verbose_name = _("correspondent")

100 verbose_name_plural = _("correspondents")

101

102

103class Tag(MatchingModel, TreeNodeModel):

104 color = models.CharField(_("color"), max_length=7, default="#a6cee3")

105 # Maximum allowed nesting depth for tags (root = 1, max depth = 5)

106 MAX_NESTING_DEPTH: Final[int] = 5

107

108 is_inbox_tag = models.BooleanField(

109 _("is inbox tag"),

110 default=False,

111 help_text=_(

112 "Marks this tag as an inbox tag: All newly consumed "

113 "documents will be tagged with inbox tags.",

114 ),

115 )

116

117 class Meta(MatchingModel.Meta, TreeNodeModel.Meta):

118 verbose_name = _("tag")

119 verbose_name_plural = _("tags")

120

121 def clean(self) -> None:

122 # Prevent self-parenting and assigning a descendant as parent

123 parent = self.get_parent()

124 if parent == self: 124 ↛ 125line 124 didn't jump to line 125 because the condition on line 124 was never true

125 raise ValidationError({"parent": _("Cannot set itself as parent.")})

126 if parent and self.pk is not None and self.is_ancestor_of(parent): 126 ↛ 127line 126 didn't jump to line 127 because the condition on line 126 was never true

127 raise ValidationError({"parent": _("Cannot set parent to a descendant.")})

128

129 # Enforce maximum nesting depth

130 new_parent_depth = 0

131 if parent:

132 new_parent_depth = parent.get_ancestors_count() + 1

133

134 height = 0 if self.pk is None else self.get_depth()

135 deepest_new_depth = (new_parent_depth + 1) + height

136 if deepest_new_depth > self.MAX_NESTING_DEPTH: 136 ↛ 137line 136 didn't jump to line 137 because the condition on line 136 was never true

137 raise ValidationError({"parent": _("Maximum nesting depth exceeded.")})

138

139 return super().clean()

140

141

142class DocumentType(MatchingModel):

143 class Meta(MatchingModel.Meta):

144 verbose_name = _("document type")

145 verbose_name_plural = _("document types")

146

147

148class StoragePath(MatchingModel):

149 path = models.TextField(

150 _("path"),

151 )

152

153 class Meta(MatchingModel.Meta):

154 verbose_name = _("storage path")

155 verbose_name_plural = _("storage paths")

156

157

158class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-missing]

159 MAX_STORED_FILENAME_LENGTH: Final[int] = 1024

160

161 correspondent = models.ForeignKey(

162 Correspondent,

163 blank=True,

164 null=True,

165 related_name="documents",

166 on_delete=models.SET_NULL,

167 verbose_name=_("correspondent"),

168 )

169

170 storage_path = models.ForeignKey(

171 StoragePath,

172 blank=True,

173 null=True,

174 related_name="documents",

175 on_delete=models.SET_NULL,

176 verbose_name=_("storage path"),

177 )

178

179 title = models.CharField(_("title"), max_length=128, blank=True, db_index=True)

180

181 document_type = models.ForeignKey(

182 DocumentType,

183 blank=True,

184 null=True,

185 related_name="documents",

186 on_delete=models.SET_NULL,

187 verbose_name=_("document type"),

188 )

189

190 content = models.TextField(

191 _("content"),

192 blank=True,

193 help_text=_(

194 "The raw, text-only data of the document. This field is "

195 "primarily used for searching.",

196 ),

197 )

198

199 content_length = models.GeneratedField(

200 expression=Length("content"),

201 output_field=PositiveIntegerField(default=0),

202 db_persist=True,

203 null=False,

204 serialize=False,

205 help_text="Length of the content field in characters. Automatically maintained by the database for faster statistics computation.",

206 )

207

208 mime_type = models.CharField(_("mime type"), max_length=256, editable=False)

209

210 tags = models.ManyToManyField(

211 Tag,

212 related_name="documents",

213 blank=True,

214 verbose_name=_("tags"),

215 )

216

217 checksum = models.CharField(

218 _("checksum"),

219 max_length=64,

220 editable=False,

221 db_index=True,

222 help_text=_("The checksum of the original document."),

223 )

224

225 archive_checksum = models.CharField(

226 _("archive checksum"),

227 max_length=64,

228 editable=False,

229 blank=True,

230 null=True,

231 db_index=True,

232 help_text=_("The checksum of the archived document."),

233 )

234

235 page_count = models.PositiveIntegerField(

236 _("page count"),

237 blank=False,

238 null=True,

239 unique=False,

240 db_index=True,

241 validators=[MinValueValidator(1)],

242 help_text=_(

243 "The number of pages of the document.",

244 ),

245 )

246

247 created = models.DateField(

248 _("created"),

249 default=datetime.date.today,

250 db_index=True,

251 )

252

253 modified = models.DateTimeField(

254 _("modified"),

255 auto_now=True,

256 editable=False,

257 db_index=True,

258 )

259

260 added = models.DateTimeField(

261 _("added"),

262 default=timezone.now,

263 editable=False,

264 db_index=True,

265 )

266

267 filename = models.FilePathField(

268 _("filename"),

269 max_length=MAX_STORED_FILENAME_LENGTH,

270 editable=False,

271 default=None,

272 unique=True,

273 null=True,

274 help_text=_("Current filename in storage"),

275 )

276

277 archive_filename = models.FilePathField(

278 _("archive filename"),

279 max_length=MAX_STORED_FILENAME_LENGTH,

280 editable=False,

281 default=None,

282 unique=True,

283 null=True,

284 help_text=_("Current archive filename in storage"),

285 )

286

287 original_filename = models.CharField(

288 _("original filename"),

289 max_length=MAX_STORED_FILENAME_LENGTH,

290 editable=False,

291 default=None,

292 unique=False,

293 null=True,

294 help_text=_("The original name of the file when it was uploaded"),

295 )

296

297 ARCHIVE_SERIAL_NUMBER_MIN: Final[int] = 0

298 ARCHIVE_SERIAL_NUMBER_MAX: Final[int] = 0xFF_FF_FF_FF

299

300 archive_serial_number = models.PositiveIntegerField(

301 _("archive serial number"),

302 blank=True,

303 null=True,

304 unique=True,

305 db_index=True,

306 validators=[

307 MaxValueValidator(ARCHIVE_SERIAL_NUMBER_MAX),

308 MinValueValidator(ARCHIVE_SERIAL_NUMBER_MIN),

309 ],

310 help_text=_(

311 "The position of this document in your physical document archive.",

312 ),

313 )

314

315 root_document = models.ForeignKey(

316 "self",

317 blank=True,

318 null=True,

319 related_name="versions",

320 on_delete=models.CASCADE,

321 verbose_name=_("root document for this version"),

322 )

323

324 version_index = models.PositiveIntegerField(

325 _("version index"),

326 blank=True,

327 null=True,

328 db_index=True,

329 help_text=_("Index of this version within the root document."),

330 )

331

332 version_label = models.CharField(

333 _("version label"),

334 max_length=64,

335 blank=True,

336 null=True,

337 help_text=_("Optional short label for a document version."),

338 )

339

340 class Meta:

341 ordering = ("-created",)

342 verbose_name = _("document")

343 verbose_name_plural = _("documents")

344 indexes = [

345 models.Index(fields=["owner", "created"]),

346 ]

347 constraints = [

348 models.UniqueConstraint(

349 fields=["root_document", "version_index"],

350 condition=models.Q(

351 root_document__isnull=False,

352 version_index__isnull=False,

353 ),

354 name="documents_document_root_version_index_uniq",

355 ),

356 ]

357

358 def __str__(self) -> str:

359 created = self.created.isoformat()

360

361 res = f"{created}"

362

363 if self.correspondent: 363 ↛ 365line 363 didn't jump to line 365 because the condition on line 363 was always true

364 res += f" {self.correspondent}"

365 if self.title: 365 ↛ 367line 365 didn't jump to line 367 because the condition on line 365 was always true

366 res += f" {self.title}"

367 return res

368

369 def get_effective_barcodes(self) -> list["DocumentBarcode"]:

370 """

371 Returns the stored barcodes for the document, like

372 get_effective_content(): for root documents those of the latest

373 version when there is one, as that is the file users see.

374 """

375 from documents.versioning import latest_version

376

377 return list(latest_version(self).barcodes.all())

378

379 def get_effective_content(self) -> str | None:

380 """

381 Returns the effective content for the document.

382

383 For root documents, this is the latest version's content when available.

384 For version documents, this is always the document's own content.

385 If the queryset already annotated ``effective_content``, that value is used.

386 """

387 # Here to avoid circular import

388 from documents.versioning import LATEST_VERSION_CONTENT_PREFETCH_ATTR

389 from documents.versioning import sort_versions_newest_first

390 from documents.versioning import versions_newest_first

391

392 if hasattr(self, "effective_content"):

393 return getattr(self, "effective_content")

394

395 if self.root_document_id is not None or self.pk is None:

396 return self.content

397

398 latest_version_prefetch = getattr(

399 self,

400 LATEST_VERSION_CONTENT_PREFETCH_ATTR,

401 None,

402 )

403 if latest_version_prefetch is not None:

404 # Empty list means prefetch ran and found no versions — use own content.

405 return (

406 latest_version_prefetch[0].content

407 if latest_version_prefetch

408 else self.content

409 )

410

411 prefetched_cache = getattr(self, "_prefetched_objects_cache", None)

412 prefetched_versions = (

413 prefetched_cache.get("versions")

414 if isinstance(prefetched_cache, dict)

415 else None

416 )

417 if prefetched_versions is not None:

418 # Empty list means prefetch ran and found no versions — use own content.

419 if not prefetched_versions:

420 return self.content

421 return sort_versions_newest_first(prefetched_versions)[0].content

422

423 latest_version_content = (

424 versions_newest_first(Document.objects.filter(root_document=self))

425 .values_list("content", flat=True)

426 .first()

427 )

428 return (

429 latest_version_content

430 if latest_version_content is not None

431 else self.content

432 )

433

434 @property

435 def suggestion_content(self):

436 """

437 Returns the document text used to generate suggestions.

438

439 If the document content length exceeds a specified limit,

440 the text is cropped to include the start and end segments.

441 Otherwise, the full content is returned.

442

443 This improves processing speed for large documents while keeping

444 enough context for accurate suggestions.

445 """

446 effective_content = self.get_effective_content()

447 if not effective_content or len(effective_content) <= 1200000:

448 return effective_content

449 else:

450 # Use 80% from the start and 20% from the end

451 # to preserve both opening and closing context.

452 head_len = 800000

453 tail_len = 200000

454

455 return " ".join(

456 (

457 effective_content[:head_len],

458 effective_content[-tail_len:],

459 ),

460 )

461

462 @property

463 def source_path(self) -> Path:

464 fname = str(self.filename) if self.filename else f"{self.pk:07}{self.file_type}"

465

466 return (settings.ORIGINALS_DIR / Path(fname)).resolve()

467

468 @property

469 def source_file(self):

470 return Path(self.source_path).open("rb")

471

472 @property

473 def has_archive_version(self) -> bool:

474 return self.archive_filename is not None

475

476 @property

477 def archive_path(self) -> Path | None:

478 if self.has_archive_version: 478 ↛ 479line 478 didn't jump to line 479 because the condition on line 478 was never true

479 return (settings.ARCHIVE_DIR / Path(str(self.archive_filename))).resolve()

480 else:

481 return None

482

483 @property

484 def archive_file(self):

485 return Path(self.archive_path).open("rb")

486

487 def get_public_filename(self, *, archive=False, counter=0, suffix=None) -> str:

488 """

489 Returns a sanitized filename for the document, not including any paths.

490 """

491 # Root owns metadata for all versions

492 context_document = (

493 self.root_document if self.root_document_id is not None else self

494 )

495 result = str(context_document)

496

497 if counter: 497 ↛ 498line 497 didn't jump to line 498 because the condition on line 497 was never true

498 result += f"_{counter:02}"

499

500 if suffix: 500 ↛ 501line 500 didn't jump to line 501 because the condition on line 500 was never true

501 result += suffix

502

503 if archive: 503 ↛ 504line 503 didn't jump to line 504 because the condition on line 503 was never true

504 result += ".pdf"

505 else:

506 result += self.file_type

507

508 return pathvalidate.sanitize_filename(result, replacement_text="-")

509

510 @property

511 def file_type(self):

512 return get_default_file_extension(self.mime_type)

513

514 @property

515 def thumbnail_path(self) -> Path:

516 webp_file_name = f"{self.pk:07}.webp"

517

518 webp_file_path = settings.THUMBNAIL_DIR / Path(webp_file_name)

519

520 return webp_file_path.resolve()

521

522 @property

523 def thumbnail_file(self):

524 return Path(self.thumbnail_path).open("rb")

525

526 @property

527 def created_date(self):

528 return self.created

529

530 def add_nested_tags(self, tags) -> None:

531 tag_ids = set()

532 for tag in tags:

533 tag_ids.add(tag.id)

534 tag_ids.update(tag.get_ancestors_pks())

535

536 tags_to_add = self.tags.model.objects.filter(id__in=tag_ids)

537 self.tags.add(*tags_to_add)

538

539 def delete(

540 self,

541 *args,

542 transaction_id=None,

543 **kwargs,

544 ):

545 # Versions must share the root's transaction ID so they are restored

546 # together by django-softdelete.

547 if transaction_id is None: 547 ↛ 548line 547 didn't jump to line 548 because the condition on line 547 was never true

548 transaction_id = uuid.uuid4()

549 if self.root_document_id is None: 549 ↛ 553line 549 didn't jump to line 553 because the condition on line 549 was always true

550 Document.objects.filter(root_document=self).delete(

551 transaction_id=transaction_id,

552 )

553 return super().delete(

554 *args,

555 transaction_id=transaction_id,

556 **kwargs,

557 )

558

559

560class SavedView(ModelWithOwner):

561 class Icon(models.TextChoices):

562 ARCHIVE = ("archive", _("Archive"))

563 BANK = ("bank", _("Bank"))

564 BASKET = ("basket", _("Basket"))

565 BELL = ("bell", _("Bell"))

566 BOOKMARK = ("bookmark", _("Bookmark"))

567 BOXES = ("boxes", _("Boxes"))

568 BRIEFCASE = ("briefcase", _("Briefcase"))

569 BUILDING = ("building", _("Building"))

570 CALCULATOR = ("calculator", _("Calculator"))

571 CALENDAR = ("calendar", _("Calendar"))

572 CAMERA = ("camera", _("Camera"))

573 CARD_CHECKLIST = ("card-checklist", _("Checklist"))

574 CASH = ("cash", _("Cash"))

575 CHAT_LEFT_TEXT = ("chat-left-text", _("Chat"))

576 CHECK_CIRCLE = ("check-circle", _("Check"))

577 CLIPBOARD = ("clipboard", _("Clipboard"))

578 CLOCK_HISTORY = ("clock-history", _("Clock"))

579 CREDIT_CARD = ("credit-card", _("Credit card"))

580 DOWNLOAD = ("download", _("Download"))

581 ENVELOPE = ("envelope", _("Envelope"))

582 EXCLAMATION_TRIANGLE = ("exclamation-triangle", _("Warning"))

583 FILE_EARMARK = ("file-earmark", _("File"))

584 FILE_EARMARK_CHECK = ("file-earmark-check", _("Checked file"))

585 FILE_EARMARK_LOCK = ("file-earmark-lock", _("Locked file"))

586 FILE_EARMARK_MEDICAL = ("file-earmark-medical", _("Medical file"))

587 FILE_EARMARK_PERSON = ("file-earmark-person", _("Person file"))

588 FILE_EARMARK_SPREADSHEET = (

589 "file-earmark-spreadsheet",

590 _("Spreadsheet"),

591 )

592 FILE_TEXT = ("file-text", _("Text file"))

593 FILES = ("files", _("Files"))

594 FOLDER = ("folder", _("Folder"))

595 FUNNEL = ("funnel", _("Filter"))

596 GEAR = ("gear", _("Gear"))

597 GLOBE = ("globe2", _("Globe"))

598 HASH = ("hash", _("Hash"))

599 HEART = ("heart", _("Heart"))

600 HOUSE = ("house", _("House"))

601 INBOX = ("inbox", _("Inbox"))

602 JOURNALS = ("journals", _("Journals"))

603 LIST_TASK = ("list-task", _("Task list"))

604 NEWSPAPER = ("newspaper", _("Newspaper"))

605 PAPERCLIP = ("paperclip", _("Attachment"))

606 PEOPLE = ("people", _("People"))

607 PERSON = ("person", _("Person"))

608 PRINTER = ("printer", _("Printer"))

609 RECEIPT = ("receipt", _("Receipt"))

610 SAFE = ("safe", _("Safe"))

611 SEARCH = ("search", _("Search"))

612 SEND = ("send", _("Send"))

613 SHOP = ("shop", _("Shop"))

614 STACK = ("stack", _("Stack"))

615 STARS = ("stars", _("Stars"))

616 TAG = ("tag", _("Tag"))

617 TAGS = ("tags", _("Tags"))

618 TELEPHONE = ("telephone", _("Telephone"))

619 TRUCK = ("truck", _("Truck"))

620 UPC_SCAN = ("upc-scan", _("Barcode"))

621 WALLET = ("wallet2", _("Wallet"))

622

623 class DisplayMode(models.TextChoices):

624 TABLE = ("table", _("Table"))

625 SMALL_CARDS = ("smallCards", _("Small Cards"))

626 LARGE_CARDS = ("largeCards", _("Large Cards"))

627

628 class DisplayFields(models.TextChoices):

629 TITLE = ("title", _("Title"))

630 CREATED = ("created", _("Created"))

631 ADDED = ("added", _("Added"))

632 TAGS = ("tag"), _("Tags")

633 CORRESPONDENT = ("correspondent", _("Correspondent"))

634 DOCUMENT_TYPE = ("documenttype", _("Document Type"))

635 STORAGE_PATH = ("storagepath", _("Storage Path"))

636 NOTES = ("note", _("Note"))

637 OWNER = ("owner", _("Owner"))

638 SHARED = ("shared", _("Shared"))

639 ASN = ("asn", _("ASN"))

640 PAGE_COUNT = ("pagecount", _("Pages"))

641 CUSTOM_FIELD = ("custom_field_%d", ("Custom Field"))

642

643 name = models.CharField(_("name"), max_length=128)

644

645 icon = models.CharField(

646 _("icon"),

647 max_length=64,

648 choices=Icon.choices,

649 default=Icon.FUNNEL,

650 )

651

652 sort_field = models.CharField(

653 _("sort field"),

654 max_length=128,

655 null=True,

656 blank=True,

657 )

658 sort_reverse = models.BooleanField(_("sort reverse"), default=False)

659

660 page_size = models.PositiveIntegerField(

661 _("View page size"),

662 null=True,

663 blank=True,

664 validators=[MinValueValidator(1)],

665 )

666

667 display_mode = models.CharField(

668 max_length=128,

669 verbose_name=_("View display mode"),

670 choices=DisplayMode.choices,

671 null=True,

672 blank=True,

673 )

674

675 display_fields = models.JSONField(

676 verbose_name=_("Document display fields"),

677 null=True,

678 blank=True,

679 )

680

681 class Meta:

682 ordering = ("name",)

683 verbose_name = _("saved view")

684 verbose_name_plural = _("saved views")

685

686 def __str__(self):

687 return f"SavedView {self.name}"

688

689

690class SavedViewFilterRule(models.Model):

691 RULE_TYPES = [

692 (0, _("title contains")),

693 (1, _("content contains")),

694 (2, _("ASN is")),

695 (3, _("correspondent is")),

696 (4, _("document type is")),

697 (5, _("is in inbox")),

698 (6, _("has tag")),

699 (7, _("has any tag")),

700 (8, _("created before")),

701 (9, _("created after")),

702 (10, _("created year is")),

703 (11, _("created month is")),

704 (12, _("created day is")),

705 (13, _("added before")),

706 (14, _("added after")),

707 (15, _("modified before")),

708 (16, _("modified after")),

709 (17, _("does not have tag")),

710 (18, _("does not have ASN")),

711 (19, _("title or content contains")),

712 (20, _("fulltext query")),

713 (21, _("more like this")),

714 (22, _("has tags in")),

715 (23, _("ASN greater than")),

716 (24, _("ASN less than")),

717 (25, _("storage path is")),

718 (26, _("has correspondent in")),

719 (27, _("does not have correspondent in")),

720 (28, _("has document type in")),

721 (29, _("does not have document type in")),

722 (30, _("has storage path in")),

723 (31, _("does not have storage path in")),

724 (32, _("owner is")),

725 (33, _("has owner in")),

726 (34, _("does not have owner")),

727 (35, _("does not have owner in")),

728 (36, _("has custom field value")),

729 (37, _("is shared by me")),

730 (38, _("has custom fields")),

731 (39, _("has custom field in")),

732 (40, _("does not have custom field in")),

733 (41, _("does not have custom field")),

734 (42, _("custom fields query")),

735 (43, _("created to")),

736 (44, _("created from")),

737 (45, _("added to")),

738 (46, _("added from")),

739 (47, _("mime type is")),

740 (48, _("simple title search")),

741 (49, _("simple text search")),

742 (50, _("has duplicates")),

743 ]

744

745 saved_view = models.ForeignKey(

746 SavedView,

747 on_delete=models.CASCADE,

748 related_name="filter_rules",

749 verbose_name=_("saved view"),

750 )

751

752 rule_type = models.PositiveSmallIntegerField(_("rule type"), choices=RULE_TYPES)

753

754 value = models.CharField(_("value"), max_length=255, blank=True, null=True)

755

756 class Meta:

757 verbose_name = _("filter rule")

758 verbose_name_plural = _("filter rules")

759

760 def __str__(self) -> str:

761 return f"SavedViewFilterRule: {self.rule_type} : {self.value}"

762

763

764# Extending User Model Using a One-To-One Link

765class UiSettings(models.Model):

766 user = models.OneToOneField(

767 User,

768 on_delete=models.CASCADE,

769 related_name="ui_settings",

770 )

771 settings = models.JSONField(null=True)

772

773 def __str__(self):

774 return self.user.username

775

776

777class PaperlessTask(ModelWithOwner):

778 """

779 Tracks background task execution for user visibility and debugging.

780

781 State transitions:

782 PENDING -> STARTED -> SUCCESS

783 PENDING -> STARTED -> FAILURE

784 PENDING -> REVOKED (if cancelled before starting)

785 """

786

787 class Status(models.TextChoices):

788 PENDING = "pending", _("Pending")

789 STARTED = "started", _("Started")

790 SUCCESS = "success", _("Success")

791 FAILURE = "failure", _("Failure")

792 REVOKED = "revoked", _("Revoked")

793

794 class TaskType(models.TextChoices):

795 CONSUME_FILE = "consume_file", _("Consume File")

796 TRAIN_CLASSIFIER = "train_classifier", _("Train Classifier")

797 SANITY_CHECK = "sanity_check", _("Sanity Check")

798 INDEX_OPTIMIZE = "index_optimize", _("Index Optimize")

799 MAIL_FETCH = "mail_fetch", _("Mail Fetch")

800 LLM_INDEX = "llm_index", _("LLM Index")

801 EMPTY_TRASH = "empty_trash", _("Empty Trash")

802 CHECK_WORKFLOWS = "check_workflows", _("Check Workflows")

803 BULK_UPDATE = "bulk_update", _("Bulk Update")

804 REPROCESS_DOCUMENT = "reprocess_document", _("Reprocess Document")

805 BUILD_SHARE_LINK = "build_share_link", _("Build Share Link")

806 BULK_DELETE = "bulk_delete", _("Bulk Delete")

807 APPLY_AI_SUGGESTIONS = "apply_ai_suggestions", _("Apply AI Suggestions")

808

809 COMPLETE_STATUSES = (

810 Status.SUCCESS,

811 Status.FAILURE,

812 Status.REVOKED,

813 )

814

815 class TriggerSource(models.TextChoices):

816 SCHEDULED = "scheduled", _("Scheduled") # Celery beat

817 WEB_UI = "web_ui", _("Web UI") # Document uploaded via web

818 API_UPLOAD = "api_upload", _("API Upload") # Document uploaded via API

819 FOLDER_CONSUME = "folder_consume", _("Folder Consume") # Consume folder

820 EMAIL_CONSUME = "email_consume", _("Email Consume") # Email attachment

821 SYSTEM = "system", _("System") # Auto-triggered (self-heal, config side-effect)

822 MANUAL = "manual", _("Manual") # User explicitly ran via /api/tasks/run/

823

824 # Identification

825 task_id = models.CharField(

826 max_length=72,

827 unique=True,

828 verbose_name=_("Task ID"),

829 help_text=_("Celery task ID"),

830 )

831

832 task_type = models.CharField(

833 max_length=50,

834 choices=TaskType.choices,

835 verbose_name=_("Task Type"),

836 help_text=_("The kind of work being performed"),

837 db_index=True,

838 )

839

840 trigger_source = models.CharField(

841 max_length=50,

842 choices=TriggerSource.choices,

843 verbose_name=_("Trigger Source"),

844 help_text=_("What initiated this task"),

845 db_index=True,

846 )

847

848 # State tracking

849 status = models.CharField(

850 max_length=30,

851 choices=Status.choices,

852 default=Status.PENDING,

853 verbose_name=_("Status"),

854 db_index=True,

855 )

856

857 # Timestamps

858 date_created = models.DateTimeField(

859 default=timezone.now,

860 verbose_name=_("Created"),

861 db_index=True,

862 )

863

864 date_started = models.DateTimeField(

865 null=True,

866 blank=True,

867 verbose_name=_("Started"),

868 )

869

870 date_done = models.DateTimeField(

871 null=True,

872 blank=True,

873 verbose_name=_("Completed"),

874 db_index=True,

875 )

876

877 # Duration fields -- populated by task_postrun signal handler

878 duration_seconds = models.FloatField(

879 null=True,

880 blank=True,

881 verbose_name=_("Duration (seconds)"),

882 help_text=_("Elapsed time from start to completion"),

883 )

884

885 wait_time_seconds = models.FloatField(

886 null=True,

887 blank=True,

888 verbose_name=_("Wait Time (seconds)"),

889 help_text=_("Time from task creation to worker pickup"),

890 )

891

892 # Input/Output data

893 input_data = models.JSONField(

894 default=dict,

895 blank=True,

896 verbose_name=_("Input Data"),

897 help_text=_("Structured input parameters for the task"),

898 )

899

900 result_data = models.JSONField(

901 null=True,

902 blank=True,

903 verbose_name=_("Result Data"),

904 help_text=_("Structured result data from task execution"),

905 )

906

907 # Acknowledgment

908 acknowledged = models.BooleanField(

909 default=False,

910 verbose_name=_("Acknowledged"),

911 db_index=True,

912 )

913

914 class Meta:

915 verbose_name = _("Task")

916 verbose_name_plural = _("Tasks")

917 ordering = ["-date_created"]

918 indexes = [

919 models.Index(fields=["status", "date_created"]),

920 models.Index(fields=["task_type", "status"]),

921 models.Index(fields=["owner", "acknowledged", "date_created"]),

922 ]

923

924 def __str__(self) -> str: # pragma: no cover

925 return f"{self.get_task_type_display()} [{self.task_id[:8]}]"

926

927 @property

928 def is_complete(self) -> bool: # pragma: no cover

929 return self.status in self.COMPLETE_STATUSES

930

931 @property

932 def related_document_ids(self) -> list[int]: # pragma: no cover

933 if not self.result_data: 933 ↛ 935line 933 didn't jump to line 935 because the condition on line 933 was always true

934 return []

935 if doc_id := self.result_data.get("document_id"):

936 return [doc_id]

937 if dup_id := self.result_data.get("duplicate_of"):

938 return [dup_id]

939 return []

940

941

942class Note(SoftDeleteModel):

943 note = models.TextField(

944 _("content"),

945 blank=True,

946 help_text=_("Note for the document"),

947 )

948

949 created = models.DateTimeField(

950 _("created"),

951 default=timezone.now,

952 db_index=True,

953 )

954

955 document = models.ForeignKey(

956 Document,

957 blank=True,

958 null=True,

959 related_name="notes",

960 on_delete=models.CASCADE,

961 verbose_name=_("document"),

962 )

963

964 user = models.ForeignKey(

965 User,

966 blank=True,

967 null=True,

968 related_name="notes",

969 on_delete=models.SET_NULL,

970 verbose_name=_("user"),

971 )

972

973 class Meta:

974 ordering = ("created",)

975 verbose_name = _("note")

976 verbose_name_plural = _("notes")

977

978 def __str__(self):

979 return self.note

980

981

982class DocumentBarcode(models.Model):

983 """

984 A barcode found in a document during consumption, kept so its content

985 can be shown and copied

986 """

987

988 document = models.ForeignKey(

989 Document,

990 related_name="barcodes",

991 on_delete=models.CASCADE,

992 verbose_name=_("document"),

993 )

994

995 page = models.PositiveIntegerField(

996 _("page"),

997 help_text=_("Page of the original file, starting at 1"),

998 )

999

1000 value = models.TextField(_("value"))

1001

1002 class Format(models.TextChoices):

1003 """

1004 The concrete barcode formats of zxing-cpp, keyed on the enum name.

1005 The labels are symbology names and aren't translated.

1006 """

1007

1008 CODABAR = "Codabar", "Codabar"

1009 CODE39 = "Code39", "Code 39"

1010 CODE39_STD = "Code39Std", "Code 39 Standard"

1011 CODE39_EXT = "Code39Ext", "Code 39 Extended"

1012 CODE32 = "Code32", "Code 32"

1013 PZN = "PZN", "Pharmazentralnummer"

1014 CODE93 = "Code93", "Code 93"

1015 CODE128 = "Code128", "Code 128"

1016 ITF = "ITF", "ITF"

1017 ITF14 = "ITF14", "ITF-14"

1018 DATA_BAR = "DataBar", "DataBar"

1019 DATA_BAR_OMNI = "DataBarOmni", "DataBar Omni"

1020 DATA_BAR_STK = "DataBarStk", "DataBar Stacked"

1021 DATA_BAR_STK_OMNI = "DataBarStkOmni", "DataBar Stacked Omni"

1022 DATA_BAR_LTD = "DataBarLtd", "DataBar Limited"

1023 DATA_BAR_EXP = "DataBarExp", "DataBar Expanded"

1024 DATA_BAR_EXP_STK = "DataBarExpStk", "DataBar Expanded Stacked"

1025 EANUPC = "EANUPC", "EAN/UPC"

1026 EAN13 = "EAN13", "EAN-13"

1027 EAN8 = "EAN8", "EAN-8"

1028 EAN5 = "EAN5", "EAN-5"

1029 EAN2 = "EAN2", "EAN-2"

1030 ISBN = "ISBN", "ISBN"

1031 UPCA = "UPCA", "UPC-A"

1032 UPCE = "UPCE", "UPC-E"

1033 TELEPEN = "Telepen", "Telepen"

1034 TELEPEN_ALPHA = "TelepenAlpha", "Telepen Alpha"

1035 TELEPEN_NUMERIC = "TelepenNumeric", "Telepen Numeric"

1036 OTHER_BARCODE = "OtherBarcode", "Other barcode"

1037 DX_FILM_EDGE = "DXFilmEdge", "DX Film Edge"

1038 PDF417 = "PDF417", "PDF417"

1039 COMPACT_PDF417 = "CompactPDF417", "Compact PDF417"

1040 MICRO_PDF417 = "MicroPDF417", "MicroPDF417"

1041 AZTEC = "Aztec", "Aztec"

1042 AZTEC_CODE = "AztecCode", "Aztec Code"

1043 AZTEC_RUNE = "AztecRune", "Aztec Rune"

1044 QR_CODE = "QRCode", "QR Code"

1045 QR_CODE_MODEL1 = "QRCodeModel1", "QR Code Model 1"

1046 QR_CODE_MODEL2 = "QRCodeModel2", "QR Code Model 2"

1047 MICRO_QR_CODE = "MicroQRCode", "Micro QR Code"

1048 RMQR_CODE = "RMQRCode", "rMQR Code"

1049 DATA_MATRIX = "DataMatrix", "Data Matrix"

1050 MAXI_CODE = "MaxiCode", "MaxiCode"

1051

1052 format = models.CharField(_("format"), max_length=32, choices=Format.choices)

1053

1054 class Meta:

1055 ordering = ("page", "id")

1056 verbose_name = _("document barcode")

1057 verbose_name_plural = _("document barcodes")

1058

1059 def __str__(self) -> str: # pragma: no cover

1060 return self.value

1061

1062

1063class ShareLink(SoftDeleteModel):

1064 class FileVersion(models.TextChoices):

1065 ARCHIVE = ("archive", _("Archive"))

1066 ORIGINAL = ("original", _("Original"))

1067

1068 created = models.DateTimeField(

1069 _("created"),

1070 default=timezone.now,

1071 db_index=True,

1072 blank=True,

1073 editable=False,

1074 )

1075

1076 expiration = models.DateTimeField(

1077 _("expiration"),

1078 blank=True,

1079 null=True,

1080 db_index=True,

1081 )

1082

1083 slug = models.SlugField(

1084 _("slug"),

1085 db_index=True,

1086 unique=True,

1087 blank=True,

1088 editable=False,

1089 )

1090

1091 document = models.ForeignKey(

1092 Document,

1093 blank=True,

1094 related_name="share_links",

1095 on_delete=models.CASCADE,

1096 verbose_name=_("document"),

1097 )

1098

1099 file_version = models.CharField(

1100 max_length=50,

1101 choices=FileVersion.choices,

1102 default=FileVersion.ARCHIVE,

1103 )

1104

1105 owner = models.ForeignKey(

1106 User,

1107 blank=True,

1108 null=True,

1109 related_name="share_links",

1110 on_delete=models.SET_NULL,

1111 verbose_name=_("owner"),

1112 )

1113

1114 class Meta:

1115 ordering = ("created",)

1116 verbose_name = _("share link")

1117 verbose_name_plural = _("share links")

1118

1119 def __str__(self):

1120 return f"Share Link for {self.document.title}"

1121

1122

1123class ShareLinkBundle(models.Model):

1124 class Status(models.TextChoices):

1125 PENDING = ("pending", _("Pending"))

1126 PROCESSING = ("processing", _("Processing"))

1127 READY = ("ready", _("Ready"))

1128 FAILED = ("failed", _("Failed"))

1129

1130 created = models.DateTimeField(

1131 _("created"),

1132 default=timezone.now,

1133 db_index=True,

1134 blank=True,

1135 editable=False,

1136 )

1137

1138 expiration = models.DateTimeField(

1139 _("expiration"),

1140 blank=True,

1141 null=True,

1142 db_index=True,

1143 )

1144

1145 slug = models.SlugField(

1146 _("slug"),

1147 db_index=True,

1148 unique=True,

1149 blank=True,

1150 editable=False,

1151 )

1152

1153 owner = models.ForeignKey(

1154 User,

1155 blank=True,

1156 null=True,

1157 related_name="share_link_bundles",

1158 on_delete=models.SET_NULL,

1159 verbose_name=_("owner"),

1160 )

1161

1162 file_version = models.CharField(

1163 max_length=50,

1164 choices=ShareLink.FileVersion.choices,

1165 default=ShareLink.FileVersion.ARCHIVE,

1166 )

1167

1168 status = models.CharField(

1169 max_length=50,

1170 choices=Status.choices,

1171 default=Status.PENDING,

1172 )

1173

1174 size_bytes = models.PositiveIntegerField(

1175 _("size (bytes)"),

1176 blank=True,

1177 null=True,

1178 )

1179

1180 last_error = models.JSONField(

1181 _("last error"),

1182 blank=True,

1183 null=True,

1184 default=None,

1185 )

1186

1187 file_path = models.CharField(

1188 _("file path"),

1189 max_length=512,

1190 blank=True,

1191 )

1192

1193 built_at = models.DateTimeField(

1194 _("built at"),

1195 null=True,

1196 blank=True,

1197 )

1198

1199 documents = models.ManyToManyField(

1200 "documents.Document",

1201 related_name="share_link_bundles",

1202 verbose_name=_("documents"),

1203 )

1204

1205 class Meta:

1206 ordering = ("-created",)

1207 verbose_name = _("share link bundle")

1208 verbose_name_plural = _("share link bundles")

1209

1210 def __str__(self):

1211 return _("Share link bundle %(slug)s") % {"slug": self.slug}

1212

1213 @property

1214 def absolute_file_path(self) -> Path | None:

1215 if not self.file_path:

1216 return None

1217 relative_path = Path(self.file_path)

1218 if relative_path.is_absolute():

1219 return None

1220

1221 bundle_dir = settings.SHARE_LINK_BUNDLE_DIR.resolve()

1222 absolute_path = (bundle_dir / relative_path).resolve()

1223 try:

1224 absolute_path.relative_to(bundle_dir)

1225 except ValueError:

1226 return None

1227 return absolute_path

1228

1229 def remove_file(self) -> None:

1230 if self.absolute_file_path is not None and self.absolute_file_path.exists():

1231 try:

1232 self.absolute_file_path.unlink()

1233 except OSError:

1234 pass

1235

1236 def delete(self, using=None, *, keep_parents=False):

1237 self.remove_file()

1238 return super().delete(using=using, keep_parents=keep_parents)

1239

1240

1241class CustomField(models.Model):

1242 """

1243 Defines the name and type of a custom field

1244 """

1245

1246 class FieldDataType(models.TextChoices):

1247 STRING = ("string", _("String"))

1248 URL = ("url", _("URL"))

1249 DATE = ("date", _("Date"))

1250 BOOL = ("boolean"), _("Boolean")

1251 INT = ("integer", _("Integer"))

1252 FLOAT = ("float", _("Float"))

1253 MONETARY = ("monetary", _("Monetary"))

1254 DOCUMENTLINK = ("documentlink", _("Document Link"))

1255 SELECT = ("select", _("Select"))

1256 LONG_TEXT = ("longtext", _("Long Text"))

1257

1258 created = models.DateTimeField(

1259 _("created"),

1260 default=timezone.now,

1261 db_index=True,

1262 editable=False,

1263 )

1264

1265 name = models.CharField(max_length=128)

1266

1267 data_type = models.CharField(

1268 _("data type"),

1269 max_length=50,

1270 choices=FieldDataType.choices,

1271 editable=False,

1272 )

1273

1274 extra_data = models.JSONField(

1275 _("extra data"),

1276 null=True,

1277 blank=True,

1278 help_text=_(

1279 "Extra data for the custom field, such as select options",

1280 ),

1281 )

1282

1283 class Meta:

1284 ordering = ("created",)

1285 verbose_name = _("custom field")

1286 verbose_name_plural = _("custom fields")

1287 constraints = [

1288 models.UniqueConstraint(

1289 fields=["name"],

1290 name="%(app_label)s_%(class)s_unique_name",

1291 ),

1292 ]

1293

1294 def __str__(self) -> str:

1295 return f"{self.name} : {self.data_type}"

1296

1297

1298class CustomFieldInstance(SoftDeleteModel):

1299 """

1300 A single instance of a field, attached to a CustomField for the name and type

1301 and attached to a single Document to be metadata for it

1302 """

1303

1304 TYPE_TO_DATA_STORE_NAME_MAP = {

1305 CustomField.FieldDataType.STRING: "value_text",

1306 CustomField.FieldDataType.URL: "value_url",

1307 CustomField.FieldDataType.DATE: "value_date",

1308 CustomField.FieldDataType.BOOL: "value_bool",

1309 CustomField.FieldDataType.INT: "value_int",

1310 CustomField.FieldDataType.FLOAT: "value_float",

1311 CustomField.FieldDataType.MONETARY: "value_monetary",

1312 CustomField.FieldDataType.DOCUMENTLINK: "value_document_ids",

1313 CustomField.FieldDataType.SELECT: "value_select",

1314 CustomField.FieldDataType.LONG_TEXT: "value_long_text",

1315 }

1316

1317 created = models.DateTimeField(

1318 _("created"),

1319 default=timezone.now,

1320 db_index=True,

1321 editable=False,

1322 )

1323

1324 document = models.ForeignKey(

1325 Document,

1326 blank=False,

1327 null=False,

1328 on_delete=models.CASCADE,

1329 related_name="custom_fields",

1330 editable=False,

1331 )

1332

1333 field = models.ForeignKey(

1334 CustomField,

1335 blank=False,

1336 null=False,

1337 on_delete=models.CASCADE,

1338 related_name="fields",

1339 editable=False,

1340 )

1341

1342 # Actual data storage

1343 value_text = models.CharField(max_length=128, null=True)

1344

1345 value_bool = models.BooleanField(null=True)

1346

1347 value_url = models.URLField(null=True)

1348

1349 value_date = models.DateField(null=True)

1350

1351 value_int = models.IntegerField(null=True)

1352

1353 value_float = models.FloatField(null=True)

1354

1355 value_monetary = models.CharField(null=True, max_length=128)

1356

1357 value_monetary_amount = models.GeneratedField(

1358 expression=Case(

1359 # If the value starts with a number and no currency symbol, use the whole string

1360 models.When(

1361 value_monetary__regex=r"^\d+",

1362 then=Cast(

1363 Substr("value_monetary", 1),

1364 output_field=models.DecimalField(decimal_places=2, max_digits=65),

1365 ),

1366 ),

1367 # If the value starts with a 3-char currency symbol, use the rest of the string

1368 default=Cast(

1369 Substr("value_monetary", 4),

1370 output_field=models.DecimalField(decimal_places=2, max_digits=65),

1371 ),

1372 output_field=models.DecimalField(decimal_places=2, max_digits=65),

1373 ),

1374 output_field=models.DecimalField(decimal_places=2, max_digits=65),

1375 db_persist=True,

1376 )

1377

1378 value_document_ids = models.JSONField(null=True)

1379

1380 value_select = models.CharField(null=True, max_length=16)

1381

1382 value_long_text = models.TextField(null=True)

1383

1384 class Meta:

1385 ordering = ("created",)

1386 verbose_name = _("custom field instance")

1387 verbose_name_plural = _("custom field instances")

1388 indexes = [

1389 models.Index(fields=["field", "value_date"]),

1390 models.Index(fields=["field", "value_int"]),

1391 models.Index(fields=["field", "value_float"]),

1392 models.Index(fields=["field", "value_monetary_amount"]),

1393 ]

1394 constraints = [

1395 models.UniqueConstraint(

1396 fields=["document", "field"],

1397 name="%(app_label)s_%(class)s_unique_document_field",

1398 ),

1399 ]

1400

1401 def __str__(self) -> str:

1402 return str(self.field.name) + f" : {self.value_for_search}"

1403

1404 @classmethod

1405 def get_value_field_name(cls, data_type: CustomField.FieldDataType):

1406 try:

1407 return cls.TYPE_TO_DATA_STORE_NAME_MAP[data_type]

1408 except KeyError: # pragma: no cover

1409 raise NotImplementedError(data_type)

1410

1411 @property

1412 def value(self):

1413 """

1414 Based on the data type, access the actual value the instance stores

1415 A little shorthand/quick way to get what is actually here

1416 """

1417 value_field_name = self.get_value_field_name(self.field.data_type)

1418 return getattr(self, value_field_name)

1419

1420 @property

1421 def value_for_search(self) -> str | None:

1422 """

1423 Return the value suitable for full-text indexing and display, or None

1424 if the value is unset.

1425

1426 For SELECT fields, resolves the human-readable label rather than the

1427 opaque option ID stored in value_select.

1428 """

1429 if self.value is None:

1430 return None

1431 if self.field.data_type == CustomField.FieldDataType.SELECT:

1432 options = (self.field.extra_data or {}).get("select_options", [])

1433 return next(

1434 (o["label"] for o in options if o.get("id") == self.value),

1435 None,

1436 )

1437 return str(self.value)

1438

1439

1440if settings.AUDIT_LOG_ENABLED: 1440 ↛ 1454line 1440 didn't jump to line 1454 because the condition on line 1440 was always true

1441 auditlog.register(

1442 Document,

1443 m2m_fields={"tags"},

1444 exclude_fields=["content_length", "modified"],

1445 )

1446 auditlog.register(Correspondent)

1447 auditlog.register(Tag)

1448 auditlog.register(DocumentType)

1449 auditlog.register(Note)

1450 auditlog.register(CustomField)

1451 auditlog.register(CustomFieldInstance)

1452

1453

1454class WorkflowTrigger(models.Model):

1455 class WorkflowTriggerMatching(models.IntegerChoices):

1456 # No auto matching

1457 NONE = MatchingModel.MATCH_NONE, _("None")

1458 ANY = MatchingModel.MATCH_ANY, _("Any word")

1459 ALL = MatchingModel.MATCH_ALL, _("All words")

1460 LITERAL = MatchingModel.MATCH_LITERAL, _("Exact match")

1461 REGEX = MatchingModel.MATCH_REGEX, _("Regular expression")

1462 FUZZY = MatchingModel.MATCH_FUZZY, _("Fuzzy word")

1463

1464 class WorkflowTriggerType(models.IntegerChoices):

1465 CONSUMPTION = 1, _("Consumption Started")

1466 DOCUMENT_ADDED = 2, _("Document Added")

1467 DOCUMENT_UPDATED = 3, _("Document Updated")

1468 SCHEDULED = 4, _("Scheduled")

1469

1470 class DocumentSourceChoices(models.IntegerChoices):

1471 CONSUME_FOLDER = DocumentSource.ConsumeFolder.value, _("Consume Folder")

1472 API_UPLOAD = DocumentSource.ApiUpload.value, _("Api Upload")

1473 MAIL_FETCH = DocumentSource.MailFetch.value, _("Mail Fetch")

1474 WEB_UI = DocumentSource.WebUI.value, _("Web UI")

1475

1476 class ScheduleDateField(models.TextChoices):

1477 ADDED = "added", _("Added")

1478 CREATED = "created", _("Created")

1479 MODIFIED = "modified", _("Modified")

1480 CUSTOM_FIELD = "custom_field", _("Custom Field")

1481

1482 type = models.PositiveSmallIntegerField(

1483 _("Workflow Trigger Type"),

1484 choices=WorkflowTriggerType.choices,

1485 default=WorkflowTriggerType.CONSUMPTION,

1486 )

1487

1488 sources = MultiSelectField(

1489 max_length=7,

1490 choices=DocumentSourceChoices.choices,

1491 default=f"{DocumentSource.ConsumeFolder},{DocumentSource.ApiUpload},{DocumentSource.MailFetch},{DocumentSource.WebUI}",

1492 )

1493

1494 filter_path = models.CharField(

1495 _("filter path"),

1496 max_length=256,

1497 null=True,

1498 blank=True,

1499 help_text=_(

1500 "Only consume documents with a path that matches "

1501 "this if specified. Wildcards specified as * are "

1502 "allowed. Case insensitive.",

1503 ),

1504 )

1505

1506 filter_filename = models.CharField(

1507 _("filter filename"),

1508 max_length=256,

1509 null=True,

1510 blank=True,

1511 help_text=_(

1512 "Only consume documents which entirely match this "

1513 "filename if specified. Wildcards such as *.pdf or "

1514 "*invoice* are allowed. Case insensitive.",

1515 ),

1516 )

1517

1518 filter_mailrule = models.ForeignKey(

1519 "paperless_mail.MailRule",

1520 null=True,

1521 blank=True,

1522 on_delete=models.SET_NULL,

1523 verbose_name=_("filter documents from this mail rule"),

1524 )

1525

1526 match = models.CharField(_("match"), max_length=256, blank=True)

1527

1528 matching_algorithm = models.PositiveSmallIntegerField(

1529 _("matching algorithm"),

1530 choices=WorkflowTriggerMatching.choices,

1531 default=WorkflowTriggerMatching.NONE,

1532 )

1533

1534 is_insensitive = models.BooleanField(_("is insensitive"), default=True)

1535

1536 filter_has_tags = models.ManyToManyField(

1537 Tag,

1538 blank=True,

1539 verbose_name=_("has these tag(s)"),

1540 )

1541

1542 filter_has_all_tags = models.ManyToManyField(

1543 Tag,

1544 blank=True,

1545 related_name="workflowtriggers_has_all",

1546 verbose_name=_("has all of these tag(s)"),

1547 )

1548

1549 filter_has_not_tags = models.ManyToManyField(

1550 Tag,

1551 blank=True,

1552 related_name="workflowtriggers_has_not",

1553 verbose_name=_("does not have these tag(s)"),

1554 )

1555

1556 filter_has_document_type = models.ForeignKey(

1557 DocumentType,

1558 null=True,

1559 blank=True,

1560 on_delete=models.SET_NULL,

1561 verbose_name=_("has this document type"),

1562 )

1563

1564 filter_has_any_document_types = models.ManyToManyField(

1565 DocumentType,

1566 blank=True,

1567 related_name="workflowtriggers_has_any_document_type",

1568 verbose_name=_("has one of these document types"),

1569 )

1570

1571 filter_has_not_document_types = models.ManyToManyField(

1572 DocumentType,

1573 blank=True,

1574 related_name="workflowtriggers_has_not_document_type",

1575 verbose_name=_("does not have these document type(s)"),

1576 )

1577

1578 filter_has_correspondent = models.ForeignKey(

1579 Correspondent,

1580 null=True,

1581 blank=True,

1582 on_delete=models.SET_NULL,

1583 verbose_name=_("has this correspondent"),

1584 )

1585

1586 filter_has_not_correspondents = models.ManyToManyField(

1587 Correspondent,

1588 blank=True,

1589 related_name="workflowtriggers_has_not_correspondent",

1590 verbose_name=_("does not have these correspondent(s)"),

1591 )

1592

1593 filter_has_any_correspondents = models.ManyToManyField(

1594 Correspondent,

1595 blank=True,

1596 related_name="workflowtriggers_has_any_correspondent",

1597 verbose_name=_("has one of these correspondents"),

1598 )

1599

1600 filter_has_storage_path = models.ForeignKey(

1601 StoragePath,

1602 null=True,

1603 blank=True,

1604 on_delete=models.SET_NULL,

1605 verbose_name=_("has this storage path"),

1606 )

1607

1608 filter_has_any_storage_paths = models.ManyToManyField(

1609 StoragePath,

1610 blank=True,

1611 related_name="workflowtriggers_has_any_storage_path",

1612 verbose_name=_("has one of these storage paths"),

1613 )

1614

1615 filter_has_not_storage_paths = models.ManyToManyField(

1616 StoragePath,

1617 blank=True,

1618 related_name="workflowtriggers_has_not_storage_path",

1619 verbose_name=_("does not have these storage path(s)"),

1620 )

1621

1622 filter_custom_field_query = models.TextField(

1623 _("filter custom field query"),

1624 null=True,

1625 blank=True,

1626 help_text=_("JSON-encoded custom field query expression."),

1627 )

1628

1629 schedule_offset_days = models.IntegerField(

1630 _("schedule offset days"),

1631 default=0,

1632 help_text=_(

1633 "The number of days to offset the schedule trigger by.",

1634 ),

1635 )

1636

1637 schedule_is_recurring = models.BooleanField(

1638 _("schedule is recurring"),

1639 default=False,

1640 help_text=_(

1641 "If the schedule should be recurring.",

1642 ),

1643 )

1644

1645 schedule_recurring_interval_days = models.PositiveIntegerField(

1646 _("schedule recurring delay in days"),

1647 default=1,

1648 validators=[MinValueValidator(1)],

1649 help_text=_(

1650 "The number of days between recurring schedule triggers.",

1651 ),

1652 )

1653

1654 schedule_date_field = models.CharField(

1655 _("schedule date field"),

1656 max_length=20,

1657 choices=ScheduleDateField.choices,

1658 default=ScheduleDateField.ADDED,

1659 help_text=_(

1660 "The field to check for a schedule trigger.",

1661 ),

1662 )

1663

1664 schedule_date_custom_field = models.ForeignKey(

1665 CustomField,

1666 null=True,

1667 blank=True,

1668 on_delete=models.SET_NULL,

1669 verbose_name=_("schedule date custom field"),

1670 )

1671

1672 class Meta:

1673 verbose_name = _("workflow trigger")

1674 verbose_name_plural = _("workflow triggers")

1675

1676 def __str__(self):

1677 return f"WorkflowTrigger {self.pk}"

1678

1679

1680class WorkflowActionEmail(models.Model):

1681 subject = models.CharField(

1682 _("email subject"),

1683 max_length=256,

1684 null=False,

1685 help_text=_(

1686 "The subject of the email, can include some placeholders, "

1687 "see documentation.",

1688 ),

1689 )

1690

1691 body = models.TextField(

1692 _("email body"),

1693 null=False,

1694 help_text=_(

1695 "The body (message) of the email, can include some placeholders, "

1696 "see documentation.",

1697 ),

1698 )

1699

1700 to = models.TextField(

1701 _("emails to"),

1702 null=False,

1703 help_text=_(

1704 "The destination email addresses, comma separated.",

1705 ),

1706 )

1707

1708 include_document = models.BooleanField(

1709 default=False,

1710 verbose_name=_("include document in email"),

1711 )

1712

1713 def __str__(self):

1714 return f"Workflow Email Action {self.pk}"

1715

1716

1717class WorkflowActionWebhook(models.Model):

1718 # We dont use the built-in URLField because it is not flexible enough

1719 # validation is handled in the serializer

1720 url = models.CharField(

1721 _("webhook url"),

1722 null=False,

1723 max_length=256,

1724 help_text=_("The destination URL for the notification."),

1725 )

1726

1727 use_params = models.BooleanField(

1728 default=True,

1729 verbose_name=_("use parameters"),

1730 )

1731

1732 as_json = models.BooleanField(

1733 default=False,

1734 verbose_name=_("send as JSON"),

1735 )

1736

1737 params = models.JSONField(

1738 _("webhook parameters"),

1739 null=True,

1740 blank=True,

1741 help_text=_("The parameters to send with the webhook URL if body not used."),

1742 )

1743

1744 body = models.TextField(

1745 _("webhook body"),

1746 null=True,

1747 blank=True,

1748 help_text=_("The body to send with the webhook URL if parameters not used."),

1749 )

1750

1751 headers = models.JSONField(

1752 _("webhook headers"),

1753 null=True,

1754 blank=True,

1755 help_text=_("The headers to send with the webhook URL."),

1756 )

1757

1758 include_document = models.BooleanField(

1759 default=False,

1760 verbose_name=_("include document in webhook"),

1761 )

1762

1763 def __str__(self):

1764 return f"Workflow Webhook Action {self.pk}"

1765

1766

1767class WorkflowAction(models.Model):

1768 class WorkflowActionType(models.IntegerChoices):

1769 ASSIGNMENT = (

1770 1,

1771 _("Assignment"),

1772 )

1773 REMOVAL = (

1774 2,

1775 _("Removal"),

1776 )

1777 EMAIL = (

1778 3,

1779 _("Email"),

1780 )

1781 WEBHOOK = (

1782 4,

1783 _("Webhook"),

1784 )

1785 PASSWORD_REMOVAL = (

1786 5,

1787 _("Password removal"),

1788 )

1789 MOVE_TO_TRASH = (

1790 6,

1791 _("Move to trash"),

1792 )

1793 REMOTE_OCR = (

1794 7,

1795 _("Remote OCR"),

1796 )

1797 APPLY_AI_SUGGESTIONS = (

1798 8,

1799 _("Apply AI suggestions"),

1800 )

1801

1802 class AISuggestionField(models.TextChoices):

1803 TITLE = ("title", _("Title"))

1804 TAGS = ("tags", _("Tags"))

1805 CORRESPONDENT = ("correspondent", _("Correspondent"))

1806 DOCUMENT_TYPE = ("document_type", _("Document type"))

1807 STORAGE_PATH = ("storage_path", _("Storage path"))

1808 CREATED = ("created", _("Created date"))

1809

1810 type = models.PositiveSmallIntegerField(

1811 _("Workflow Action Type"),

1812 choices=WorkflowActionType.choices,

1813 default=WorkflowActionType.ASSIGNMENT,

1814 )

1815

1816 order = models.PositiveIntegerField(_("order"), default=0)

1817

1818 assign_title = models.TextField(

1819 _("assign title"),

1820 null=True,

1821 blank=True,

1822 help_text=_(

1823 "Assign a document title, must be a Jinja2 template, see documentation.",

1824 ),

1825 )

1826

1827 assign_tags = models.ManyToManyField(

1828 Tag,

1829 blank=True,

1830 related_name="+",

1831 verbose_name=_("assign this tag"),

1832 )

1833

1834 assign_document_type = models.ForeignKey(

1835 DocumentType,

1836 null=True,

1837 blank=True,

1838 on_delete=models.SET_NULL,

1839 related_name="+",

1840 verbose_name=_("assign this document type"),

1841 )

1842

1843 assign_correspondent = models.ForeignKey(

1844 Correspondent,

1845 null=True,

1846 blank=True,

1847 on_delete=models.SET_NULL,

1848 related_name="+",

1849 verbose_name=_("assign this correspondent"),

1850 )

1851

1852 assign_storage_path = models.ForeignKey(

1853 StoragePath,

1854 null=True,

1855 blank=True,

1856 on_delete=models.SET_NULL,

1857 related_name="+",

1858 verbose_name=_("assign this storage path"),

1859 )

1860

1861 assign_owner = models.ForeignKey(

1862 User,

1863 null=True,

1864 blank=True,

1865 on_delete=models.SET_NULL,

1866 related_name="+",

1867 verbose_name=_("assign this owner"),

1868 )

1869

1870 assign_view_users = models.ManyToManyField(

1871 User,

1872 blank=True,

1873 related_name="+",

1874 verbose_name=_("grant view permissions to these users"),

1875 )

1876

1877 assign_view_groups = models.ManyToManyField(

1878 Group,

1879 blank=True,

1880 related_name="+",

1881 verbose_name=_("grant view permissions to these groups"),

1882 )

1883

1884 assign_change_users = models.ManyToManyField(

1885 User,

1886 blank=True,

1887 related_name="+",

1888 verbose_name=_("grant change permissions to these users"),

1889 )

1890

1891 assign_change_groups = models.ManyToManyField(

1892 Group,

1893 blank=True,

1894 related_name="+",

1895 verbose_name=_("grant change permissions to these groups"),

1896 )

1897

1898 assign_custom_fields = models.ManyToManyField(

1899 CustomField,

1900 blank=True,

1901 related_name="+",

1902 verbose_name=_("assign these custom fields"),

1903 )

1904

1905 assign_custom_fields_values = models.JSONField(

1906 _("custom field values"),

1907 null=True,

1908 blank=True,

1909 help_text=_(

1910 "Optional values to assign to the custom fields.",

1911 ),

1912 default=dict,

1913 )

1914

1915 remove_tags = models.ManyToManyField(

1916 Tag,

1917 blank=True,

1918 related_name="+",

1919 verbose_name=_("remove these tag(s)"),

1920 )

1921

1922 remove_all_tags = models.BooleanField(

1923 default=False,

1924 verbose_name=_("remove all tags"),

1925 )

1926

1927 remove_document_types = models.ManyToManyField(

1928 DocumentType,

1929 blank=True,

1930 related_name="+",

1931 verbose_name=_("remove these document type(s)"),

1932 )

1933

1934 remove_all_document_types = models.BooleanField(

1935 default=False,

1936 verbose_name=_("remove all document types"),

1937 )

1938

1939 remove_correspondents = models.ManyToManyField(

1940 Correspondent,

1941 blank=True,

1942 related_name="+",

1943 verbose_name=_("remove these correspondent(s)"),

1944 )

1945

1946 remove_all_correspondents = models.BooleanField(

1947 default=False,

1948 verbose_name=_("remove all correspondents"),

1949 )

1950

1951 remove_storage_paths = models.ManyToManyField(

1952 StoragePath,

1953 blank=True,

1954 related_name="+",

1955 verbose_name=_("remove these storage path(s)"),

1956 )

1957

1958 remove_all_storage_paths = models.BooleanField(

1959 default=False,

1960 verbose_name=_("remove all storage paths"),

1961 )

1962

1963 remove_owners = models.ManyToManyField(

1964 User,

1965 blank=True,

1966 related_name="+",

1967 verbose_name=_("remove these owner(s)"),

1968 )

1969

1970 remove_all_owners = models.BooleanField(

1971 default=False,

1972 verbose_name=_("remove all owners"),

1973 )

1974

1975 remove_view_users = models.ManyToManyField(

1976 User,

1977 blank=True,

1978 related_name="+",

1979 verbose_name=_("remove view permissions for these users"),

1980 )

1981

1982 remove_view_groups = models.ManyToManyField(

1983 Group,

1984 blank=True,

1985 related_name="+",

1986 verbose_name=_("remove view permissions for these groups"),

1987 )

1988

1989 remove_change_users = models.ManyToManyField(

1990 User,

1991 blank=True,

1992 related_name="+",

1993 verbose_name=_("remove change permissions for these users"),

1994 )

1995

1996 remove_change_groups = models.ManyToManyField(

1997 Group,

1998 blank=True,

1999 related_name="+",

2000 verbose_name=_("remove change permissions for these groups"),

2001 )

2002

2003 remove_all_permissions = models.BooleanField(

2004 default=False,

2005 verbose_name=_("remove all permissions"),

2006 )

2007

2008 remove_custom_fields = models.ManyToManyField(

2009 CustomField,

2010 blank=True,

2011 related_name="+",

2012 verbose_name=_("remove these custom fields"),

2013 )

2014

2015 remove_all_custom_fields = models.BooleanField(

2016 default=False,

2017 verbose_name=_("remove all custom fields"),

2018 )

2019

2020 email = models.ForeignKey(

2021 WorkflowActionEmail,

2022 null=True,

2023 blank=True,

2024 on_delete=models.SET_NULL,

2025 related_name="action",

2026 verbose_name=_("email"),

2027 )

2028

2029 webhook = models.ForeignKey(

2030 WorkflowActionWebhook,

2031 null=True,

2032 blank=True,

2033 on_delete=models.SET_NULL,

2034 related_name="action",

2035 verbose_name=_("webhook"),

2036 )

2037

2038 passwords = models.JSONField(

2039 _("passwords"),

2040 null=True,

2041 blank=True,

2042 help_text=_(

2043 "Passwords to try when removing PDF protection. Separate with commas or new lines.",

2044 ),

2045 )

2046

2047 ai_suggestion_fields = models.JSONField(

2048 _("AI suggestion fields"),

2049 null=True,

2050 blank=True,

2051 help_text=_(

2052 "Which of the AI-suggested fields to apply to the document.",

2053 ),

2054 )

2055

2056 ai_create_missing = models.BooleanField(

2057 _("create missing objects"),

2058 default=False,

2059 help_text=_(

2060 "Create suggested tags, correspondents, document types and storage "

2061 "paths that do not already exist instead of skipping them.",

2062 ),

2063 )

2064

2065 ai_overwrite_existing = models.BooleanField(

2066 _("overwrite existing values"),

2067 default=False,

2068 help_text=_(

2069 "Apply suggestions even if the document already has a value for that "

2070 "field. Tags are always added to, never replaced.",

2071 ),

2072 )

2073

2074 class Meta:

2075 verbose_name = _("workflow action")

2076 verbose_name_plural = _("workflow actions")

2077

2078 def __str__(self):

2079 return f"WorkflowAction {self.pk}"

2080

2081

2082class Workflow(models.Model):

2083 name = models.CharField(_("name"), max_length=256, unique=True)

2084

2085 order = models.IntegerField(_("order"), default=0)

2086

2087 triggers = models.ManyToManyField(

2088 WorkflowTrigger,

2089 related_name="workflows",

2090 blank=False,

2091 verbose_name=_("triggers"),

2092 )

2093

2094 actions = models.ManyToManyField(

2095 WorkflowAction,

2096 related_name="workflows",

2097 blank=False,

2098 verbose_name=_("actions"),

2099 )

2100

2101 enabled = models.BooleanField(_("enabled"), default=True)

2102

2103 def __str__(self):

2104 return f"Workflow: {self.name}"

2105

2106

2107class WorkflowRun(SoftDeleteModel):

2108 workflow = models.ForeignKey(

2109 Workflow,

2110 on_delete=models.CASCADE,

2111 related_name="runs",

2112 verbose_name=_("workflow"),

2113 )

2114

2115 type = models.PositiveSmallIntegerField(

2116 _("workflow trigger type"),

2117 choices=WorkflowTrigger.WorkflowTriggerType.choices,

2118 null=True,

2119 )

2120

2121 document = models.ForeignKey(

2122 Document,

2123 null=True,

2124 on_delete=models.CASCADE,

2125 related_name="workflow_runs",

2126 verbose_name=_("document"),

2127 )

2128

2129 run_at = models.DateTimeField(

2130 _("date run"),

2131 default=timezone.now,

2132 db_index=True,

2133 )

2134

2135 class Meta:

2136 verbose_name = _("workflow run")

2137 verbose_name_plural = _("workflow runs")

2138

2139 def __str__(self) -> str:

2140 return f"WorkflowRun of {self.workflow} at {self.run_at} on {self.document}"