Coverage for documents/views.py: 63%
2084 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-10-10 09:07 +0000
1import itertools
2import logging
3import os
4import platform
5import re
6import tempfile
7import zipfile
8from collections import defaultdict
9from collections import deque
10from collections.abc import Callable
11from datetime import datetime
12from datetime import timedelta
13from http import HTTPStatus
14from pathlib import Path
15from time import mktime
16from time import sleep
17from typing import TYPE_CHECKING
18from typing import Any
19from typing import Final
20from typing import Literal
21from typing import NamedTuple
22from unicodedata import normalize
23from urllib.parse import quote
24from urllib.parse import urlparse
26import httpx
27import magic
28import pathvalidate
29from django.conf import settings
30from django.contrib.auth.models import Group
31from django.contrib.auth.models import User
32from django.contrib.contenttypes.models import ContentType
33from django.core.cache import cache
34from django.db import connections
35from django.db.migrations.loader import MigrationLoader
36from django.db.migrations.recorder import MigrationRecorder
37from django.db.models import Avg
38from django.db.models import Case
39from django.db.models import Count
40from django.db.models import IntegerField
41from django.db.models import Max
42from django.db.models import Model
43from django.db.models import OuterRef
44from django.db.models import Prefetch
45from django.db.models import Q
46from django.db.models import QuerySet
47from django.db.models import Subquery
48from django.db.models import Sum
49from django.db.models import When
50from django.db.models.functions import Coalesce
51from django.db.models.functions import Lower
52from django.http import FileResponse
53from django.http import Http404
54from django.http import HttpRequest
55from django.http import HttpResponse
56from django.http import HttpResponseBadRequest
57from django.http import HttpResponseForbidden
58from django.http import HttpResponseRedirect
59from django.http import HttpResponseServerError
60from django.http import StreamingHttpResponse
61from django.shortcuts import get_object_or_404
62from django.utils import timezone
63from django.utils.decorators import method_decorator
64from django.utils.timezone import make_aware
65from django.utils.translation import get_language
66from django.utils.translation import gettext_lazy as _
67from django.views import View
68from django.views.decorators.cache import cache_control
69from django.views.decorators.csrf import ensure_csrf_cookie
70from django.views.decorators.http import condition
71from django.views.generic import TemplateView
72from django_filters.rest_framework import DjangoFilterBackend
73from drf_spectacular.openapi import AutoSchema
74from drf_spectacular.types import OpenApiTypes
75from drf_spectacular.utils import OpenApiParameter
76from drf_spectacular.utils import extend_schema
77from drf_spectacular.utils import extend_schema_serializer
78from drf_spectacular.utils import extend_schema_view
79from drf_spectacular.utils import inline_serializer
80from guardian.utils import get_group_obj_perms_model
81from guardian.utils import get_user_obj_perms_model
82from langdetect import detect
83from packaging import version as packaging_version
84from redis import Redis
85from rest_framework import parsers
86from rest_framework import serializers
87from rest_framework import status
88from rest_framework.decorators import action
89from rest_framework.exceptions import NotFound
90from rest_framework.exceptions import PermissionDenied
91from rest_framework.exceptions import ValidationError
92from rest_framework.filters import OrderingFilter
93from rest_framework.filters import SearchFilter
94from rest_framework.generics import GenericAPIView
95from rest_framework.mixins import CreateModelMixin
96from rest_framework.mixins import DestroyModelMixin
97from rest_framework.mixins import ListModelMixin
98from rest_framework.mixins import RetrieveModelMixin
99from rest_framework.mixins import UpdateModelMixin
100from rest_framework.permissions import IsAuthenticated
101from rest_framework.request import Request
102from rest_framework.response import Response
103from rest_framework.viewsets import GenericViewSet
104from rest_framework.viewsets import ModelViewSet
105from rest_framework.viewsets import ReadOnlyModelViewSet
106from rest_framework.viewsets import ViewSet
108from documents import bulk_edit
109from documents.bulk_download import ArchiveOnlyStrategy
110from documents.bulk_download import OriginalAndArchiveStrategy
111from documents.bulk_download import OriginalsOnlyStrategy
112from documents.caching import get_llm_suggestion_cache
113from documents.caching import get_metadata_cache
114from documents.caching import get_suggestion_cache
115from documents.caching import refresh_llm_suggestions_cache
116from documents.caching import refresh_metadata_cache
117from documents.caching import refresh_suggestions_cache
118from documents.caching import set_llm_suggestions_cache
119from documents.caching import set_metadata_cache
120from documents.caching import set_suggestions_cache
121from documents.classifier import load_classifier
122from documents.conditionals import metadata_etag
123from documents.conditionals import metadata_last_modified
124from documents.conditionals import preview_etag
125from documents.conditionals import preview_last_modified
126from documents.conditionals import suggestions_etag
127from documents.conditionals import suggestions_last_modified
128from documents.conditionals import thumbnail_etag
129from documents.conditionals import thumbnail_last_modified
130from documents.data_models import ConsumableDocument
131from documents.data_models import DocumentMetadataOverrides
132from documents.data_models import DocumentSource
133from documents.file_handling import format_filename
134from documents.filters import CorrespondentFilterSet
135from documents.filters import CustomFieldFilterSet
136from documents.filters import DocumentFilterSet
137from documents.filters import DocumentsOrderingFilter
138from documents.filters import DocumentTypeFilterSet
139from documents.filters import EffectiveContentFilter
140from documents.filters import PaperlessTaskFilterSet
141from documents.filters import PermittedObjectsFilter
142from documents.filters import ShareLinkBundleFilterSet
143from documents.filters import ShareLinkFilterSet
144from documents.filters import StoragePathFilterSet
145from documents.filters import TagFilterSet
146from documents.filters import TitleContentFilter
147from documents.mail import EmailAttachment
148from documents.mail import send_email
149from documents.matching import match_correspondents
150from documents.matching import match_document_types
151from documents.matching import match_storage_paths
152from documents.matching import match_tags
153from documents.models import Correspondent
154from documents.models import CustomField
155from documents.models import CustomFieldInstance
156from documents.models import Document
157from documents.models import DocumentType
158from documents.models import Note
159from documents.models import PaperlessTask
160from documents.models import SavedView
161from documents.models import ShareLink
162from documents.models import ShareLinkBundle
163from documents.models import StoragePath
164from documents.models import Tag
165from documents.models import UiSettings
166from documents.models import Workflow
167from documents.models import WorkflowAction
168from documents.models import WorkflowTrigger
169from documents.permissions import AcknowledgeTasksPermissions
170from documents.permissions import PaperlessAdminPermissions
171from documents.permissions import PaperlessNotePermissions
172from documents.permissions import PaperlessObjectPermissions
173from documents.permissions import TrashPermissions
174from documents.permissions import ViewDocumentsPermissions
175from documents.permissions import annotate_document_count_by_ids
176from documents.permissions import annotate_document_count_for_related_queryset
177from documents.permissions import get_document_count_filter_for_user
178from documents.permissions import get_objects_for_user_owner_aware
179from documents.permissions import has_global_statistics_permission
180from documents.permissions import has_perms_owner_aware
181from documents.permissions import has_system_status_permission
182from documents.permissions import permitted_document_ids
183from documents.permissions import permitted_object_ids
184from documents.permissions import set_permissions_for_objects
185from documents.permissions import user_is_unrestricted
186from documents.plugins.date_parsing import get_date_parser
187from documents.schema import generate_object_with_permissions_schema
188from documents.search import SearchHit
189from documents.serialisers import AcknowledgeTasksViewSerializer
190from documents.serialisers import BulkDownloadSerializer
191from documents.serialisers import BulkEditObjectsSerializer
192from documents.serialisers import BulkEditSerializer
193from documents.serialisers import CorrespondentSerializer
194from documents.serialisers import CustomFieldSerializer
195from documents.serialisers import DeleteDocumentsSerializer
196from documents.serialisers import DocumentBarcodeSerializer
197from documents.serialisers import DocumentSelectionSerializer
198from documents.serialisers import DocumentSerializer
199from documents.serialisers import DocumentTypeSerializer
200from documents.serialisers import DocumentVersionLabelSerializer
201from documents.serialisers import DocumentVersionSerializer
202from documents.serialisers import EditPdfDocumentsSerializer
203from documents.serialisers import EmailSerializer
204from documents.serialisers import MergeDocumentsAsVersionsSerializer
205from documents.serialisers import MergeDocumentsSerializer
206from documents.serialisers import NotesSerializer
207from documents.serialisers import PostDocumentSerializer
208from documents.serialisers import RemovePasswordDocumentsSerializer
209from documents.serialisers import ReprocessDocumentsSerializer
210from documents.serialisers import RotateDocumentsSerializer
211from documents.serialisers import RunTaskSerializer
212from documents.serialisers import SavedViewSerializer
213from documents.serialisers import SearchResultSerializer
214from documents.serialisers import SerializerWithPerms
215from documents.serialisers import ShareLinkBundleSerializer
216from documents.serialisers import ShareLinkSerializer
217from documents.serialisers import StoragePathSerializer
218from documents.serialisers import StoragePathTestSerializer
219from documents.serialisers import TagSerializer
220from documents.serialisers import TaskSerializerV9
221from documents.serialisers import TaskSerializerV10
222from documents.serialisers import TaskSummarySerializer
223from documents.serialisers import TrashSerializer
224from documents.serialisers import UiSettingsViewSerializer
225from documents.serialisers import WorkflowActionSerializer
226from documents.serialisers import WorkflowSerializer
227from documents.serialisers import WorkflowTriggerSerializer
228from documents.signals import document_updated
229from documents.tasks import build_share_link_bundle
230from documents.tasks import consume_file
231from documents.tasks import empty_trash
232from documents.tasks import llmindex_index
233from documents.tasks import sanity_check
234from documents.tasks import train_classifier
235from documents.tasks import update_document_parent_tags
236from documents.utils import get_boolean
237from documents.versioning import VersionResolutionError
238from documents.versioning import annotate_effective_content
239from documents.versioning import get_latest_version_for_root
240from documents.versioning import get_request_version_param
241from documents.versioning import get_root_document
242from documents.versioning import latest_version_content_prefetch
243from documents.versioning import resolve_requested_version_for_root
244from documents.versioning import versions_newest_first
245from paperless import version
246from paperless.celery import app as celery_app
247from paperless.config import AIConfig
248from paperless.config import GeneralConfig
249from paperless.config import RemoteOCRConfig
250from paperless.models import ApplicationConfiguration
251from paperless.parsers.registry import get_parser_registry
252from paperless.parsers.remote import RemoteEngineConfig
253from paperless.serialisers import GroupSerializer
254from paperless.serialisers import UserSerializer
255from paperless.views import StandardPagination
256from paperless_ai.ai_classifier import get_ai_document_classification
257from paperless_ai.ai_classifier import get_llm_output_language
258from paperless_ai.chat import stream_chat_with_documents
259from paperless_ai.exceptions import LLMProviderError
260from paperless_ai.exceptions import LLMTimeoutError
261from paperless_ai.matching import extract_unmatched_names
262from paperless_ai.matching import match_correspondents_by_name
263from paperless_ai.matching import match_document_types_by_name
264from paperless_ai.matching import match_storage_paths_by_name
265from paperless_ai.matching import match_tags_by_name
266from paperless_ai.matching import resolve_correspondent_ids
267from paperless_ai.matching import resolve_document_type_ids
268from paperless_ai.matching import resolve_storage_path_ids
269from paperless_ai.matching import resolve_tag_ids
270from paperless_mail.models import MailAccount
271from paperless_mail.models import MailRule
272from paperless_mail.oauth import PaperlessMailOAuth2Manager
273from paperless_mail.serialisers import MailAccountSerializer
274from paperless_mail.serialisers import MailRuleSerializer
276if settings.AUDIT_LOG_ENABLED: 276 ↛ 279line 276 didn't jump to line 279 because the condition on line 276 was always true
277 from auditlog.models import LogEntry
279if TYPE_CHECKING: 279 ↛ 280line 279 didn't jump to line 280 because the condition on line 279 was never true
280 from paperless_ai.base_model import TaxonomyChoiceDict
283logger = logging.getLogger("paperless.api")
285# Crossover point for intersect_and_order: below this count use a targeted
286# IN-clause query; at or above this count fall back to a full-table scan +
287# Python set intersection. The IN-clause is faster for small result sets but
288# degrades on SQLite with thousands of parameters. PostgreSQL handles large IN
289# clauses efficiently, so this threshold mainly protects SQLite users.
290_TANTIVY_INTERSECT_THRESHOLD = 5_000
291_TANTIVY_SEARCH_PARAM_NAMES = ("text", "title_search", "query", "more_like_id")
293# whoosh-compat's fieldname tagger (used only for SearchMode.QUERY, via the
294# whoosh grammar in parse_user_query) is O(n^2) in plain word characters:
295# measured at ~0.96s/10k chars, ~3.67s/20k, ~14.4s/40k against the real field
296# registry. Django's DATA_UPLOAD_MAX_MEMORY_SIZE default (2.5 MB) does not
297# bound this on the POST-body selection-filter path, so an unbounded query
298# is a single-request CPU exhaustion vector. 4096 chars caps the worst case
299# at roughly 0.16s (quadratic extrapolation from the measurements above),
300# far beyond any plausible hand-typed advanced query, while still being fast
301# enough to absorb inside a request handler. Applied to all three modes at
302# this shared choke point: TEXT and TITLE route through simple_search_tokens
303# instead and measure linear even at 20k chars, so the cap is hygiene for
304# them, not a fix, but a single limit here is simpler than one exemption.
305# Not exposed as a PAPERLESS_* setting: this is a hard security boundary,
306# not a tunable, and a raisable ceiling would let a misconfiguration
307# reintroduce the exact hazard this exists to close.
308_MAX_QUERY_LENGTH: Final[int] = 4096
311def _get_tantivy_query_and_mode(params):
312 from documents.search import QueryTooLongError
313 from documents.search import SearchMode
315 if "text" in params:
316 raw, mode = str(params["text"]), SearchMode.TEXT
317 elif "title_search" in params:
318 raw, mode = str(params["title_search"]), SearchMode.TITLE
319 elif "query" in params: 319 ↛ 322line 319 didn't jump to line 322 because the condition on line 319 was always true
320 raw, mode = str(params["query"]), SearchMode.QUERY
321 else:
322 return None # pragma: no cover
324 if len(raw) > _MAX_QUERY_LENGTH: 324 ↛ 325line 324 didn't jump to line 325 because the condition on line 324 was never true
325 raise QueryTooLongError(len(raw), _MAX_QUERY_LENGTH)
326 return raw, mode
329def _get_more_like_id(query_params: dict[str, Any], user: User | None) -> int:
330 try:
331 more_like_doc_id = int(query_params["more_like_id"])
332 more_like_doc = Document.objects.select_related("owner").get(
333 pk=more_like_doc_id,
334 )
335 except (TypeError, ValueError, Document.DoesNotExist):
336 raise PermissionDenied(_("Invalid more_like_id"))
338 if user and not has_perms_owner_aware(
339 user,
340 "view_document",
341 more_like_doc,
342 ):
343 raise PermissionDenied(_("Insufficient permissions."))
345 return more_like_doc_id
348class SearchParams(NamedTuple):
349 sort_field_name: str | None
350 sort_reverse: bool
351 use_tantivy_sort: bool
352 page_num: int
353 page_size: int
356class SearchResultPage(NamedTuple):
357 ordered_ids: list[int]
358 hits: list[SearchHit]
359 page_offset: int
362class ResolvedRequestDocs(NamedTuple):
363 request_doc: Document
364 root_doc: Document
367class IndexView(TemplateView):
368 template_name = "index.html"
370 def get_frontend_language(self):
371 if hasattr(
372 self.request.user,
373 "ui_settings",
374 ) and self.request.user.ui_settings.settings.get("language"):
375 lang = self.request.user.ui_settings.settings.get("language")
376 else:
377 lang = get_language()
378 # This is here for the following reason:
379 # Django identifies languages in the form "en-us"
380 # However, angular generates locales as "en-US".
381 # this translates between these two forms.
382 if "-" in lang:
383 first = lang[: lang.index("-")]
384 second = lang[lang.index("-") + 1 :]
385 return f"{first}-{second.upper()}"
386 return lang
388 def get_context_data(self, **kwargs):
389 context = super().get_context_data(**kwargs)
390 context["cookie_prefix"] = settings.COOKIE_PREFIX
391 context["username"] = self.request.user.username
392 context["full_name"] = self.request.user.get_full_name()
393 context["styles_css"] = f"frontend/{self.get_frontend_language()}/styles.css"
394 context["polyfills_js"] = (
395 f"frontend/{self.get_frontend_language()}/polyfills.js"
396 )
397 context["main_js"] = f"frontend/{self.get_frontend_language()}/main.js"
398 context["webmanifest"] = (
399 f"frontend/{self.get_frontend_language()}/manifest.webmanifest"
400 )
401 context["apple_touch_icon"] = (
402 f"frontend/{self.get_frontend_language()}/apple-touch-icon.png"
403 )
404 return context
407class PassUserMixin(GenericAPIView[Any]):
408 """
409 Pass a user object to serializer
410 """
412 def get_serializer(self, *args, **kwargs):
413 serializer_class = self.get_serializer_class()
414 if isinstance(serializer_class, type) and issubclass(
415 serializer_class,
416 SerializerWithPerms,
417 ):
418 kwargs.setdefault("user", self.request.user)
419 try:
420 full_perms = get_boolean(
421 str(self.request.query_params.get("full_perms", "false")),
422 )
423 except ValueError:
424 full_perms = False
425 kwargs.setdefault(
426 "full_perms",
427 full_perms,
428 )
429 return super().get_serializer(*args, **kwargs)
432class BulkPermissionMixin:
433 """
434 Prefetch Django-Guardian permissions for a list before serialization, to avoid N+1 queries.
435 """
437 def _get_object_perms(
438 self,
439 objects: list,
440 perm_codenames: list[str],
441 actor: Literal["users", "groups"],
442 ) -> dict[int, dict[str, list[int]]]:
443 """
444 Collect object-level permissions for either users or groups.
445 """
446 model = self.queryset.model
447 obj_perm_model = (
448 get_user_obj_perms_model(model)
449 if actor == "users"
450 else get_group_obj_perms_model(model)
451 )
452 id_field = "user_id" if actor == "users" else "group_id"
453 ctype = ContentType.objects.get_for_model(model)
454 object_pks = [obj.pk for obj in objects]
456 perms_qs = obj_perm_model.objects.filter(
457 content_type=ctype,
458 object_pk__in=object_pks,
459 permission__codename__in=perm_codenames,
460 ).values_list("object_pk", id_field, "permission__codename")
462 perms: dict[int, dict[str, list[int]]] = defaultdict(lambda: defaultdict(list))
463 for object_pk, actor_id, codename in perms_qs: 463 ↛ 464line 463 didn't jump to line 464 because the loop on line 463 never started
464 perms[int(object_pk)][codename].append(actor_id)
466 # Ensure that all objects have all codenames, even if empty
467 for pk in object_pks:
468 for codename in perm_codenames:
469 perms[pk][codename]
471 return perms
473 def get_serializer_context(self):
474 """
475 Get all permissions of the current list of objects at once and pass them to the serializer.
476 This avoid fetching permissions object by object in database.
477 """
478 context = super().get_serializer_context()
480 if getattr(self, "action", None) != "list":
481 # Batching only pays off across a page of objects; for single-object
482 # actions (retrieve, update, ...) the per-object fallback in
483 # get_user_can_change()/_get_perms() is cheap and avoids scanning
484 # the whole queryset here.
485 return context
487 # Check which objects are being paginated
488 page = getattr(self, "paginator", None)
489 if page and hasattr(page, "page"):
490 queryset = page.page.object_list
491 elif hasattr(self, "page"): 491 ↛ 492line 491 didn't jump to line 492 because the condition on line 491 was never true
492 queryset = self.page
493 else:
494 queryset = self.filter_queryset(self.get_queryset())
496 model_name = self.queryset.model.__name__.lower()
497 permission_name_view = f"view_{model_name}"
498 permission_name_change = f"change_{model_name}"
500 user_perms = self._get_object_perms(
501 objects=queryset,
502 perm_codenames=[permission_name_view, permission_name_change],
503 actor="users",
504 )
505 group_perms = self._get_object_perms(
506 objects=queryset,
507 perm_codenames=[permission_name_view, permission_name_change],
508 actor="groups",
509 )
511 context["users_view_perms"] = {
512 pk: user_perms[pk][permission_name_view] for pk in user_perms
513 }
514 context["users_change_perms"] = {
515 pk: user_perms[pk][permission_name_change] for pk in user_perms
516 }
517 context["groups_view_perms"] = {
518 pk: group_perms[pk][permission_name_view] for pk in group_perms
519 }
520 context["groups_change_perms"] = {
521 pk: group_perms[pk][permission_name_change] for pk in group_perms
522 }
524 return context
527class PermissionsAwareDocumentCountMixin(BulkPermissionMixin, PassUserMixin):
528 """Mixin to add document count to queryset, permissions-aware if needed"""
530 # Direct FK/M2M relation name from this model to Document, used for the
531 # cheap Count(filter=...) path (Correspondent, DocumentType, StoragePath).
532 document_count_related_name: str = "documents"
534 # Set both of these instead, for models that only reach Document through
535 # an M2M/through-model table (Tag, CustomField). A plain Count(filter=...)
536 # over such a relation is fine for a direct FK, but forces a much more
537 # expensive plan once an M2M bridge table is involved -- see
538 # annotate_document_count_for_related_queryset() for why.
539 document_count_through: type[Model] | None = None
540 document_count_source_field: str | None = None
542 def _get_document_count_source_field(self) -> str:
543 if self.document_count_source_field is None: 543 ↛ 544line 543 didn't jump to line 544 because the condition on line 543 was never true
544 msg = (
545 "document_count_source_field must be set when "
546 "document_count_through is configured"
547 )
548 raise ValueError(msg)
549 return self.document_count_source_field
551 def get_document_count_filter(self):
552 request = getattr(self, "request", None)
553 user = getattr(request, "user", None) if request else None
554 return get_document_count_filter_for_user(
555 user,
556 related_name=self.document_count_related_name,
557 )
559 def get_queryset(self):
560 base_qs = super().get_queryset()
562 if self.document_count_through:
563 user = getattr(getattr(self, "request", None), "user", None)
564 return annotate_document_count_for_related_queryset(
565 base_qs,
566 through_model=self.document_count_through,
567 related_object_field=self._get_document_count_source_field(),
568 user=user,
569 )
571 filter = self.get_document_count_filter()
572 return base_qs.annotate(
573 document_count=Count(
574 self.document_count_related_name,
575 filter=filter,
576 distinct=True,
577 ),
578 )
581@extend_schema_view(**generate_object_with_permissions_schema(CorrespondentSerializer))
582class CorrespondentViewSet(
583 PermissionsAwareDocumentCountMixin,
584 ModelViewSet[Correspondent],
585):
586 model = Correspondent
588 queryset = Correspondent.objects.select_related("owner").order_by(Lower("name"))
590 serializer_class = CorrespondentSerializer
591 pagination_class = StandardPagination
592 permission_classes = (IsAuthenticated, PaperlessObjectPermissions)
593 filter_backends = (
594 DjangoFilterBackend,
595 OrderingFilter,
596 PermittedObjectsFilter,
597 )
598 filterset_class = CorrespondentFilterSet
599 ordering_fields = (
600 "name",
601 "matching_algorithm",
602 "match",
603 "document_count",
604 "last_correspondence",
605 )
607 def list(self, request, *args, **kwargs):
608 if request.query_params.get("last_correspondence", None): 608 ↛ 609line 608 didn't jump to line 609 because the condition on line 608 was never true
609 self.queryset = self.queryset.annotate(
610 last_correspondence=Max(
611 "documents__created",
612 filter=self.get_document_count_filter(),
613 ),
614 )
615 return super().list(request, *args, **kwargs)
617 def retrieve(self, request, *args, **kwargs):
618 self.queryset = self.queryset.annotate(
619 last_correspondence=Max(
620 "documents__created",
621 filter=self.get_document_count_filter(),
622 ),
623 )
624 return super().retrieve(request, *args, **kwargs)
627@extend_schema_view(**generate_object_with_permissions_schema(TagSerializer))
628class TagViewSet(PermissionsAwareDocumentCountMixin, ModelViewSet[Tag]):
629 model = Tag
630 serializer_class = TagSerializer
631 document_count_through = Document.tags.through
632 document_count_source_field = "tag_id"
634 queryset = Tag.objects.select_related("owner").order_by(
635 Lower("name"),
636 )
638 pagination_class = StandardPagination
639 permission_classes = (IsAuthenticated, PaperlessObjectPermissions)
640 filter_backends = (
641 DjangoFilterBackend,
642 OrderingFilter,
643 PermittedObjectsFilter,
644 )
645 filterset_class = TagFilterSet
646 ordering_fields = ("color", "name", "matching_algorithm", "match", "document_count")
648 def get_serializer_context(self):
649 context = super().get_serializer_context()
650 context["document_count_filter"] = self.get_document_count_filter()
651 if hasattr(self, "_children_map"):
652 context["children_map"] = self._children_map
653 return context
655 def list(self, request, *args, **kwargs):
656 """
657 Build a children map once to avoid per-parent queries in the serializer.
658 """
659 queryset = self.filter_queryset(self.get_queryset())
660 ordering = OrderingFilter().get_ordering(request, queryset, self) or (
661 Lower("name"),
662 )
663 queryset = queryset.order_by(*ordering)
665 all_tags = list(queryset)
666 descendant_pks = {pk for tag in all_tags for pk in tag.get_descendants_pks()}
668 if descendant_pks:
669 user = getattr(getattr(self, "request", None), "user", None)
670 children_source = list(
671 annotate_document_count_for_related_queryset(
672 Tag.objects.filter(
673 pk__in=descendant_pks | {t.pk for t in all_tags},
674 )
675 .filter(pk__in=permitted_object_ids(user, Tag, "view_tag"))
676 .select_related("owner"),
677 through_model=self.document_count_through,
678 related_object_field=self._get_document_count_source_field(),
679 user=user,
680 ).order_by(*ordering),
681 )
682 else:
683 children_source = all_tags
685 children_map = {}
686 for tag in children_source:
687 children_map.setdefault(tag.tn_parent_id, []).append(tag)
688 self._children_map = children_map
690 page = self.paginate_queryset(queryset)
691 serializer = self.get_serializer(page, many=True)
692 response = self.get_paginated_response(serializer.data)
693 response.data["display_count"] = len(children_source)
694 api_version = int(request.version or settings.REST_FRAMEWORK["DEFAULT_VERSION"])
695 if descendant_pks and api_version < 10: 695 ↛ 697line 695 didn't jump to line 697 because the condition on line 695 was never true
696 # Include children in the "all" field, if needed
697 response.data["all"] = [tag.pk for tag in children_source]
698 return response
700 def perform_update(self, serializer):
701 old_parent = self.get_object().get_parent()
702 tag = serializer.save()
703 new_parent = tag.get_parent()
704 if new_parent and old_parent != new_parent: 704 ↛ 705line 704 didn't jump to line 705 because the condition on line 704 was never true
705 update_document_parent_tags(tag, new_parent)
708@extend_schema_view(**generate_object_with_permissions_schema(DocumentTypeSerializer))
709class DocumentTypeViewSet(
710 PermissionsAwareDocumentCountMixin,
711 ModelViewSet[DocumentType],
712):
713 model = DocumentType
715 queryset = DocumentType.objects.select_related("owner").order_by(Lower("name"))
717 serializer_class = DocumentTypeSerializer
718 pagination_class = StandardPagination
719 permission_classes = (IsAuthenticated, PaperlessObjectPermissions)
720 filter_backends = (
721 DjangoFilterBackend,
722 OrderingFilter,
723 PermittedObjectsFilter,
724 )
725 filterset_class = DocumentTypeFilterSet
726 ordering_fields = ("name", "matching_algorithm", "match", "document_count")
729@extend_schema_serializer(
730 component_name="EmailDocumentRequest",
731 exclude_fields=("documents",),
732)
733class EmailDocumentDetailSchema(EmailSerializer):
734 pass
737@extend_schema_view(
738 retrieve=extend_schema(
739 description="Retrieve a single document",
740 responses={
741 200: DocumentSerializer(all_fields=True),
742 400: None,
743 },
744 parameters=[
745 OpenApiParameter(
746 name="full_perms",
747 type=OpenApiTypes.BOOL,
748 location=OpenApiParameter.QUERY,
749 ),
750 OpenApiParameter(
751 name="fields",
752 type=OpenApiTypes.STR,
753 many=True,
754 location=OpenApiParameter.QUERY,
755 ),
756 ],
757 ),
758 download=extend_schema(
759 description="Download the document",
760 parameters=[
761 OpenApiParameter(
762 name="original",
763 type=OpenApiTypes.BOOL,
764 location=OpenApiParameter.QUERY,
765 ),
766 OpenApiParameter(
767 name="follow_formatting",
768 description="Whether or not to use the filename on disk",
769 type=OpenApiTypes.BOOL,
770 location=OpenApiParameter.QUERY,
771 ),
772 ],
773 responses={200: OpenApiTypes.BINARY},
774 ),
775 history=extend_schema(
776 description="View the document history",
777 responses={
778 200: inline_serializer(
779 name="LogEntry",
780 many=True,
781 fields={
782 "id": serializers.IntegerField(),
783 "timestamp": serializers.DateTimeField(),
784 "action": serializers.CharField(),
785 "changes": serializers.DictField(),
786 "actor": inline_serializer(
787 name="Actor",
788 fields={
789 "id": serializers.IntegerField(),
790 "username": serializers.CharField(),
791 },
792 ),
793 },
794 ),
795 400: None,
796 403: None,
797 404: None,
798 },
799 ),
800 metadata=extend_schema(
801 description="View the document metadata",
802 responses={
803 200: inline_serializer(
804 name="Metadata",
805 fields={
806 "original_checksum": serializers.CharField(),
807 "original_size": serializers.IntegerField(),
808 "original_mime_type": serializers.CharField(),
809 "media_filename": serializers.CharField(),
810 "has_archive_version": serializers.BooleanField(),
811 "original_metadata": serializers.ListField(
812 child=inline_serializer(
813 name="OriginalMetadataEntry",
814 fields={
815 "namespace": serializers.CharField(),
816 "prefix": serializers.CharField(),
817 "key": serializers.CharField(),
818 "value": serializers.CharField(),
819 },
820 ),
821 ),
822 "archive_checksum": serializers.CharField(
823 allow_null=True,
824 required=False,
825 ),
826 "archive_media_filename": serializers.CharField(
827 allow_null=True,
828 required=False,
829 ),
830 "original_filename": serializers.CharField(),
831 "archive_size": serializers.IntegerField(
832 allow_null=True,
833 required=False,
834 ),
835 "archive_metadata": serializers.ListField(
836 child=inline_serializer(
837 name="ArchiveMetadataEntry",
838 fields={
839 "namespace": serializers.CharField(),
840 "prefix": serializers.CharField(),
841 "key": serializers.CharField(),
842 "value": serializers.CharField(),
843 },
844 ),
845 allow_null=True,
846 required=False,
847 ),
848 "lang": serializers.CharField(),
849 "barcodes": DocumentBarcodeSerializer(many=True),
850 },
851 ),
852 HTTPStatus.BAD_REQUEST: None,
853 HTTPStatus.FORBIDDEN: None,
854 HTTPStatus.NOT_FOUND: None,
855 },
856 ),
857 notes=extend_schema(
858 description="View, add, or delete notes for the document",
859 methods=["GET", "POST", "DELETE"],
860 request=inline_serializer(
861 name="NoteCreateRequest",
862 fields={
863 "note": serializers.CharField(),
864 },
865 ),
866 parameters=[
867 OpenApiParameter(
868 name="id",
869 type=OpenApiTypes.INT,
870 location=OpenApiParameter.QUERY,
871 required=False,
872 description="Note ID to delete (used only for DELETE requests)",
873 ),
874 ],
875 responses={
876 200: NotesSerializer(many=True),
877 400: None,
878 403: None,
879 404: None,
880 },
881 ),
882 suggestions=extend_schema(
883 description="View suggestions for the document",
884 responses={
885 200: inline_serializer(
886 name="Suggestions",
887 fields={
888 "correspondents": serializers.ListField(
889 child=serializers.IntegerField(),
890 ),
891 "tags": serializers.ListField(child=serializers.IntegerField()),
892 "document_types": serializers.ListField(
893 child=serializers.IntegerField(),
894 ),
895 "storage_paths": serializers.ListField(
896 child=serializers.IntegerField(),
897 ),
898 "dates": serializers.ListField(child=serializers.CharField()),
899 },
900 ),
901 400: None,
902 403: None,
903 404: None,
904 },
905 ),
906 ai_suggestions=extend_schema(
907 description="View AI suggestions for the document",
908 responses={
909 200: inline_serializer(
910 name="AISuggestions",
911 fields={
912 "title": serializers.CharField(allow_null=True),
913 "correspondents": serializers.ListField(
914 child=serializers.IntegerField(),
915 ),
916 "suggested_correspondents": serializers.ListField(
917 child=serializers.CharField(),
918 ),
919 "tags": serializers.ListField(child=serializers.IntegerField()),
920 "suggested_tags": serializers.ListField(
921 child=serializers.CharField(),
922 ),
923 "document_types": serializers.ListField(
924 child=serializers.IntegerField(),
925 ),
926 "suggested_document_types": serializers.ListField(
927 child=serializers.CharField(),
928 ),
929 "storage_paths": serializers.ListField(
930 child=serializers.IntegerField(),
931 ),
932 "suggested_storage_paths": serializers.ListField(
933 child=serializers.CharField(),
934 ),
935 "dates": serializers.ListField(child=serializers.CharField()),
936 },
937 ),
938 400: None,
939 403: None,
940 404: None,
941 },
942 ),
943 thumb=extend_schema(
944 description="View the document thumbnail",
945 responses={200: OpenApiTypes.BINARY},
946 ),
947 preview=extend_schema(
948 description="View the document preview",
949 responses={200: OpenApiTypes.BINARY},
950 ),
951 share_links=extend_schema(
952 operation_id="document_share_links",
953 description="View share links for the document",
954 parameters=[
955 OpenApiParameter(
956 name="id",
957 type=OpenApiTypes.STR,
958 location=OpenApiParameter.PATH,
959 ),
960 ],
961 responses={
962 200: {
963 "type": "array",
964 "items": {
965 "type": "object",
966 "properties": {
967 "id": {"type": "integer"},
968 "created": {"type": "string", "format": "date-time"},
969 "expiration": {"type": "string", "format": "date-time"},
970 "slug": {"type": "string"},
971 },
972 },
973 },
974 400: None,
975 403: None,
976 404: None,
977 },
978 ),
979 email_document=extend_schema(
980 description="Email the document to one or more recipients as an attachment.",
981 request=EmailDocumentDetailSchema,
982 responses={
983 200: inline_serializer(
984 name="EmailDocumentResponse",
985 fields={"message": serializers.CharField()},
986 ),
987 400: None,
988 403: None,
989 404: None,
990 500: None,
991 },
992 deprecated=True,
993 ),
994 email_documents=extend_schema(
995 operation_id="email_documents",
996 description="Email one or more documents as attachments to one or more recipients.",
997 request=EmailSerializer,
998 responses={
999 200: inline_serializer(
1000 name="EmailDocumentsResponse",
1001 fields={"message": serializers.CharField()},
1002 ),
1003 400: None,
1004 403: None,
1005 404: None,
1006 500: None,
1007 },
1008 ),
1009)
1010class DocumentViewSet(
1011 BulkPermissionMixin,
1012 PassUserMixin,
1013 RetrieveModelMixin,
1014 UpdateModelMixin,
1015 DestroyModelMixin,
1016 ListModelMixin,
1017 GenericViewSet[Document],
1018):
1019 model = Document
1020 queryset = Document.objects.all()
1021 serializer_class = DocumentSerializer
1022 pagination_class = StandardPagination
1023 permission_classes = (IsAuthenticated, PaperlessObjectPermissions)
1024 filter_backends = (
1025 DjangoFilterBackend,
1026 SearchFilter,
1027 DocumentsOrderingFilter,
1028 PermittedObjectsFilter,
1029 )
1030 filterset_class = DocumentFilterSet
1031 search_fields = ("title", "correspondent__name", "effective_content")
1032 ordering_fields = (
1033 "id",
1034 "title",
1035 "correspondent__name",
1036 "document_type__name",
1037 "storage_path__name",
1038 "created",
1039 "modified",
1040 "added",
1041 "archive_serial_number",
1042 "num_notes",
1043 "owner",
1044 "page_count",
1045 "custom_field_",
1046 )
1048 def _get_selection_data_for_queryset(self, queryset):
1049 # Resolve once instead of once per model below. `queryset` can carry an
1050 # arbitrarily expensive WHERE clause (user filters plus the permission
1051 # filter); re-embedding it as a subquery inside 5 separate Count(...)
1052 # calls forces the database to re-evaluate that whole thing 5 times, and
1053 # -- for FK relations especially -- can defeat semi-join planning
1054 # entirely at scale. A concrete id list is cheap to reuse.
1055 # order_by() drops the default/user ordering -- irrelevant for a plain
1056 # id list, but left in place it forces a sort over the full filtered
1057 # set before the ids can even be collected.
1058 document_ids = list(queryset.order_by().values_list("pk", flat=True))
1060 correspondents = Correspondent.objects.annotate(
1061 document_count=Count(
1062 "documents",
1063 filter=Q(documents__id__in=document_ids),
1064 distinct=True,
1065 ),
1066 )
1067 document_types = DocumentType.objects.annotate(
1068 document_count=Count(
1069 "documents",
1070 filter=Q(documents__id__in=document_ids),
1071 distinct=True,
1072 ),
1073 )
1074 storage_paths = StoragePath.objects.annotate(
1075 document_count=Count(
1076 "documents",
1077 filter=Q(documents__id__in=document_ids),
1078 distinct=True,
1079 ),
1080 )
1081 # Tag and CustomField reach Document through an M2M/through-model table;
1082 # a plain Count(filter=...) there is a much more expensive plan than the
1083 # FK relations above once the bridge table is large -- see
1084 # annotate_document_count_by_ids() for why.
1085 tags = annotate_document_count_by_ids(
1086 Tag.objects.all(),
1087 through_model=Document.tags.through,
1088 related_object_field="tag_id",
1089 document_ids=document_ids,
1090 )
1091 custom_fields = annotate_document_count_by_ids(
1092 CustomField.objects.all(),
1093 through_model=CustomFieldInstance,
1094 related_object_field="field_id",
1095 document_ids=document_ids,
1096 )
1097 return {
1098 "selected_correspondents": [
1099 {"id": t.id, "document_count": t.document_count} for t in correspondents
1100 ],
1101 "selected_tags": [
1102 {"id": t.id, "document_count": t.document_count} for t in tags
1103 ],
1104 "selected_document_types": [
1105 {"id": t.id, "document_count": t.document_count} for t in document_types
1106 ],
1107 "selected_storage_paths": [
1108 {"id": t.id, "document_count": t.document_count} for t in storage_paths
1109 ],
1110 "selected_custom_fields": [
1111 {"id": t.id, "document_count": t.document_count} for t in custom_fields
1112 ],
1113 }
1115 @classmethod
1116 def _content_filter_params(cls) -> tuple[str, ...]:
1117 """
1118 Query params whose filtering needs effective_content evaluated in SQL
1119 against every candidate row -- see
1120 _needs_effective_content_annotation(). Derived rather than
1121 hand-maintained so a new content-filtering param counts automatically.
1122 """
1123 params = [
1124 name
1125 for name, f in DocumentFilterSet.declared_filters.items()
1126 if isinstance(f, (TitleContentFilter, EffectiveContentFilter))
1127 ]
1128 if "effective_content" in cls.search_fields: 1128 ↛ 1130line 1128 didn't jump to line 1130 because the condition on line 1128 was always true
1129 params.append(SearchFilter().search_param)
1130 return tuple(params)
1132 def _needs_effective_content_annotation(self) -> bool:
1133 # effective_content is a per-row correlated subquery resolving each
1134 # document's latest version. Filtering *on* it forces the database to
1135 # evaluate it for every candidate row before reaching the LIMIT, which
1136 # the root_document_id self-join makes pathological on MariaDB
1137 # specifically once real candidate counts get large; otherwise the
1138 # "versions" prefetch + Document.get_effective_content() resolves only
1139 # the page that survives pagination. Every param here is deprecated in
1140 # favor of the Tantivy-backed search endpoint (see filters.py's
1141 # TitleContentFilter/EffectiveContentFilter docs), so pay that cost
1142 # only when one is actually used. Blank values don't count, matching
1143 # how those filters themselves no-op on them -- an empty `?search=`
1144 # applies no predicate.
1145 params = self.request.query_params
1146 return any(
1147 params.get(param, "").strip() for param in self._content_filter_params()
1148 )
1150 def _requested_fields(self) -> list[str] | None:
1151 # The sparse-fieldset `fields` param, as DynamicFieldsModelSerializer
1152 # wants it: None means "no restriction, serialize everything", which
1153 # a blank value means too. get_queryset() and get_serializer() both
1154 # branch on this, and they have to read it identically -- a queryset
1155 # that skips the content prefetch for a response that still
1156 # serializes content reintroduces get_effective_content()'s
1157 # per-instance fallback.
1158 fields_param = self.request.query_params.get("fields")
1159 return fields_param.split(",") if fields_param else None
1161 def _needs_effective_content_prefetch(self) -> bool:
1162 # The prefetch spares get_effective_content() a per-instance fallback
1163 # query, but only earns itself when content can reach the response.
1164 fields = self._requested_fields()
1165 return fields is None or "content" in fields
1167 def get_queryset(self):
1168 # A correlated subquery avoids the LEFT JOIN + Count() this used to
1169 # be, which forced a GROUP BY aggregate over every matching document
1170 # before the query could even be sorted or limited.
1171 note_count = Subquery(
1172 Note.objects.filter(document=OuterRef("pk"))
1173 .order_by()
1174 .values("document")
1175 .annotate(count=Count("pk"))
1176 .values("count"),
1177 output_field=IntegerField(),
1178 )
1179 # No .distinct() here: nothing in this base queryset can produce
1180 # duplicate document rows (select_related below is all FK-to-PK;
1181 # permission filtering is a boolean id__in predicate, not a join).
1182 # M2M-based filters that *do* introduce a join (e.g. tags__id__in)
1183 # already call .distinct() themselves where they need it -- see
1184 # ObjectFilter.filter(). A blanket .distinct() here forces the
1185 # database to fully sort and dedupe every visible document before
1186 # it can apply LIMIT, which is disastrous at scale.
1187 prefetches = [
1188 Prefetch(
1189 "versions",
1190 queryset=Document.objects.only(
1191 "id",
1192 "added",
1193 "checksum",
1194 "version_label",
1195 "root_document_id",
1196 "version_index",
1197 "page_count",
1198 ),
1199 ),
1200 "tags",
1201 Prefetch(
1202 "custom_fields",
1203 queryset=CustomFieldInstance.objects.select_related("field"),
1204 ),
1205 # NotesSerializer nests the author, this avoids query per note
1206 Prefetch("notes", queryset=Note.objects.select_related("user")),
1207 ]
1208 if self._needs_effective_content_prefetch():
1209 prefetches.append(latest_version_content_prefetch())
1210 queryset = (
1211 Document.objects.filter(root_document__isnull=True)
1212 .order_by("-created", "-id")
1213 .annotate(num_notes=Coalesce(note_count, 0))
1214 .select_related("correspondent", "storage_path", "document_type", "owner")
1215 .prefetch_related(*prefetches)
1216 )
1217 if self._needs_effective_content_annotation():
1218 queryset = annotate_effective_content(queryset)
1219 return queryset
1221 def get_serializer(self, *args, **kwargs):
1222 truncate_content = self.request.query_params.get("truncate_content", "False")
1223 kwargs.setdefault("context", self.get_serializer_context())
1224 kwargs.setdefault("fields", self._requested_fields())
1225 kwargs.setdefault("truncate_content", truncate_content.lower() in ["true", "1"])
1226 try:
1227 full_perms = get_boolean(
1228 str(self.request.query_params.get("full_perms", "false")),
1229 )
1230 except ValueError:
1231 full_perms = False
1232 kwargs.setdefault(
1233 "full_perms",
1234 full_perms,
1235 )
1236 return super().get_serializer(*args, **kwargs)
1238 @extend_schema(
1239 operation_id="documents_root",
1240 responses=inline_serializer(
1241 name="DocumentRootResponse",
1242 fields={
1243 "root_id": serializers.IntegerField(),
1244 },
1245 ),
1246 )
1247 @action(methods=["get"], detail=True, url_path="root")
1248 def root(self, request, pk=None):
1249 try:
1250 doc = Document.global_objects.select_related(
1251 "owner",
1252 "root_document",
1253 ).get(pk=pk)
1254 except Document.DoesNotExist:
1255 raise Http404
1257 root_doc = get_root_document(doc)
1258 if request.user is not None and not has_perms_owner_aware(
1259 request.user,
1260 "view_document",
1261 root_doc,
1262 ):
1263 return HttpResponseForbidden("Insufficient permissions")
1265 return Response({"root_id": root_doc.id})
1267 def retrieve(
1268 self,
1269 request: Request,
1270 *args: Any,
1271 **kwargs: Any,
1272 ) -> Response:
1273 response = super().retrieve(request, *args, **kwargs)
1274 if (
1275 "version" not in request.query_params
1276 or not isinstance(response.data, dict)
1277 or not ({"content", "page_count"} & response.data.keys())
1278 ):
1279 return response
1281 root_doc = self.get_object()
1282 content_doc = self._resolve_file_doc(root_doc, request)
1283 if "content" in response.data:
1284 response.data["content"] = content_doc.content or ""
1285 if "page_count" in response.data:
1286 response.data["page_count"] = content_doc.page_count
1287 return response
1289 def update(self, request, *args, **kwargs):
1290 partial = kwargs.pop("partial", False)
1291 root_doc = self.get_object()
1292 content_doc = (
1293 self._resolve_file_doc(root_doc, request)
1294 if "version" in request.query_params
1295 else get_latest_version_for_root(root_doc)
1296 )
1297 content_updated = "content" in request.data
1298 updated_content = request.data.get("content") if content_updated else None
1300 data = request.data.copy()
1301 serializer_partial = partial
1302 if content_updated and content_doc.id != root_doc.id:
1303 if updated_content is None:
1304 raise ValidationError({"content": ["This field may not be null."]})
1305 data.pop("content", None)
1306 serializer_partial = True
1308 serializer = self.get_serializer(
1309 root_doc,
1310 data=data,
1311 partial=serializer_partial,
1312 )
1313 serializer.is_valid(raise_exception=True)
1314 self.perform_update(serializer)
1316 if content_updated and content_doc.id != root_doc.id:
1317 content_doc.content = (
1318 str(updated_content) if updated_content is not None else ""
1319 )
1320 content_doc.save(update_fields=["content", "modified"])
1322 refreshed_doc = self.get_queryset().get(pk=root_doc.pk)
1323 response_data = self.get_serializer(refreshed_doc).data
1324 if "version" in request.query_params and "content" in response_data:
1325 response_data["content"] = content_doc.content
1326 response = Response(response_data)
1328 from documents.search import get_backend
1330 get_backend().add_or_update(refreshed_doc)
1332 document_updated.send(
1333 sender=self.__class__,
1334 document=refreshed_doc,
1335 )
1337 return response
1339 def list(self, request, *args, **kwargs):
1340 if not get_boolean( 1340 ↛ 1345line 1340 didn't jump to line 1345 because the condition on line 1340 was always true
1341 str(request.query_params.get("include_selection_data", "false")),
1342 ):
1343 return super().list(request, *args, **kwargs)
1345 queryset = self.filter_queryset(self.get_queryset())
1346 selection_data = self._get_selection_data_for_queryset(queryset)
1348 page = self.paginate_queryset(queryset)
1349 if page is not None:
1350 serializer = self.get_serializer(page, many=True)
1351 response = self.get_paginated_response(serializer.data)
1352 response.data["selection_data"] = selection_data
1353 return response
1355 serializer = self.get_serializer(queryset, many=True)
1356 return Response({"results": serializer.data, "selection_data": selection_data})
1358 def destroy(self, request, *args, **kwargs):
1359 from documents.search import get_backend
1361 get_backend().remove(self.get_object().pk)
1362 try:
1363 return super().destroy(request, *args, **kwargs)
1364 except Exception as e:
1365 if "Data too long for column" in str(e):
1366 logger.warning(
1367 "Detected a possible incompatible database column. See https://docs.paperless-ngx.com/troubleshooting/#convert-uuid-field",
1368 )
1369 logger.error(f"Error deleting document: {e!s}")
1370 return HttpResponseBadRequest(
1371 "Error deleting document, check logs for more detail.",
1372 )
1374 @staticmethod
1375 def original_requested(request):
1376 return (
1377 "original" in request.query_params
1378 and request.query_params["original"] == "true"
1379 )
1381 def _resolve_file_doc(self, root_doc: Document, request):
1382 version_requested = get_request_version_param(request) is not None
1383 resolution = resolve_requested_version_for_root(
1384 root_doc,
1385 request,
1386 include_deleted=version_requested,
1387 )
1388 if resolution.error == VersionResolutionError.INVALID:
1389 raise NotFound("Invalid version parameter")
1390 if resolution.document is None:
1391 raise Http404
1392 return resolution.document
1394 def _get_effective_file_doc(
1395 self,
1396 request_doc: Document,
1397 root_doc: Document,
1398 request: Request,
1399 ) -> Document:
1400 if (
1401 request_doc.root_document_id is not None
1402 and get_request_version_param(request) is None
1403 ):
1404 return request_doc
1405 return self._resolve_file_doc(root_doc, request)
1407 def _resolve_request_and_root_doc(
1408 self,
1409 pk,
1410 request: Request,
1411 *,
1412 include_deleted: bool = False,
1413 ) -> ResolvedRequestDocs | HttpResponseForbidden:
1414 manager = Document.global_objects if include_deleted else Document.objects
1415 try:
1416 request_doc = manager.select_related(
1417 "owner",
1418 "root_document",
1419 ).get(id=pk)
1420 except Document.DoesNotExist:
1421 raise Http404
1423 root_doc = get_root_document(
1424 request_doc,
1425 include_deleted=include_deleted,
1426 )
1427 if request.user is not None and not has_perms_owner_aware(
1428 request.user,
1429 "view_document",
1430 root_doc,
1431 ):
1432 return HttpResponseForbidden("Insufficient permissions")
1433 return ResolvedRequestDocs(request_doc=request_doc, root_doc=root_doc)
1435 def file_response(self, pk, request, disposition):
1436 resolved = self._resolve_request_and_root_doc(
1437 pk,
1438 request,
1439 include_deleted=True,
1440 )
1441 if isinstance(resolved, HttpResponseForbidden):
1442 return resolved
1443 file_doc = self._get_effective_file_doc(
1444 resolved.request_doc,
1445 resolved.root_doc,
1446 request,
1447 )
1448 return serve_file(
1449 doc=file_doc,
1450 use_archive=not self.original_requested(request)
1451 and file_doc.has_archive_version,
1452 disposition=disposition,
1453 follow_formatting=request.query_params.get("follow_formatting", False),
1454 )
1456 def get_metadata(self, file, mime_type):
1457 if not Path(file).is_file():
1458 return None
1460 parser_class = get_parser_registry().get_parser_for_file(
1461 mime_type,
1462 Path(file).name,
1463 Path(file),
1464 )
1465 if parser_class:
1466 try:
1467 with parser_class() as parser:
1468 return parser.extract_metadata(file, mime_type)
1469 except Exception: # pragma: no cover
1470 logger.exception(f"Issue getting metadata for {file}")
1471 return []
1472 else: # pragma: no cover
1473 logger.warning(f"No parser for {mime_type}")
1474 return []
1476 def get_filesize(self, filename):
1477 if Path(filename).is_file():
1478 return Path(filename).stat().st_size
1479 return None
1481 @action(methods=["get"], detail=True, filter_backends=[])
1482 @method_decorator(cache_control(no_cache=True))
1483 @method_decorator(
1484 condition(etag_func=metadata_etag, last_modified_func=metadata_last_modified),
1485 )
1486 def metadata(self, request, pk=None):
1487 resolved = self._resolve_request_and_root_doc(pk, request)
1488 if isinstance(resolved, HttpResponseForbidden):
1489 return resolved
1491 # Choose the effective document (newest version by default,
1492 # or explicit via ?version=).
1493 doc = self._get_effective_file_doc(
1494 resolved.request_doc,
1495 resolved.root_doc,
1496 request,
1497 )
1499 document_cached_metadata = get_metadata_cache(doc.pk)
1501 archive_metadata = None
1502 archive_filesize = (
1503 self.get_filesize(doc.archive_path) if doc.has_archive_version else None
1504 )
1505 if document_cached_metadata is not None:
1506 original_metadata = document_cached_metadata.original_metadata
1507 archive_metadata = document_cached_metadata.archive_metadata
1508 refresh_metadata_cache(doc.pk)
1509 else:
1510 original_metadata = self.get_metadata(doc.source_path, doc.mime_type)
1512 if doc.has_archive_version:
1513 archive_metadata = self.get_metadata(
1514 doc.archive_path,
1515 "application/pdf",
1516 )
1517 set_metadata_cache(doc, original_metadata, archive_metadata)
1519 meta = {
1520 "original_checksum": doc.checksum,
1521 "original_size": self.get_filesize(doc.source_path),
1522 "original_mime_type": doc.mime_type,
1523 "media_filename": doc.filename,
1524 "has_archive_version": doc.has_archive_version,
1525 "original_metadata": original_metadata,
1526 "archive_checksum": doc.archive_checksum,
1527 "archive_media_filename": doc.archive_filename,
1528 "original_filename": doc.original_filename,
1529 "archive_size": archive_filesize,
1530 "archive_metadata": archive_metadata,
1531 "barcodes": DocumentBarcodeSerializer(doc.barcodes.all(), many=True).data,
1532 }
1534 lang = "en"
1535 try:
1536 lang = detect(doc.content)
1537 except Exception:
1538 pass
1539 meta["lang"] = lang
1541 return Response(meta)
1543 @action(methods=["get"], detail=True, filter_backends=[])
1544 @method_decorator(cache_control(no_cache=True))
1545 @method_decorator(
1546 condition(
1547 etag_func=suggestions_etag,
1548 last_modified_func=suggestions_last_modified,
1549 ),
1550 )
1551 def suggestions(self, request, pk=None):
1552 doc = get_object_or_404(
1553 Document.objects.select_related("owner").prefetch_related("versions"),
1554 pk=pk,
1555 )
1556 if request.user is not None and not has_perms_owner_aware(
1557 request.user,
1558 "change_document",
1559 doc,
1560 ):
1561 return HttpResponseForbidden("Insufficient permissions")
1563 document_suggestions = get_suggestion_cache(doc.pk)
1565 if document_suggestions is not None:
1566 refresh_suggestions_cache(doc.pk)
1567 return Response(document_suggestions.suggestions)
1569 classifier = load_classifier()
1571 dates = []
1572 if settings.NUMBER_OF_SUGGESTED_DATES > 0:
1573 with get_date_parser() as date_parser:
1574 gen = date_parser.parse(doc.filename, doc.content)
1575 dates = sorted(
1576 {
1577 i
1578 for i in itertools.islice(
1579 gen,
1580 settings.NUMBER_OF_SUGGESTED_DATES,
1581 )
1582 },
1583 )
1585 resp_data = {
1586 "correspondents": [
1587 c.id for c in match_correspondents(doc, classifier, request.user)
1588 ],
1589 "tags": [t.id for t in match_tags(doc, classifier, request.user)],
1590 "document_types": [
1591 dt.id for dt in match_document_types(doc, classifier, request.user)
1592 ],
1593 "storage_paths": [
1594 dt.id for dt in match_storage_paths(doc, classifier, request.user)
1595 ],
1596 "dates": [date.strftime("%Y-%m-%d") for date in dates if date is not None],
1597 }
1599 # Cache the suggestions and the classifier hash for later
1600 set_suggestions_cache(doc.pk, resp_data, classifier)
1602 return Response(resp_data)
1604 @action(
1605 methods=["get"],
1606 detail=True,
1607 filter_backends=[],
1608 url_path="ai_suggestions",
1609 )
1610 @method_decorator(cache_control(no_cache=True))
1611 def ai_suggestions(self, request, pk=None):
1612 doc = get_object_or_404(
1613 Document.objects.select_related("owner").prefetch_related("versions"),
1614 pk=pk,
1615 )
1616 if request.user is not None and not has_perms_owner_aware(
1617 request.user,
1618 "change_document",
1619 doc,
1620 ):
1621 return HttpResponseForbidden("Insufficient permissions")
1623 ai_config = AIConfig()
1624 if not ai_config.ai_enabled:
1625 return HttpResponseBadRequest("AI is required for this feature")
1627 output_language = get_llm_output_language(
1628 ai_config=ai_config,
1629 user=request.user,
1630 )
1631 llm_cache_backend = ":".join(
1632 part
1633 for part in (
1634 ai_config.llm_backend,
1635 ai_config.llm_model,
1636 ai_config.llm_endpoint,
1637 output_language,
1638 f"user={request.user.pk}",
1639 )
1640 if part
1641 )
1643 cached_llm_suggestions = get_llm_suggestion_cache(
1644 doc.pk,
1645 backend=llm_cache_backend,
1646 )
1648 if cached_llm_suggestions:
1649 # Only the raw model choices are cached, never resolved object
1650 # ids. resolve_choice() below still runs permission filtering
1651 # freshly for this requester on every request, cache hit or not,
1652 # so a resolved id cached for one user's visibility can never be
1653 # handed unfiltered to a second, less-privileged requester of
1654 # the same (backend + user-keyed) cache entry.
1655 refresh_llm_suggestions_cache(
1656 doc.pk,
1657 backend=llm_cache_backend,
1658 )
1659 llm_suggestions = cached_llm_suggestions.suggestions
1660 else:
1661 try:
1662 llm_suggestions = get_ai_document_classification(
1663 doc,
1664 request.user,
1665 output_language,
1666 )
1667 except ValueError as exc:
1668 logger.exception(
1669 "Invalid AI configuration while generating suggestions for "
1670 "document %s: %s",
1671 doc.pk,
1672 exc,
1673 exc_info=True,
1674 )
1675 raise ValidationError(
1676 {"ai": [_("Invalid AI configuration.")]},
1677 ) from exc
1678 except LLMTimeoutError as exc:
1679 logger.exception(
1680 "AI backend timed out while generating suggestions for "
1681 "document %s: %s",
1682 doc.pk,
1683 exc,
1684 exc_info=True,
1685 )
1686 return Response(
1687 {"ai": [_("AI backend request timed out.")]},
1688 status=status.HTTP_503_SERVICE_UNAVAILABLE,
1689 )
1690 except LLMProviderError:
1691 logger.exception(
1692 "AI backend rejected the request for document %s",
1693 doc.pk,
1694 )
1695 return Response(
1696 {
1697 "ai": [
1698 _(
1699 "AI backend rejected the request. "
1700 "Check logs for details.",
1701 ),
1702 ],
1703 },
1704 status=status.HTTP_502_BAD_GATEWAY,
1705 )
1706 set_llm_suggestions_cache(
1707 doc.pk,
1708 llm_suggestions,
1709 backend=llm_cache_backend,
1710 )
1712 tags_choice: TaxonomyChoiceDict = llm_suggestions["tags"]
1713 correspondents_choice: TaxonomyChoiceDict = llm_suggestions["correspondents"]
1714 document_types_choice: TaxonomyChoiceDict = llm_suggestions["document_types"]
1715 storage_paths_choice: TaxonomyChoiceDict = llm_suggestions["storage_paths"]
1717 def resolve_choice(
1718 choice: "TaxonomyChoiceDict",
1719 resolve_ids: Callable[[list[int], User], list],
1720 match_names: Callable[[list[str], User], list],
1721 ) -> list:
1722 """The ids the model picked from the candidates it was shown, plus
1723 name matches for the values it proposed as new. The schema allows
1724 the same object to satisfy both an existing_id and a new_name in
1725 one valid response, so results are deduplicated by pk (keeping
1726 first-seen order) rather than trusting the two lookups to be
1727 disjoint.
1728 """
1729 matched = resolve_ids(choice["existing_ids"], request.user) + match_names(
1730 choice["new_names"],
1731 request.user,
1732 )
1733 seen_ids: set[int] = set()
1734 deduped = []
1735 for obj in matched:
1736 if obj.pk in seen_ids:
1737 continue
1738 seen_ids.add(obj.pk)
1739 deduped.append(obj)
1740 return deduped
1742 matched_tags = resolve_choice(
1743 tags_choice,
1744 resolve_tag_ids,
1745 match_tags_by_name,
1746 )
1747 matched_correspondents = resolve_choice(
1748 correspondents_choice,
1749 resolve_correspondent_ids,
1750 match_correspondents_by_name,
1751 )
1752 matched_types = resolve_choice(
1753 document_types_choice,
1754 resolve_document_type_ids,
1755 match_document_types_by_name,
1756 )
1757 matched_paths = resolve_choice(
1758 storage_paths_choice,
1759 resolve_storage_path_ids,
1760 match_storage_paths_by_name,
1761 )
1763 resp_data = {
1764 "title": llm_suggestions["title"],
1765 "tags": [t.id for t in matched_tags],
1766 "suggested_tags": extract_unmatched_names(
1767 tags_choice["new_names"],
1768 matched_tags,
1769 ),
1770 "correspondents": [c.id for c in matched_correspondents],
1771 "suggested_correspondents": extract_unmatched_names(
1772 correspondents_choice["new_names"],
1773 matched_correspondents,
1774 ),
1775 "document_types": [d.id for d in matched_types],
1776 "suggested_document_types": extract_unmatched_names(
1777 document_types_choice["new_names"],
1778 matched_types,
1779 ),
1780 "storage_paths": [s.id for s in matched_paths],
1781 "suggested_storage_paths": extract_unmatched_names(
1782 storage_paths_choice["new_names"],
1783 matched_paths,
1784 ),
1785 "dates": llm_suggestions["dates"],
1786 }
1788 return Response(resp_data)
1790 @action(methods=["get"], detail=True, filter_backends=[])
1791 @method_decorator(cache_control(no_cache=True))
1792 @method_decorator(
1793 condition(etag_func=preview_etag, last_modified_func=preview_last_modified),
1794 )
1795 def preview(self, request, pk=None):
1796 resolved = self._resolve_request_and_root_doc(pk, request, include_deleted=True)
1797 if isinstance(resolved, HttpResponseForbidden):
1798 return resolved
1800 try:
1801 file_doc = self._get_effective_file_doc(
1802 resolved.request_doc,
1803 resolved.root_doc,
1804 request,
1805 )
1807 return serve_file(
1808 doc=file_doc,
1809 use_archive=not self.original_requested(request)
1810 and file_doc.has_archive_version,
1811 disposition="inline",
1812 )
1813 except FileNotFoundError:
1814 raise Http404
1816 @action(methods=["get"], detail=True, filter_backends=[])
1817 @method_decorator(cache_control(no_cache=True))
1818 @method_decorator(
1819 condition(
1820 etag_func=thumbnail_etag,
1821 last_modified_func=thumbnail_last_modified,
1822 ),
1823 )
1824 def thumb(self, request, pk=None):
1825 resolved = self._resolve_request_and_root_doc(pk, request, include_deleted=True)
1826 if isinstance(resolved, HttpResponseForbidden):
1827 return resolved
1829 try:
1830 file_doc = self._get_effective_file_doc(
1831 resolved.request_doc,
1832 resolved.root_doc,
1833 request,
1834 )
1835 handle = file_doc.thumbnail_file
1837 return FileResponse(handle, content_type="image/webp")
1838 except FileNotFoundError:
1839 raise Http404
1841 @action(methods=["get"], detail=True)
1842 def download(self, request, pk=None):
1843 try:
1844 return self.file_response(pk, request, "attachment")
1845 except (FileNotFoundError, Document.DoesNotExist):
1846 raise Http404
1848 @action(
1849 methods=["get", "post", "delete"],
1850 detail=True,
1851 permission_classes=[PaperlessNotePermissions],
1852 pagination_class=None,
1853 filter_backends=[],
1854 )
1855 def notes(self, request, pk=None):
1856 currentUser = request.user
1857 try:
1858 doc = (
1859 Document.objects.select_related("owner")
1860 .prefetch_related("notes")
1861 .only("pk", "owner__id")
1862 .get(pk=pk)
1863 )
1864 if currentUser is not None and not has_perms_owner_aware(
1865 currentUser,
1866 "view_document",
1867 doc,
1868 ):
1869 return HttpResponseForbidden("Insufficient permissions to view notes")
1870 except Document.DoesNotExist:
1871 raise Http404
1873 serializer = self.get_serializer(doc)
1875 if request.method == "GET":
1876 try:
1877 notes = serializer.to_representation(doc).get("notes")
1878 return Response(notes)
1879 except Exception as e:
1880 logger.warning(f"An error occurred retrieving notes: {e!s}")
1881 return Response(
1882 {"error": "Error retrieving notes, check logs for more detail."},
1883 )
1884 elif request.method == "POST":
1885 try:
1886 if currentUser is not None and not has_perms_owner_aware(
1887 currentUser,
1888 "change_document",
1889 doc,
1890 ):
1891 return HttpResponseForbidden(
1892 "Insufficient permissions to create notes",
1893 )
1895 c = Note.objects.create(
1896 document=doc,
1897 note=request.data["note"],
1898 user=currentUser,
1899 )
1900 # If audit log is enabled make an entry in the log
1901 # about this note change
1902 if settings.AUDIT_LOG_ENABLED:
1903 LogEntry.objects.log_create(
1904 instance=doc,
1905 changes={
1906 "Note Added": ["None", c.id],
1907 },
1908 action=LogEntry.Action.UPDATE,
1909 )
1911 doc.modified = timezone.now()
1912 doc.save(update_fields=["modified"])
1914 from documents.search import get_backend
1916 get_backend().add_or_update(doc)
1918 notes = serializer.to_representation(doc).get("notes")
1920 return Response(notes)
1921 except Exception as e:
1922 logger.warning(f"An error occurred saving note: {e!s}")
1923 return Response(
1924 {
1925 "error": "Error saving note, check logs for more detail.",
1926 },
1927 )
1928 elif request.method == "DELETE":
1929 if currentUser is not None and not has_perms_owner_aware(
1930 currentUser,
1931 "change_document",
1932 doc,
1933 ):
1934 return HttpResponseForbidden("Insufficient permissions to delete notes")
1936 note_id = request.GET.get("id")
1937 if not note_id:
1938 raise ValidationError({"id": "This field is required."})
1939 try:
1940 note_id_int = int(note_id)
1941 except ValueError:
1942 raise ValidationError({"id": "A valid integer is required."})
1943 note = get_object_or_404(Note, id=note_id_int, document=doc)
1944 if settings.AUDIT_LOG_ENABLED:
1945 LogEntry.objects.log_create(
1946 instance=doc,
1947 changes={
1948 "Note Deleted": [note.id, "None"],
1949 },
1950 action=LogEntry.Action.UPDATE,
1951 )
1953 note.delete()
1955 doc.modified = timezone.now()
1956 doc.save(update_fields=["modified"])
1958 from documents.search import get_backend
1960 get_backend().add_or_update(doc)
1962 notes = serializer.to_representation(doc).get("notes")
1964 return Response(notes)
1966 return Response(
1967 {
1968 "error": "error",
1969 },
1970 )
1972 @action(methods=["get"], detail=True, filter_backends=[])
1973 def share_links(self, request, pk=None):
1974 currentUser = request.user
1975 try:
1976 doc = Document.objects.select_related("owner").get(pk=pk)
1977 if currentUser is not None and not has_perms_owner_aware(
1978 currentUser,
1979 "change_document",
1980 doc,
1981 ):
1982 return HttpResponseForbidden(
1983 "Insufficient permissions to add share link",
1984 )
1985 except Document.DoesNotExist:
1986 raise Http404
1988 if request.method == "GET":
1989 now = timezone.now()
1990 links = (
1991 ShareLink.objects.filter(document=doc)
1992 .select_related("document")
1993 .only(
1994 "pk",
1995 "created",
1996 "expiration",
1997 "slug",
1998 "document__title",
1999 )
2000 .exclude(expiration__lt=now)
2001 .order_by("-created")
2002 )
2003 serializer = ShareLinkSerializer(links, many=True)
2004 return Response(serializer.data)
2006 @action(methods=["get"], detail=True, name="Audit Trail", filter_backends=[])
2007 def history(self, request, pk=None):
2008 if not settings.AUDIT_LOG_ENABLED: 2008 ↛ 2009line 2008 didn't jump to line 2009 because the condition on line 2008 was never true
2009 return HttpResponseBadRequest("Audit log is disabled")
2010 try:
2011 doc = Document.objects.get(pk=pk)
2012 if not request.user.has_perm("auditlog.view_logentry") or (
2013 doc.owner is not None
2014 and doc.owner != request.user
2015 and not request.user.is_superuser
2016 ):
2017 return HttpResponseForbidden(
2018 "Insufficient permissions",
2019 )
2020 except Document.DoesNotExist: # pragma: no cover
2021 raise Http404
2023 # documents
2024 entries = [
2025 {
2026 "id": entry.id,
2027 "timestamp": entry.timestamp,
2028 "action": entry.get_action_display(),
2029 "changes": entry.changes,
2030 "actor": (
2031 {"id": entry.actor.id, "username": entry.actor.username}
2032 if entry.actor
2033 else None
2034 ),
2035 }
2036 for entry in LogEntry.objects.get_for_object(doc).select_related(
2037 "actor",
2038 )
2039 ]
2041 # custom fields
2042 for entry in LogEntry.objects.get_for_objects(
2043 doc.custom_fields.all(),
2044 ).select_related("actor"):
2045 entries.append(
2046 {
2047 "id": entry.id,
2048 "timestamp": entry.timestamp,
2049 "action": entry.get_action_display(),
2050 "changes": {
2051 "custom_fields": {
2052 "type": "custom_field",
2053 "field": str(entry.object_repr).split(":")[0].strip(),
2054 "value": str(entry.object_repr).split(":")[1].strip(),
2055 },
2056 },
2057 "actor": (
2058 {"id": entry.actor.id, "username": entry.actor.username}
2059 if entry.actor
2060 else None
2061 ),
2062 },
2063 )
2065 return Response(sorted(entries, key=lambda x: x["timestamp"], reverse=True))
2067 @extend_schema(
2068 operation_id="documents_email_document",
2069 deprecated=True,
2070 )
2071 @action(
2072 methods=["post"],
2073 detail=True,
2074 url_path="email",
2075 permission_classes=[IsAuthenticated, ViewDocumentsPermissions],
2076 )
2077 # TODO: deprecated, remove with drop of support for API v9
2078 def email_document(self, request, pk=None):
2079 request_data = request.data.copy()
2080 request_data.setlist("documents", [pk])
2081 return self.email_documents(request, data=request_data)
2083 @action(
2084 methods=["post"],
2085 detail=False,
2086 url_path="email",
2087 serializer_class=EmailSerializer,
2088 permission_classes=[IsAuthenticated, ViewDocumentsPermissions],
2089 )
2090 def email_documents(self, request, data=None):
2091 serializer = EmailSerializer(data=data or request.data)
2092 serializer.is_valid(raise_exception=True)
2094 validated_data = serializer.validated_data
2095 document_ids = validated_data.get("documents")
2096 addresses = validated_data.get("addresses").split(",")
2097 addresses = [addr.strip() for addr in addresses]
2098 subject = validated_data.get("subject")
2099 message = validated_data.get("message")
2100 use_archive_version = validated_data.get("use_archive_version", True)
2102 documents = Document.objects.filter(pk__in=document_ids)
2103 if (
2104 request.user is not None
2105 and documents.exclude(
2106 pk__in=permitted_document_ids(request.user),
2107 ).exists()
2108 ):
2109 return HttpResponseForbidden("Insufficient permissions")
2111 try:
2112 attachments: list[EmailAttachment] = []
2113 for doc in documents:
2114 attachment_path = (
2115 doc.archive_path
2116 if use_archive_version and doc.has_archive_version
2117 else doc.source_path
2118 )
2119 attachments.append(
2120 EmailAttachment(
2121 path=attachment_path,
2122 mime_type=doc.mime_type,
2123 friendly_name=doc.get_public_filename(
2124 archive=use_archive_version and doc.has_archive_version,
2125 ),
2126 ),
2127 )
2129 send_email(
2130 subject=subject,
2131 body=message,
2132 to=addresses,
2133 attachments=attachments,
2134 )
2136 logger.debug(
2137 f"Sent documents {[doc.id for doc in documents]} via email to {addresses}",
2138 )
2139 return Response({"message": "Email sent"})
2140 except Exception as e:
2141 logger.warning(f"An error occurred emailing documents: {e!s}")
2142 return HttpResponseServerError(
2143 "Error emailing documents, check logs for more detail.",
2144 )
2146 @extend_schema(
2147 operation_id="documents_update_version",
2148 request=DocumentVersionSerializer,
2149 responses={
2150 200: OpenApiTypes.STR,
2151 },
2152 )
2153 @action(methods=["post"], detail=True, parser_classes=[parsers.MultiPartParser])
2154 def update_version(self, request, pk=None):
2155 serializer = DocumentVersionSerializer(data=request.data)
2156 serializer.is_valid(raise_exception=True)
2158 try:
2159 request_doc = Document.objects.select_related(
2160 "owner",
2161 "root_document",
2162 ).get(pk=pk)
2163 root_doc = get_root_document(request_doc)
2164 if request.user is not None and (
2165 not request.user.has_perm("documents.change_document")
2166 or not has_perms_owner_aware(
2167 request.user,
2168 "change_document",
2169 root_doc,
2170 )
2171 ):
2172 return HttpResponseForbidden("Insufficient permissions")
2173 except Document.DoesNotExist:
2174 raise Http404
2176 try:
2177 doc_name, doc_data = serializer.validated_data.get("document")
2178 version_label = serializer.validated_data.get("version_label")
2180 t = int(mktime(datetime.now().timetuple()))
2182 settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
2184 temp_file_path = Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR)) / Path(
2185 pathvalidate.sanitize_filename(doc_name),
2186 )
2188 temp_file_path.write_bytes(doc_data)
2190 os.utime(temp_file_path, times=(t, t))
2192 input_doc = ConsumableDocument(
2193 source=DocumentSource.ApiUpload,
2194 original_file=temp_file_path,
2195 root_document_id=root_doc.pk,
2196 )
2198 overrides = DocumentMetadataOverrides()
2199 if version_label:
2200 overrides.version_label = version_label.strip()
2201 if request.user is not None:
2202 overrides.owner_id = request.user.id
2203 overrides.actor_id = request.user.id
2205 async_task = consume_file.apply_async(
2206 kwargs={"input_doc": input_doc, "overrides": overrides},
2207 headers={"trigger_source": PaperlessTask.TriggerSource.WEB_UI},
2208 )
2209 logger.debug(
2210 f"Updated document {root_doc.id} with new version",
2211 )
2212 return Response(async_task.id)
2213 except Exception as e:
2214 logger.warning(f"An error occurred updating document: {e!s}")
2215 return HttpResponseServerError(
2216 "Error updating document, check logs for more detail.",
2217 )
2219 def _get_root_doc_for_version_action(self, pk) -> Document:
2220 try:
2221 root_doc = Document.objects.select_related(
2222 "owner",
2223 "root_document",
2224 ).get(pk=pk)
2225 except Document.DoesNotExist:
2226 raise Http404
2227 return get_root_document(root_doc)
2229 def _get_version_doc_for_root(self, root_doc: Document, version_id) -> Document:
2230 try:
2231 version_doc = Document.objects.select_related("owner").get(
2232 pk=version_id,
2233 )
2234 except Document.DoesNotExist:
2235 raise Http404
2237 if (
2238 version_doc.id != root_doc.id
2239 and version_doc.root_document_id != root_doc.id
2240 ):
2241 raise Http404
2242 return version_doc
2244 @extend_schema(
2245 operation_id="documents_delete_version",
2246 parameters=[
2247 OpenApiParameter(
2248 name="version_id",
2249 type=OpenApiTypes.INT,
2250 location=OpenApiParameter.PATH,
2251 ),
2252 ],
2253 responses=inline_serializer(
2254 name="DeleteDocumentVersionResult",
2255 fields={
2256 "result": serializers.CharField(),
2257 "current_version_id": serializers.IntegerField(),
2258 },
2259 ),
2260 )
2261 @action(
2262 methods=["delete"],
2263 detail=True,
2264 url_path=r"versions/(?P<version_id>\d+)",
2265 )
2266 def delete_version(self, request, pk=None, version_id=None):
2267 root_doc = self._get_root_doc_for_version_action(pk)
2269 if request.user is not None and not has_perms_owner_aware(
2270 request.user,
2271 "delete_document",
2272 root_doc,
2273 ):
2274 return HttpResponseForbidden("Insufficient permissions")
2276 version_doc = self._get_version_doc_for_root(root_doc, version_id)
2278 if version_doc.id == root_doc.id:
2279 return HttpResponseBadRequest(
2280 "Cannot delete the root/original version. Delete the document instead.",
2281 )
2283 from documents.search import get_backend
2285 _backend = get_backend()
2286 _backend.remove(version_doc.pk)
2287 version_doc_id = version_doc.id
2288 version_doc.delete()
2289 root_doc.modified = timezone.now()
2290 Document.objects.filter(pk=root_doc.pk).update(modified=root_doc.modified)
2291 _backend.add_or_update(root_doc)
2292 if settings.AUDIT_LOG_ENABLED:
2293 actor = (
2294 request.user if request.user and request.user.is_authenticated else None
2295 )
2296 LogEntry.objects.log_create(
2297 instance=root_doc,
2298 changes={
2299 "Version Deleted": ["None", version_doc_id],
2300 },
2301 action=LogEntry.Action.UPDATE,
2302 actor=actor,
2303 additional_data={
2304 "reason": "Version deleted",
2305 "version_id": version_doc_id,
2306 },
2307 )
2309 current = versions_newest_first(
2310 Document.objects.filter(Q(id=root_doc.id) | Q(root_document=root_doc)),
2311 ).first()
2313 document_updated.send(
2314 sender=self.__class__,
2315 document=root_doc,
2316 )
2317 return Response(
2318 {
2319 "result": "OK",
2320 "current_version_id": current.id if current else root_doc.id,
2321 },
2322 )
2324 @extend_schema(
2325 operation_id="documents_update_version_label",
2326 request=DocumentVersionLabelSerializer,
2327 parameters=[
2328 OpenApiParameter(
2329 name="version_id",
2330 type=OpenApiTypes.INT,
2331 location=OpenApiParameter.PATH,
2332 ),
2333 ],
2334 responses=inline_serializer(
2335 name="UpdateDocumentVersionLabelResult",
2336 fields={
2337 "id": serializers.IntegerField(),
2338 "added": serializers.DateTimeField(),
2339 "version_label": serializers.CharField(
2340 required=False,
2341 allow_null=True,
2342 ),
2343 "checksum": serializers.CharField(
2344 required=False,
2345 allow_null=True,
2346 ),
2347 "is_root": serializers.BooleanField(),
2348 },
2349 ),
2350 )
2351 @delete_version.mapping.patch
2352 def update_version_label(self, request, pk=None, version_id=None):
2353 serializer = DocumentVersionLabelSerializer(data=request.data)
2354 serializer.is_valid(raise_exception=True)
2356 root_doc = self._get_root_doc_for_version_action(pk)
2357 if request.user is not None and not has_perms_owner_aware(
2358 request.user,
2359 "change_document",
2360 root_doc,
2361 ):
2362 return HttpResponseForbidden("Insufficient permissions")
2364 version_doc = self._get_version_doc_for_root(root_doc, version_id)
2365 old_label = version_doc.version_label
2366 version_doc.version_label = serializer.validated_data["version_label"]
2367 version_doc.save(update_fields=["version_label"])
2368 root_doc.modified = timezone.now()
2369 Document.objects.filter(pk=root_doc.pk).update(modified=root_doc.modified)
2371 if settings.AUDIT_LOG_ENABLED and old_label != version_doc.version_label:
2372 actor = (
2373 request.user if request.user and request.user.is_authenticated else None
2374 )
2375 LogEntry.objects.log_create(
2376 instance=root_doc,
2377 changes={
2378 "Version Label": [old_label, version_doc.version_label],
2379 },
2380 action=LogEntry.Action.UPDATE,
2381 actor=actor,
2382 additional_data={
2383 "reason": "Version label updated",
2384 "version_id": version_doc.id,
2385 },
2386 )
2388 document_updated.send(
2389 sender=self.__class__,
2390 document=root_doc,
2391 )
2393 return Response(
2394 {
2395 "id": version_doc.id,
2396 "added": version_doc.added,
2397 "version_label": version_doc.version_label,
2398 "checksum": version_doc.checksum,
2399 "is_root": version_doc.id == root_doc.id,
2400 },
2401 )
2404class ChatStreamingSerializer(serializers.Serializer[dict[str, Any]]):
2405 q = serializers.CharField(required=True, max_length=4000)
2406 document_id = serializers.IntegerField(required=False, allow_null=True)
2409@method_decorator(
2410 [
2411 ensure_csrf_cookie,
2412 cache_control(no_cache=True),
2413 ],
2414 name="dispatch",
2415)
2416class ChatStreamingView(GenericAPIView[Any]):
2417 permission_classes = (IsAuthenticated, ViewDocumentsPermissions)
2418 serializer_class = ChatStreamingSerializer
2420 def post(self, request, *args, **kwargs):
2421 ai_config = AIConfig()
2422 if not ai_config.ai_enabled: 2422 ↛ 2425line 2422 didn't jump to line 2425 because the condition on line 2422 was always true
2423 return HttpResponseBadRequest("AI is required for this feature")
2425 serializer = self.get_serializer(data=request.data)
2426 serializer.is_valid(raise_exception=True)
2427 question = serializer.validated_data["q"]
2429 doc_id = serializer.validated_data.get("document_id")
2431 if doc_id:
2432 try:
2433 document = Document.objects.get(id=doc_id)
2434 except Document.DoesNotExist:
2435 return HttpResponseBadRequest("Document not found")
2437 if not has_perms_owner_aware(request.user, "view_document", document):
2438 return HttpResponseForbidden("Insufficient permissions")
2440 documents = Document.objects.filter(pk=document.pk)
2441 unrestricted = False
2442 else:
2443 documents = Document.objects.filter(
2444 id__in=permitted_document_ids(request.user),
2445 )
2446 unrestricted = user_is_unrestricted(request.user)
2448 output_language = get_llm_output_language(
2449 ai_config=ai_config,
2450 user=request.user,
2451 )
2453 response = StreamingHttpResponse(
2454 stream_chat_with_documents(
2455 query_str=question,
2456 documents=documents,
2457 unrestricted=unrestricted,
2458 output_language=output_language,
2459 ),
2460 content_type="text/event-stream",
2461 )
2462 return response
2465@extend_schema_view(
2466 list=extend_schema(
2467 description="Document views including search",
2468 parameters=[
2469 OpenApiParameter(
2470 name="text",
2471 type=OpenApiTypes.STR,
2472 location=OpenApiParameter.QUERY,
2473 description="Simple Tantivy-backed text search query string",
2474 ),
2475 OpenApiParameter(
2476 name="title_search",
2477 type=OpenApiTypes.STR,
2478 location=OpenApiParameter.QUERY,
2479 description="Simple Tantivy-backed title-only search query string",
2480 ),
2481 OpenApiParameter(
2482 name="query",
2483 type=OpenApiTypes.STR,
2484 location=OpenApiParameter.QUERY,
2485 description="Advanced Tantivy search query string",
2486 ),
2487 OpenApiParameter(
2488 name="full_perms",
2489 type=OpenApiTypes.BOOL,
2490 location=OpenApiParameter.QUERY,
2491 ),
2492 OpenApiParameter(
2493 name="fields",
2494 type=OpenApiTypes.STR,
2495 many=True,
2496 location=OpenApiParameter.QUERY,
2497 ),
2498 ],
2499 responses={
2500 200: DocumentSerializer(many=True, all_fields=True),
2501 },
2502 ),
2503 next_asn=extend_schema(
2504 description="Get the next available Archive Serial Number (ASN) for a new document",
2505 responses={
2506 200: OpenApiTypes.INT,
2507 },
2508 ),
2509)
2510class UnifiedSearchViewSet(DocumentViewSet):
2511 def get_serializer_class(self):
2512 if self._is_search_request():
2513 return SearchResultSerializer
2514 return DocumentSerializer
2516 def get_serializer_context(self):
2517 if self._is_search_request():
2518 # BulkPermissionMixin.get_serializer_context() (inherited via
2519 # DocumentViewSet) assumes it's batching permissions for a page of
2520 # real Document instances. Tantivy search results are SearchHit/
2521 # dict-like objects instead, so skip straight past it here.
2522 return super(BulkPermissionMixin, self).get_serializer_context()
2523 return super().get_serializer_context()
2525 def _get_active_search_params(self, request: Request | None = None) -> list[str]:
2526 request = request or self.request
2527 return [
2528 param
2529 for param in _TANTIVY_SEARCH_PARAM_NAMES
2530 if param in request.query_params
2531 ]
2533 def _is_search_request(self):
2534 return bool(self._get_active_search_params())
2536 def list(self, request, *args, **kwargs):
2537 if not self._is_search_request():
2538 return super().list(request)
2540 from documents.search import SearchHit
2541 from documents.search import SearchQueryError
2542 from documents.search import TantivyBackend
2543 from documents.search import TantivyRelevanceList
2544 from documents.search import get_backend
2545 from documents.search import search_query_error_messages
2547 def parse_search_params() -> SearchParams:
2548 """Extract query string, search mode, and ordering from request."""
2549 active = self._get_active_search_params(request)
2550 if len(active) > 1:
2551 raise ValidationError(
2552 {
2553 "detail": _(
2554 "Specify only one of text, title_search, query, or more_like_id.",
2555 ),
2556 },
2557 )
2559 ordering_param = request.query_params.get("ordering", "")
2560 sort_reverse = ordering_param.startswith("-")
2561 sort_field_name = ordering_param.lstrip("-") or None
2562 # "score" means relevance order — Tantivy handles it natively,
2563 # so treat it as a Tantivy sort to preserve the ranked order through
2564 # the ORM intersection step.
2565 use_tantivy_sort = (
2566 sort_field_name in TantivyBackend.SORTABLE_FIELDS
2567 or sort_field_name is None
2568 or sort_field_name == "score"
2569 )
2571 try:
2572 page_num = int(request.query_params.get("page", 1))
2573 except (TypeError, ValueError):
2574 page_num = 1
2575 page_size = (
2576 self.paginator.get_page_size(request) or self.paginator.page_size
2577 )
2579 return SearchParams(
2580 sort_field_name=sort_field_name,
2581 sort_reverse=sort_reverse,
2582 use_tantivy_sort=use_tantivy_sort,
2583 page_num=page_num,
2584 page_size=page_size,
2585 )
2587 def intersect_and_order(
2588 all_ids: list[int],
2589 filtered_qs: QuerySet[Document],
2590 *,
2591 use_tantivy_sort: bool,
2592 ) -> list[int]:
2593 """Intersect search IDs with ORM-visible IDs, preserving order."""
2594 if not all_ids: 2594 ↛ 2596line 2594 didn't jump to line 2596 because the condition on line 2594 was always true
2595 return []
2596 if use_tantivy_sort:
2597 if len(all_ids) <= _TANTIVY_INTERSECT_THRESHOLD:
2598 # Small result set: targeted IN-clause avoids a full-table scan.
2599 visible_ids = set(
2600 filtered_qs.filter(pk__in=all_ids).values_list("pk", flat=True),
2601 )
2602 else:
2603 # Large result set: full-table scan + Python intersection is faster
2604 # than a large IN-clause on SQLite.
2605 visible_ids = set(
2606 filtered_qs.values_list("pk", flat=True),
2607 )
2608 return [doc_id for doc_id in all_ids if doc_id in visible_ids]
2609 return list(
2610 filtered_qs.filter(id__in=all_ids).values_list("pk", flat=True),
2611 )
2613 def run_text_search(
2614 backend: TantivyBackend,
2615 user: User | None,
2616 filtered_qs: QuerySet[Document],
2617 ) -> SearchResultPage:
2618 """Handle text/title/query search: IDs, ORM intersection, page highlights."""
2619 query_str, search_mode = _get_tantivy_query_and_mode(request.query_params)
2621 # "score" is not a real Tantivy sort field — it means relevance order,
2622 # which is Tantivy's default when no sort field is specified.
2623 is_score_sort = sort_field_name == "score"
2624 all_ids = backend.search_ids(
2625 query_str,
2626 user=user,
2627 sort_field=(
2628 None if (not use_tantivy_sort or is_score_sort) else sort_field_name
2629 ),
2630 sort_reverse=sort_reverse,
2631 search_mode=search_mode,
2632 )
2633 ordered_ids = intersect_and_order(
2634 all_ids,
2635 filtered_qs,
2636 use_tantivy_sort=use_tantivy_sort,
2637 )
2638 # Tantivy returns relevance results best-first (descending score).
2639 # ordering=score (ascending, worst-first) requires a reversal.
2640 if is_score_sort and not sort_reverse: 2640 ↛ 2641line 2640 didn't jump to line 2641 because the condition on line 2640 was never true
2641 ordered_ids = list(reversed(ordered_ids))
2643 page_offset = (page_num - 1) * page_size
2644 page_ids = ordered_ids[page_offset : page_offset + page_size]
2645 page_hits = backend.highlight_hits(
2646 query_str,
2647 page_ids,
2648 search_mode=search_mode,
2649 rank_start=page_offset + 1,
2650 )
2651 return SearchResultPage(
2652 ordered_ids=ordered_ids,
2653 hits=page_hits,
2654 page_offset=page_offset,
2655 )
2657 def run_more_like_this(
2658 backend: TantivyBackend,
2659 user: User | None,
2660 filtered_qs: QuerySet[Document],
2661 ) -> SearchResultPage:
2662 """Handle more_like_id search: permission check, IDs, stub hits."""
2663 more_like_doc_id = _get_more_like_id(request.query_params, user)
2665 all_ids = backend.more_like_this_ids(more_like_doc_id, user=user)
2666 ordered_ids = intersect_and_order(
2667 all_ids,
2668 filtered_qs,
2669 use_tantivy_sort=True,
2670 )
2672 page_offset = (page_num - 1) * page_size
2673 page_ids = ordered_ids[page_offset : page_offset + page_size]
2674 page_hits = [
2675 SearchHit(id=doc_id, score=0.0, rank=rank, highlights={})
2676 for rank, doc_id in enumerate(page_ids, start=page_offset + 1)
2677 ]
2678 return SearchResultPage(
2679 ordered_ids=ordered_ids,
2680 hits=page_hits,
2681 page_offset=page_offset,
2682 )
2684 try:
2685 sort_field_name, sort_reverse, use_tantivy_sort, page_num, page_size = (
2686 parse_search_params()
2687 )
2689 backend = get_backend()
2690 filtered_qs = self.filter_queryset(self.get_queryset())
2691 user = None if request.user.is_superuser else request.user
2693 if "more_like_id" in request.query_params: 2693 ↛ 2694line 2693 didn't jump to line 2694 because the condition on line 2693 was never true
2694 result = run_more_like_this(backend, user, filtered_qs)
2695 else:
2696 result = run_text_search(backend, user, filtered_qs)
2698 rl = TantivyRelevanceList(
2699 result.ordered_ids,
2700 result.hits,
2701 result.page_offset,
2702 )
2703 page = self.paginate_queryset(rl)
2705 if page is not None: 2705 ↛ 2723line 2705 didn't jump to line 2723 because the condition on line 2705 was always true
2706 serializer = self.get_serializer(page, many=True)
2707 response = self.get_paginated_response(serializer.data)
2708 response.data["corrected_query"] = None
2709 if get_boolean( 2709 ↛ 2716line 2709 didn't jump to line 2716 because the condition on line 2709 was never true
2710 str(request.query_params.get("include_selection_data", "false")),
2711 ):
2712 # NOTE: pk__in=ordered_ids generates a large SQL IN clause
2713 # for big result sets. Acceptable today but may need a temp
2714 # table or chunked approach if selection_data becomes slow
2715 # at scale (tens of thousands of matching documents).
2716 response.data["selection_data"] = (
2717 self._get_selection_data_for_queryset(
2718 filtered_qs.filter(pk__in=result.ordered_ids),
2719 )
2720 )
2721 return response
2723 serializer = self.get_serializer(result.hits, many=True)
2724 return Response(serializer.data)
2726 except NotFound:
2727 raise
2728 except PermissionDenied as e:
2729 invalid_more_like_id_message = _("Invalid more_like_id")
2730 if str(e.detail) == str(invalid_more_like_id_message):
2731 return HttpResponseForbidden(invalid_more_like_id_message)
2732 return HttpResponseForbidden(_("Insufficient permissions."))
2733 except ValidationError:
2734 raise
2735 except SearchQueryError as e:
2736 # User-fixable query error(s) (e.g. unparsable dates/numbers):
2737 # surface every offending field's message, not just the first,
2738 # so the user can fix them all in one round-trip.
2739 raise ValidationError({"query": search_query_error_messages(e)}) from e
2741 @action(detail=False, methods=["GET"], name="Get Next ASN")
2742 def next_asn(self, request, *args, **kwargs):
2743 max_asn = Document.objects.aggregate(
2744 Max("archive_serial_number", default=0),
2745 ).get(
2746 "archive_serial_number__max",
2747 )
2748 return Response(max_asn + 1)
2751@extend_schema_view(
2752 list=extend_schema(
2753 description="Logs view",
2754 responses={
2755 (200, "application/json"): serializers.ListSerializer(
2756 child=serializers.CharField(),
2757 ),
2758 },
2759 ),
2760 retrieve=extend_schema(
2761 description="Single log view",
2762 operation_id="retrieve_log",
2763 parameters=[
2764 OpenApiParameter(
2765 name="id",
2766 type=OpenApiTypes.STR,
2767 location=OpenApiParameter.PATH,
2768 ),
2769 OpenApiParameter(
2770 name="limit",
2771 type=OpenApiTypes.INT,
2772 location=OpenApiParameter.QUERY,
2773 description="Return only the last N entries from the log file",
2774 required=False,
2775 ),
2776 ],
2777 responses={
2778 (200, "application/json"): serializers.ListSerializer(
2779 child=serializers.CharField(),
2780 ),
2781 (404, "application/json"): None,
2782 },
2783 ),
2784)
2785class LogViewSet(ViewSet):
2786 permission_classes = (IsAuthenticated, PaperlessAdminPermissions)
2788 ALLOWED_LOG_FILES = {
2789 "paperless": "paperless.log",
2790 "mail": "mail.log",
2791 "celery": "celery.log",
2792 }
2794 def get_log_file(self, log_key: str) -> Path:
2795 return Path(settings.LOGGING_DIR) / self.ALLOWED_LOG_FILES[log_key]
2797 def retrieve(self, request, *args, **kwargs):
2798 log_key = kwargs.get("pk")
2799 if log_key not in self.ALLOWED_LOG_FILES:
2800 raise Http404
2802 log_file = self.get_log_file(log_key)
2804 if not log_file.is_file(): 2804 ↛ 2805line 2804 didn't jump to line 2805 because the condition on line 2804 was never true
2805 raise Http404
2807 limit_param = request.query_params.get("limit")
2808 if limit_param is not None:
2809 try:
2810 limit = int(limit_param)
2811 except (TypeError, ValueError):
2812 raise ValidationError({"limit": "Must be a positive integer"})
2813 if limit < 1:
2814 raise ValidationError({"limit": "Must be a positive integer"})
2815 else:
2816 limit = None
2818 with log_file.open() as f:
2819 if limit is None:
2820 lines = [line.rstrip() for line in f.readlines()]
2821 else:
2822 lines = [line.rstrip() for line in deque(f, maxlen=limit)]
2824 return Response(lines)
2826 def list(self, request, *args, **kwargs):
2827 existing_logs = [
2828 log_key
2829 for log_key in self.ALLOWED_LOG_FILES
2830 if self.get_log_file(log_key).is_file()
2831 ]
2832 return Response(existing_logs)
2835@extend_schema_view(**generate_object_with_permissions_schema(SavedViewSerializer))
2836class SavedViewViewSet(BulkPermissionMixin, PassUserMixin, ModelViewSet[SavedView]):
2837 model = SavedView
2839 queryset = SavedView.objects.select_related("owner").prefetch_related(
2840 "filter_rules",
2841 )
2842 serializer_class = SavedViewSerializer
2843 pagination_class = StandardPagination
2844 permission_classes = (IsAuthenticated, PaperlessObjectPermissions)
2845 filter_backends = (
2846 OrderingFilter,
2847 PermittedObjectsFilter,
2848 )
2849 ordering_fields = ("name",)
2852class DocumentSelectionMixin:
2853 def _get_search_document_ids(
2854 self,
2855 *,
2856 user: User,
2857 filters: dict[str, Any],
2858 ) -> list[int] | None:
2859 search_filters = [
2860 filter_name
2861 for filter_name in _TANTIVY_SEARCH_PARAM_NAMES
2862 if filter_name in filters
2863 ]
2864 if not search_filters: 2864 ↛ 2866line 2864 didn't jump to line 2866 because the condition on line 2864 was always true
2865 return None
2866 if len(search_filters) > 1:
2867 raise ValidationError(
2868 {
2869 "detail": _(
2870 "Specify only one of text, title_search, query, or more_like_id.",
2871 ),
2872 },
2873 )
2875 from documents.search import SearchQueryError
2876 from documents.search import get_backend
2877 from documents.search import search_query_error_messages
2879 filter_name = search_filters[0]
2880 backend = get_backend()
2881 search_user = None if user.is_superuser else user
2883 try:
2884 if filter_name == "more_like_id":
2885 more_like_doc_id = _get_more_like_id(filters, user)
2887 search_ids = backend.more_like_this_ids(
2888 more_like_doc_id,
2889 user=search_user,
2890 )
2891 else:
2892 query_str, search_mode = _get_tantivy_query_and_mode(filters)
2893 search_ids = backend.search_ids(
2894 query_str,
2895 user=search_user,
2896 search_mode=search_mode,
2897 )
2898 except SearchQueryError as e:
2899 # Same user-fixable-query mapping as the search list endpoint:
2900 # a bad date/number in a bulk selection filter is a 400 naming
2901 # the value, never a 500.
2902 raise ValidationError({"query": search_query_error_messages(e)}) from e
2904 return search_ids
2906 def _resolve_document_ids(
2907 self,
2908 *,
2909 user: User,
2910 validated_data: dict[str, Any],
2911 ) -> list[int]:
2912 if not validated_data.get("all", False):
2913 # if all is not true, just pass through the provided document ids
2914 return validated_data["documents"]
2916 # otherwise, reconstruct the document list based on the provided filters
2917 filters = validated_data.get("filters") or {}
2918 orm_filters = {
2919 key: value
2920 for key, value in filters.items()
2921 if key not in _TANTIVY_SEARCH_PARAM_NAMES
2922 }
2923 # Operations are addressed to roots, a caller that wants
2924 # to act on a specific version passes its id explicitly instead
2925 permitted_documents = Document.objects.filter(
2926 id__in=permitted_document_ids(user),
2927 root_document__isnull=True,
2928 )
2929 # orm-filtered docs
2930 filtered_documents = DocumentFilterSet(
2931 data=orm_filters,
2932 queryset=permitted_documents,
2933 user=user,
2934 ).qs.distinct()
2935 # tantivy-filtered docs (if search params provided)
2936 search_filtered_ids = self._get_search_document_ids(
2937 user=user,
2938 filters=filters,
2939 )
2940 if search_filtered_ids is not None: 2940 ↛ 2941line 2940 didn't jump to line 2941 because the condition on line 2940 was never true
2941 filtered_documents = filtered_documents.filter(pk__in=search_filtered_ids)
2942 if validated_data.get("excluded_documents"):
2943 filtered_documents = filtered_documents.exclude(
2944 pk__in=validated_data["excluded_documents"],
2945 )
2946 return list(filtered_documents.values_list("pk", flat=True))
2949class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
2950 permission_classes = (IsAuthenticated,)
2951 parser_classes = (parsers.JSONParser,)
2952 METHOD_NAMES_REQUIRING_USER = {
2953 "split",
2954 "merge",
2955 "rotate",
2956 "delete_pages",
2957 "edit_pdf",
2958 "remove_password",
2959 "merge_as_versions",
2960 }
2961 # merge_as_versions doesn't queue any consume tasks
2962 METHOD_NAMES_REQUIRING_TRIGGER_SOURCE = METHOD_NAMES_REQUIRING_USER - {
2963 "merge_as_versions",
2964 }
2966 def _has_document_permissions(
2967 self,
2968 *,
2969 user: User,
2970 documents: list[int],
2971 method,
2972 parameters: dict[str, Any],
2973 ) -> bool:
2974 if user.is_superuser: 2974 ↛ 2977line 2974 didn't jump to line 2977 because the condition on line 2974 was always true
2975 return True
2977 root_docs = {
2978 get_root_document(doc)
2979 for doc in Document.objects.select_related(
2980 "owner",
2981 "root_document__owner",
2982 ).filter(pk__in=documents)
2983 }
2984 user_is_owner_of_all_documents = all(
2985 (doc.owner == user or doc.owner is None) for doc in root_docs
2986 )
2988 # check global and object permissions for all documents
2989 has_perms = (
2990 user.has_perm(
2991 "documents.change_document",
2992 )
2993 and not Document.global_objects.filter(
2994 pk__in=[doc.pk for doc in root_docs],
2995 )
2996 .exclude(
2997 pk__in=permitted_document_ids(user, perm="change_document"),
2998 )
2999 .exists()
3000 )
3002 # check ownership for methods that change original document
3003 if (
3004 (
3005 has_perms
3006 and method
3007 in [
3008 bulk_edit.set_permissions,
3009 bulk_edit.delete,
3010 bulk_edit.rotate,
3011 bulk_edit.delete_pages,
3012 bulk_edit.edit_pdf,
3013 bulk_edit.merge_as_versions,
3014 bulk_edit.remove_password,
3015 ]
3016 )
3017 or (
3018 method in [bulk_edit.merge, bulk_edit.split]
3019 and parameters.get("delete_originals")
3020 )
3021 or (method == bulk_edit.edit_pdf and parameters.get("update_document"))
3022 ):
3023 has_perms = has_perms and user_is_owner_of_all_documents
3025 # check global add permissions for methods that create documents
3026 if (
3027 has_perms
3028 and (
3029 method in [bulk_edit.split, bulk_edit.merge]
3030 or (
3031 method in [bulk_edit.edit_pdf, bulk_edit.remove_password]
3032 and not parameters.get("update_document")
3033 )
3034 )
3035 and not user.has_perm("documents.add_document")
3036 ):
3037 has_perms = False
3039 # check global delete permissions for methods that delete documents
3040 if (
3041 has_perms
3042 and (
3043 method == bulk_edit.delete
3044 # Sources stop being documents of their own, and removing one
3045 # again afterwards needs delete_document
3046 or method == bulk_edit.merge_as_versions
3047 or (
3048 method in [bulk_edit.merge, bulk_edit.split]
3049 and parameters.get("delete_originals")
3050 )
3051 or (
3052 method in [bulk_edit.edit_pdf, bulk_edit.remove_password]
3053 and parameters.get("delete_original")
3054 and not parameters.get("update_document")
3055 )
3056 )
3057 and not user.has_perm("documents.delete_document")
3058 ):
3059 has_perms = False
3061 return has_perms
3063 def _execute_document_action(
3064 self,
3065 *,
3066 method,
3067 validated_data: dict[str, Any],
3068 operation_label: str,
3069 ):
3070 documents = self._resolve_document_ids(
3071 user=self.request.user,
3072 validated_data=validated_data,
3073 )
3074 parameters = {
3075 k: v
3076 for k, v in validated_data.items()
3077 if k
3078 not in {
3079 "documents",
3080 "all",
3081 "filters",
3082 "excluded_documents",
3083 "from_webui",
3084 }
3085 }
3086 user = self.request.user
3087 from_webui = validated_data.get("from_webui", False)
3089 if method.__name__ in self.METHOD_NAMES_REQUIRING_USER:
3090 parameters["user"] = user
3091 if method.__name__ in self.METHOD_NAMES_REQUIRING_TRIGGER_SOURCE:
3092 parameters["trigger_source"] = (
3093 PaperlessTask.TriggerSource.WEB_UI
3094 if from_webui
3095 else PaperlessTask.TriggerSource.API_UPLOAD
3096 )
3098 if not self._has_document_permissions( 3098 ↛ 3104line 3098 didn't jump to line 3104 because the condition on line 3098 was never true
3099 user=user,
3100 documents=documents,
3101 method=method,
3102 parameters=parameters,
3103 ):
3104 return HttpResponseForbidden("Insufficient permissions")
3106 try:
3107 result = method(documents, **parameters)
3108 return Response({"result": result})
3109 except Exception as e:
3110 logger.warning(f"An error occurred performing {operation_label}: {e!s}")
3111 return HttpResponseBadRequest(
3112 f"Error performing {operation_label}, check logs for more detail.",
3113 )
3116@extend_schema_view(
3117 post=extend_schema(
3118 operation_id="bulk_edit",
3119 description="Perform a bulk edit operation on a list of documents",
3120 external_docs={
3121 "description": "Further documentation",
3122 "url": "https://docs.paperless-ngx.com/api/#bulk-editing",
3123 },
3124 responses={
3125 200: inline_serializer(
3126 name="BulkEditDocumentsResult",
3127 fields={
3128 "result": serializers.CharField(),
3129 },
3130 ),
3131 },
3132 ),
3133)
3134class BulkEditView(DocumentOperationPermissionMixin):
3135 MODIFIED_FIELD_BY_METHOD = {
3136 "set_correspondent": "correspondent",
3137 "set_document_type": "document_type",
3138 "set_storage_path": "storage_path",
3139 "add_tag": "tags",
3140 "remove_tag": "tags",
3141 "modify_tags": "tags",
3142 "modify_custom_fields": "custom_fields",
3143 "set_permissions": None,
3144 "delete": "deleted_at",
3145 # These operations create new documents/versions no longer altering
3146 # fields on the selected document in place
3147 "rotate": None,
3148 "delete_pages": None,
3149 "split": None,
3150 "merge": None,
3151 "edit_pdf": None,
3152 "reprocess": "checksum",
3153 "remove_password": None,
3154 }
3156 serializer_class = BulkEditSerializer
3158 @staticmethod
3159 def _snapshot_field(doc_ids: list[int], field: str) -> dict[int, Any]:
3160 """
3161 Returns each document's current value of field, for the audit log.
3163 Tags and custom fields are one row per value, so they are gathered
3164 into a sorted list of pks per document (empty when there are none).
3165 Reading them through Document.values() instead would join those rows
3166 and return one arbitrary value per document.
3167 """
3168 if field == "tags": 3168 ↛ 3169line 3168 didn't jump to line 3169 because the condition on line 3168 was never true
3169 rows = (
3170 Document.tags.through.objects.filter(document_id__in=doc_ids)
3171 .order_by("tag_id")
3172 .values_list("document_id", "tag_id")
3173 )
3174 elif field == "custom_fields": 3174 ↛ 3175line 3174 didn't jump to line 3175 because the condition on line 3174 was never true
3175 rows = (
3176 CustomFieldInstance.objects.filter(document_id__in=doc_ids)
3177 .order_by("pk")
3178 .values_list("document_id", "pk")
3179 )
3180 else:
3181 return dict(
3182 Document.objects.filter(pk__in=doc_ids).values_list("pk", field),
3183 )
3185 values: dict[int, list[int]] = {doc_id: [] for doc_id in doc_ids}
3186 for doc_id, pk in rows:
3187 values[doc_id].append(pk)
3188 return values
3190 def post(self, request, *args, **kwargs):
3191 request_method = request.data.get("method")
3192 api_version = int(request.version or settings.REST_FRAMEWORK["DEFAULT_VERSION"])
3193 # TODO: remove this and related backwards compatibility code when API v9 is dropped
3194 if request_method in BulkEditSerializer.LEGACY_DOCUMENT_ACTION_METHODS:
3195 endpoint = BulkEditSerializer.MOVED_DOCUMENT_ACTION_ENDPOINTS[
3196 request_method
3197 ]
3198 logger.warning(
3199 "Deprecated bulk_edit method '%s' requested on API version %s. "
3200 "Use '%s' instead.",
3201 request_method,
3202 api_version,
3203 endpoint,
3204 )
3206 serializer = self.get_serializer(data=request.data)
3207 serializer.is_valid(raise_exception=True)
3209 user = self.request.user
3210 method = serializer.validated_data.get("method")
3211 parameters = serializer.validated_data.get("parameters")
3212 from_webui = serializer.validated_data.get("from_webui", False)
3213 documents = self._resolve_document_ids(
3214 user=user,
3215 validated_data=serializer.validated_data,
3216 )
3217 if method.__name__ in self.METHOD_NAMES_REQUIRING_USER:
3218 parameters["user"] = user
3219 if method.__name__ in self.METHOD_NAMES_REQUIRING_TRIGGER_SOURCE:
3220 parameters["trigger_source"] = (
3221 PaperlessTask.TriggerSource.WEB_UI
3222 if from_webui
3223 else PaperlessTask.TriggerSource.API_UPLOAD
3224 )
3225 if not self._has_document_permissions( 3225 ↛ 3231line 3225 didn't jump to line 3231 because the condition on line 3225 was never true
3226 user=user,
3227 documents=documents,
3228 method=method,
3229 parameters=parameters,
3230 ):
3231 return HttpResponseForbidden("Insufficient permissions")
3233 try:
3234 modified_field = self.MODIFIED_FIELD_BY_METHOD.get(method.__name__, None)
3235 if settings.AUDIT_LOG_ENABLED and modified_field:
3236 old_values = self._snapshot_field(documents, modified_field)
3238 result = method(documents, **parameters)
3240 if settings.AUDIT_LOG_ENABLED and modified_field:
3241 new_values = self._snapshot_field(documents, modified_field)
3242 for doc in Document.objects.filter(pk__in=documents): 3242 ↛ 3243line 3242 didn't jump to line 3243 because the loop on line 3242 never started
3243 LogEntry.objects.log_create(
3244 instance=doc,
3245 changes={
3246 modified_field: [
3247 old_values[doc.pk],
3248 new_values[doc.pk],
3249 ],
3250 },
3251 action=LogEntry.Action.UPDATE,
3252 actor=user,
3253 additional_data={
3254 "reason": f"Bulk edit: {method.__name__}",
3255 },
3256 )
3258 return Response({"result": result})
3259 except Exception as e:
3260 logger.warning(f"An error occurred performing bulk edit: {e!s}")
3261 return HttpResponseBadRequest(
3262 "Error performing bulk edit, check logs for more detail.",
3263 )
3266@extend_schema_view(
3267 post=extend_schema(
3268 operation_id="documents_rotate",
3269 description="Rotate one or more documents",
3270 responses={
3271 200: inline_serializer(
3272 name="RotateDocumentsResult",
3273 fields={
3274 "result": serializers.CharField(),
3275 },
3276 ),
3277 },
3278 ),
3279)
3280class RotateDocumentsView(DocumentOperationPermissionMixin):
3281 serializer_class = RotateDocumentsSerializer
3283 def post(self, request, *args, **kwargs):
3284 serializer = self.get_serializer(data=request.data)
3285 serializer.is_valid(raise_exception=True)
3286 return self._execute_document_action(
3287 method=bulk_edit.rotate,
3288 validated_data=serializer.validated_data,
3289 operation_label="document rotate",
3290 )
3293@extend_schema_view(
3294 post=extend_schema(
3295 operation_id="documents_merge",
3296 description="Merge selected documents into a new document",
3297 responses={
3298 200: inline_serializer(
3299 name="MergeDocumentsResult",
3300 fields={
3301 "result": serializers.CharField(),
3302 },
3303 ),
3304 },
3305 ),
3306)
3307class MergeDocumentsView(DocumentOperationPermissionMixin):
3308 serializer_class = MergeDocumentsSerializer
3310 def post(self, request, *args, **kwargs):
3311 serializer = self.get_serializer(data=request.data)
3312 serializer.is_valid(raise_exception=True)
3313 return self._execute_document_action(
3314 method=bulk_edit.merge,
3315 validated_data=serializer.validated_data,
3316 operation_label="document merge",
3317 )
3320@extend_schema_view(
3321 post=extend_schema(
3322 operation_id="documents_merge_as_versions",
3323 description="Merge selected documents as versions of a chosen root document",
3324 responses={
3325 200: inline_serializer(
3326 name="MergeDocumentsAsVersionsResult",
3327 fields={
3328 "result": serializers.CharField(),
3329 },
3330 ),
3331 },
3332 ),
3333)
3334class MergeDocumentsAsVersionsView(DocumentOperationPermissionMixin):
3335 serializer_class = MergeDocumentsAsVersionsSerializer
3337 def post(self, request, *args, **kwargs):
3338 serializer = self.get_serializer(data=request.data)
3339 serializer.is_valid(raise_exception=True)
3340 return self._execute_document_action(
3341 method=bulk_edit.merge_as_versions,
3342 validated_data=serializer.validated_data,
3343 operation_label="document merge as versions",
3344 )
3347@extend_schema_view(
3348 post=extend_schema(
3349 operation_id="documents_delete",
3350 description="Move selected documents to trash",
3351 responses={
3352 200: inline_serializer(
3353 name="DeleteDocumentsResult",
3354 fields={
3355 "result": serializers.CharField(),
3356 },
3357 ),
3358 },
3359 ),
3360)
3361class DeleteDocumentsView(DocumentOperationPermissionMixin):
3362 serializer_class = DeleteDocumentsSerializer
3364 def post(self, request, *args, **kwargs):
3365 serializer = self.get_serializer(data=request.data)
3366 serializer.is_valid(raise_exception=True)
3367 return self._execute_document_action(
3368 method=bulk_edit.delete,
3369 validated_data=serializer.validated_data,
3370 operation_label="document delete",
3371 )
3374@extend_schema_view(
3375 post=extend_schema(
3376 operation_id="documents_reprocess",
3377 description="Reprocess selected documents",
3378 responses={
3379 200: inline_serializer(
3380 name="ReprocessDocumentsResult",
3381 fields={
3382 "result": serializers.CharField(),
3383 },
3384 ),
3385 },
3386 ),
3387)
3388class ReprocessDocumentsView(DocumentOperationPermissionMixin):
3389 serializer_class = ReprocessDocumentsSerializer
3391 def post(self, request, *args, **kwargs):
3392 serializer = self.get_serializer(data=request.data)
3393 serializer.is_valid(raise_exception=True)
3394 return self._execute_document_action(
3395 method=bulk_edit.reprocess,
3396 validated_data=serializer.validated_data,
3397 operation_label="document reprocess",
3398 )
3401@extend_schema_view(
3402 post=extend_schema(
3403 operation_id="documents_edit_pdf",
3404 description="Perform PDF edit operations on a selected document",
3405 responses={
3406 200: inline_serializer(
3407 name="EditPdfDocumentsResult",
3408 fields={
3409 "result": serializers.CharField(),
3410 },
3411 ),
3412 },
3413 ),
3414)
3415class EditPdfDocumentsView(DocumentOperationPermissionMixin):
3416 serializer_class = EditPdfDocumentsSerializer
3418 def post(self, request, *args, **kwargs):
3419 serializer = self.get_serializer(data=request.data)
3420 serializer.is_valid(raise_exception=True)
3421 return self._execute_document_action(
3422 method=bulk_edit.edit_pdf,
3423 validated_data=serializer.validated_data,
3424 operation_label="PDF edit",
3425 )
3428@extend_schema_view(
3429 post=extend_schema(
3430 operation_id="documents_remove_password",
3431 description="Remove password protection from selected PDFs",
3432 responses={
3433 200: inline_serializer(
3434 name="RemovePasswordDocumentsResult",
3435 fields={
3436 "result": serializers.CharField(),
3437 },
3438 ),
3439 },
3440 ),
3441)
3442class RemovePasswordDocumentsView(DocumentOperationPermissionMixin):
3443 serializer_class = RemovePasswordDocumentsSerializer
3445 def post(self, request, *args, **kwargs):
3446 serializer = self.get_serializer(data=request.data)
3447 serializer.is_valid(raise_exception=True)
3448 return self._execute_document_action(
3449 method=bulk_edit.remove_password,
3450 validated_data=serializer.validated_data,
3451 operation_label="password removal",
3452 )
3455@extend_schema_view(
3456 post=extend_schema(
3457 description="Upload a document via the API",
3458 external_docs={
3459 "description": "Further documentation",
3460 "url": "https://docs.paperless-ngx.com/api/#file-uploads",
3461 },
3462 responses={
3463 (200, "application/json"): OpenApiTypes.STR,
3464 },
3465 ),
3466)
3467class PostDocumentView(GenericAPIView[Any]):
3468 permission_classes = (IsAuthenticated,)
3469 serializer_class = PostDocumentSerializer
3470 parser_classes = (parsers.MultiPartParser,)
3472 def post(self, request, *args, **kwargs):
3473 if not request.user.has_perm("documents.add_document"): 3473 ↛ 3474line 3473 didn't jump to line 3474 because the condition on line 3473 was never true
3474 return HttpResponseForbidden("Insufficient permissions")
3475 serializer = self.get_serializer(data=request.data)
3476 serializer.is_valid(raise_exception=True)
3478 doc_name, doc_data = serializer.validated_data.get("document")
3479 doc_name = normalize("NFC", doc_name)
3480 correspondent_id = serializer.validated_data.get("correspondent")
3481 document_type_id = serializer.validated_data.get("document_type")
3482 storage_path_id = serializer.validated_data.get("storage_path")
3483 tag_ids = serializer.validated_data.get("tags")
3484 title = serializer.validated_data.get("title")
3485 created = serializer.validated_data.get("created")
3486 archive_serial_number = serializer.validated_data.get("archive_serial_number")
3487 cf = serializer.validated_data.get("custom_fields")
3488 from_webui = serializer.validated_data.get("from_webui")
3490 t = int(mktime(datetime.now().timetuple()))
3492 settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
3494 temp_file_path = Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR)) / Path(
3495 pathvalidate.sanitize_filename(doc_name),
3496 )
3498 temp_file_path.write_bytes(doc_data)
3500 os.utime(temp_file_path, times=(t, t))
3502 input_doc = ConsumableDocument(
3503 source=DocumentSource.WebUI if from_webui else DocumentSource.ApiUpload,
3504 original_file=temp_file_path,
3505 )
3506 custom_fields = None
3507 if isinstance(cf, dict) and cf: 3507 ↛ 3508line 3507 didn't jump to line 3508 because the condition on line 3507 was never true
3508 custom_fields = cf
3509 elif isinstance(cf, list) and cf: 3509 ↛ 3510line 3509 didn't jump to line 3510 because the condition on line 3509 was never true
3510 custom_fields = dict.fromkeys(cf, None)
3511 input_doc_overrides = DocumentMetadataOverrides(
3512 filename=doc_name,
3513 title=title,
3514 correspondent_id=correspondent_id,
3515 document_type_id=document_type_id,
3516 storage_path_id=storage_path_id,
3517 tag_ids=tag_ids,
3518 created=created,
3519 asn=archive_serial_number,
3520 owner_id=request.user.id,
3521 custom_fields=custom_fields,
3522 )
3524 async_task = consume_file.apply_async(
3525 kwargs={"input_doc": input_doc, "overrides": input_doc_overrides},
3526 headers={
3527 "trigger_source": (
3528 PaperlessTask.TriggerSource.WEB_UI
3529 if from_webui
3530 else PaperlessTask.TriggerSource.API_UPLOAD
3531 ),
3532 },
3533 )
3535 return Response(async_task.id)
3538@extend_schema_view(
3539 post=extend_schema(
3540 description="Get selection data for the selected documents",
3541 responses={
3542 (200, "application/json"): inline_serializer(
3543 name="SelectionData",
3544 fields={
3545 "selected_correspondents": serializers.ListSerializer(
3546 child=inline_serializer(
3547 name="CorrespondentCounts",
3548 fields={
3549 "id": serializers.IntegerField(),
3550 "document_count": serializers.IntegerField(),
3551 },
3552 ),
3553 ),
3554 "selected_tags": serializers.ListSerializer(
3555 child=inline_serializer(
3556 name="TagCounts",
3557 fields={
3558 "id": serializers.IntegerField(),
3559 "document_count": serializers.IntegerField(),
3560 },
3561 ),
3562 ),
3563 "selected_document_types": serializers.ListSerializer(
3564 child=inline_serializer(
3565 name="DocumentTypeCounts",
3566 fields={
3567 "id": serializers.IntegerField(),
3568 "document_count": serializers.IntegerField(),
3569 },
3570 ),
3571 ),
3572 "selected_storage_paths": serializers.ListSerializer(
3573 child=inline_serializer(
3574 name="StoragePathCounts",
3575 fields={
3576 "id": serializers.IntegerField(),
3577 "document_count": serializers.IntegerField(),
3578 },
3579 ),
3580 ),
3581 "selected_custom_fields": serializers.ListSerializer(
3582 child=inline_serializer(
3583 name="CustomFieldCounts",
3584 fields={
3585 "id": serializers.IntegerField(),
3586 "document_count": serializers.IntegerField(),
3587 },
3588 ),
3589 ),
3590 },
3591 ),
3592 },
3593 ),
3594)
3595class SelectionDataView(DocumentSelectionMixin, GenericAPIView[Any]):
3596 permission_classes = (IsAuthenticated, ViewDocumentsPermissions)
3597 serializer_class = DocumentSelectionSerializer
3598 parser_classes = (parsers.MultiPartParser, parsers.JSONParser)
3600 def post(self, request, format=None):
3601 serializer = self.get_serializer(data=request.data)
3602 serializer.is_valid(raise_exception=True)
3604 ids = self._resolve_document_ids(
3605 user=request.user,
3606 validated_data=serializer.validated_data,
3607 )
3608 permitted_documents = Document.objects.filter(
3609 id__in=permitted_document_ids(request.user),
3610 )
3611 if permitted_documents.filter(pk__in=ids).count() != len(ids): 3611 ↛ 3612line 3611 didn't jump to line 3612 because the condition on line 3611 was never true
3612 return HttpResponseForbidden("Insufficient permissions")
3614 correspondents = Correspondent.objects.annotate(
3615 document_count=Count(
3616 Case(When(documents__id__in=ids, then=1), output_field=IntegerField()),
3617 ),
3618 )
3620 tags = Tag.objects.annotate(
3621 document_count=Count(
3622 Case(When(documents__id__in=ids, then=1), output_field=IntegerField()),
3623 ),
3624 )
3626 types = DocumentType.objects.annotate(
3627 document_count=Count(
3628 Case(When(documents__id__in=ids, then=1), output_field=IntegerField()),
3629 ),
3630 )
3632 storage_paths = StoragePath.objects.annotate(
3633 document_count=Count(
3634 Case(When(documents__id__in=ids, then=1), output_field=IntegerField()),
3635 ),
3636 )
3638 custom_fields = CustomField.objects.annotate(
3639 document_count=Count(
3640 Case(
3641 When(
3642 fields__document__id__in=ids,
3643 then=1,
3644 ),
3645 output_field=IntegerField(),
3646 ),
3647 ),
3648 )
3650 r = Response(
3651 {
3652 "selected_correspondents": [
3653 {"id": t.id, "document_count": t.document_count}
3654 for t in correspondents
3655 ],
3656 "selected_tags": [
3657 {"id": t.id, "document_count": t.document_count} for t in tags
3658 ],
3659 "selected_document_types": [
3660 {"id": t.id, "document_count": t.document_count} for t in types
3661 ],
3662 "selected_storage_paths": [
3663 {"id": t.id, "document_count": t.document_count}
3664 for t in storage_paths
3665 ],
3666 "selected_custom_fields": [
3667 {"id": t.id, "document_count": t.document_count}
3668 for t in custom_fields
3669 ],
3670 },
3671 )
3673 return r
3676@extend_schema_view(
3677 get=extend_schema(
3678 description="Get a list of all available tags",
3679 parameters=[
3680 OpenApiParameter(
3681 name="term",
3682 required=False,
3683 type=str,
3684 description="Term to search for",
3685 ),
3686 OpenApiParameter(
3687 name="limit",
3688 required=False,
3689 type=int,
3690 description="Number of completions to return",
3691 ),
3692 ],
3693 responses={
3694 (200, "application/json"): serializers.ListSerializer(
3695 child=serializers.CharField(),
3696 ),
3697 },
3698 ),
3699)
3700class SearchAutoCompleteView(GenericAPIView[Any]):
3701 permission_classes = (IsAuthenticated, ViewDocumentsPermissions)
3703 def get(self, request, format=None):
3704 user = self.request.user if hasattr(self.request, "user") else None
3706 if "term" in request.query_params:
3707 term = request.query_params["term"].strip()
3708 else:
3709 return HttpResponseBadRequest("Term required")
3711 if "limit" in request.query_params:
3712 limit = int(request.query_params["limit"])
3713 if limit <= 0:
3714 return HttpResponseBadRequest("Invalid limit")
3715 else:
3716 limit = 10
3718 from documents.search import get_backend
3720 return Response(get_backend().autocomplete(term, limit, user))
3723@extend_schema_view(
3724 get=extend_schema(
3725 description="Global search",
3726 parameters=[
3727 OpenApiParameter(
3728 name="query",
3729 required=True,
3730 type=str,
3731 description="Query to search for",
3732 ),
3733 OpenApiParameter(
3734 name="db_only",
3735 required=False,
3736 type=bool,
3737 description="Search only the database",
3738 ),
3739 ],
3740 responses={
3741 (200, "application/json"): inline_serializer(
3742 name="SearchResult",
3743 fields={
3744 "total": serializers.IntegerField(),
3745 "documents": DocumentSerializer(many=True),
3746 "saved_views": SavedViewSerializer(many=True),
3747 "tags": TagSerializer(many=True),
3748 "correspondents": CorrespondentSerializer(many=True),
3749 "document_types": DocumentTypeSerializer(many=True),
3750 "storage_paths": StoragePathSerializer(many=True),
3751 "users": UserSerializer(many=True),
3752 "groups": GroupSerializer(many=True),
3753 "mail_rules": MailRuleSerializer(many=True),
3754 "mail_accounts": MailAccountSerializer(many=True),
3755 "workflows": WorkflowSerializer(many=True),
3756 "custom_fields": CustomFieldSerializer(many=True),
3757 },
3758 ),
3759 },
3760 ),
3761)
3762class GlobalSearchView(PassUserMixin):
3763 permission_classes = (IsAuthenticated,)
3764 serializer_class = SearchResultSerializer
3766 def get(self, request, *args, **kwargs):
3767 from documents.search import SearchMode
3768 from documents.search import get_backend
3770 query = request.query_params.get("query", None)
3771 if query is None:
3772 return HttpResponseBadRequest("Query required")
3773 if len(query) < 3:
3774 return HttpResponseBadRequest("Query must be at least 3 characters")
3775 if len(query) > _MAX_QUERY_LENGTH: 3775 ↛ 3776line 3775 didn't jump to line 3776 because the condition on line 3775 was never true
3776 return HttpResponseBadRequest(
3777 f"Query must be at most {_MAX_QUERY_LENGTH} characters",
3778 )
3780 db_only = request.query_params.get("db_only", False)
3782 OBJECT_LIMIT = 3
3783 docs = []
3784 if request.user.has_perm("documents.view_document"): 3784 ↛ 3809line 3784 didn't jump to line 3809 because the condition on line 3784 was always true
3785 # Never more than OBJECT_LIMIT rows come back here, so annotating
3786 # is cheap -- and without it these results show the root
3787 # document's superseded content.
3788 all_docs = annotate_effective_content(
3789 Document.objects.filter(
3790 id__in=permitted_document_ids(request.user),
3791 ),
3792 )
3793 if db_only:
3794 docs = all_docs.filter(title__icontains=query)[:OBJECT_LIMIT]
3795 else:
3796 user = None if request.user.is_superuser else request.user
3797 matching_ids = get_backend().search_ids(
3798 query,
3799 user=user,
3800 search_mode=SearchMode.TEXT,
3801 limit=OBJECT_LIMIT * 3,
3802 )
3803 docs_by_id = all_docs.in_bulk(matching_ids)
3804 docs = [
3805 docs_by_id[doc_id]
3806 for doc_id in matching_ids
3807 if doc_id in docs_by_id
3808 ][:OBJECT_LIMIT]
3809 saved_views = (
3810 get_objects_for_user_owner_aware(
3811 request.user,
3812 "view_savedview",
3813 SavedView,
3814 ).filter(name__icontains=query)
3815 if request.user.has_perm("documents.view_savedview")
3816 else []
3817 )
3818 saved_views = saved_views[:OBJECT_LIMIT]
3819 tags = (
3820 get_objects_for_user_owner_aware(request.user, "view_tag", Tag).filter(
3821 name__icontains=query,
3822 )
3823 if request.user.has_perm("documents.view_tag")
3824 else []
3825 )
3826 tags = tags[:OBJECT_LIMIT]
3827 correspondents = (
3828 get_objects_for_user_owner_aware(
3829 request.user,
3830 "view_correspondent",
3831 Correspondent,
3832 ).filter(name__icontains=query)
3833 if request.user.has_perm("documents.view_correspondent")
3834 else []
3835 )
3836 correspondents = correspondents[:OBJECT_LIMIT]
3837 document_types = (
3838 get_objects_for_user_owner_aware(
3839 request.user,
3840 "view_documenttype",
3841 DocumentType,
3842 ).filter(name__icontains=query)
3843 if request.user.has_perm("documents.view_documenttype")
3844 else []
3845 )
3846 document_types = document_types[:OBJECT_LIMIT]
3847 storage_paths = (
3848 get_objects_for_user_owner_aware(
3849 request.user,
3850 "view_storagepath",
3851 StoragePath,
3852 ).filter(name__icontains=query)
3853 if request.user.has_perm("documents.view_storagepath")
3854 else []
3855 )
3856 storage_paths = storage_paths[:OBJECT_LIMIT]
3857 users = (
3858 User.objects.filter(username__icontains=query)
3859 if request.user.has_perm("auth.view_user")
3860 else []
3861 )
3862 users = users[:OBJECT_LIMIT]
3863 groups = (
3864 Group.objects.filter(name__icontains=query)
3865 if request.user.has_perm("auth.view_group")
3866 else []
3867 )
3868 groups = groups[:OBJECT_LIMIT]
3869 mail_rules = (
3870 get_objects_for_user_owner_aware(
3871 request.user,
3872 "view_mailrule",
3873 MailRule,
3874 ).filter(name__icontains=query)
3875 if request.user.has_perm("paperless_mail.view_mailrule")
3876 else []
3877 )
3878 mail_rules = mail_rules[:OBJECT_LIMIT]
3879 mail_accounts = (
3880 get_objects_for_user_owner_aware(
3881 request.user,
3882 "view_mailaccount",
3883 MailAccount,
3884 ).filter(name__icontains=query)
3885 if request.user.has_perm("paperless_mail.view_mailaccount")
3886 else []
3887 )
3888 mail_accounts = mail_accounts[:OBJECT_LIMIT]
3889 workflows = (
3890 Workflow.objects.filter(name__icontains=query)
3891 if request.user.has_perm("documents.view_workflow")
3892 else []
3893 )
3894 workflows = workflows[:OBJECT_LIMIT]
3895 custom_fields = (
3896 CustomField.objects.filter(name__icontains=query)
3897 if request.user.has_perm("documents.view_customfield")
3898 else []
3899 )
3900 custom_fields = custom_fields[:OBJECT_LIMIT]
3902 context = {
3903 "request": request,
3904 }
3906 docs_serializer = DocumentSerializer(docs, many=True, context=context)
3907 saved_views_serializer = SavedViewSerializer(
3908 saved_views,
3909 many=True,
3910 context=context,
3911 )
3912 tags_serializer = TagSerializer(tags, many=True, context=context)
3913 correspondents_serializer = CorrespondentSerializer(
3914 correspondents,
3915 many=True,
3916 context=context,
3917 )
3918 document_types_serializer = DocumentTypeSerializer(
3919 document_types,
3920 many=True,
3921 context=context,
3922 )
3923 storage_paths_serializer = StoragePathSerializer(
3924 storage_paths,
3925 many=True,
3926 context=context,
3927 )
3928 users_serializer = UserSerializer(users, many=True, context=context)
3929 groups_serializer = GroupSerializer(groups, many=True, context=context)
3930 mail_rules_serializer = MailRuleSerializer(
3931 mail_rules,
3932 many=True,
3933 context=context,
3934 )
3935 mail_accounts_serializer = MailAccountSerializer(
3936 mail_accounts,
3937 many=True,
3938 context=context,
3939 )
3940 workflows_serializer = WorkflowSerializer(workflows, many=True, context=context)
3941 custom_fields_serializer = CustomFieldSerializer(
3942 custom_fields,
3943 many=True,
3944 context=context,
3945 )
3947 return Response(
3948 {
3949 "total": len(docs)
3950 + len(saved_views)
3951 + len(tags)
3952 + len(correspondents)
3953 + len(document_types)
3954 + len(storage_paths)
3955 + len(users)
3956 + len(groups)
3957 + len(mail_rules)
3958 + len(mail_accounts)
3959 + len(workflows)
3960 + len(custom_fields),
3961 "documents": docs_serializer.data,
3962 "saved_views": saved_views_serializer.data,
3963 "tags": tags_serializer.data,
3964 "correspondents": correspondents_serializer.data,
3965 "document_types": document_types_serializer.data,
3966 "storage_paths": storage_paths_serializer.data,
3967 "users": users_serializer.data,
3968 "groups": groups_serializer.data,
3969 "mail_rules": mail_rules_serializer.data,
3970 "mail_accounts": mail_accounts_serializer.data,
3971 "workflows": workflows_serializer.data,
3972 "custom_fields": custom_fields_serializer.data,
3973 },
3974 )
3977@extend_schema_view(
3978 get=extend_schema(
3979 description="Get statistics for the current user",
3980 responses={
3981 (200, "application/json"): OpenApiTypes.OBJECT,
3982 },
3983 ),
3984)
3985class StatisticsView(GenericAPIView[Any]):
3986 permission_classes = (IsAuthenticated,)
3988 def get(self, request, format=None):
3989 user = request.user if request.user is not None else None
3990 can_view_global_stats = has_global_statistics_permission(user) or user is None
3992 documents = (
3993 Document.objects.all()
3994 if can_view_global_stats
3995 else Document.objects.filter(id__in=permitted_document_ids(user))
3996 ).filter(root_document__isnull=True)
3997 tags = (
3998 Tag.objects.all()
3999 if can_view_global_stats
4000 else get_objects_for_user_owner_aware(user, "documents.view_tag", Tag)
4001 ).only("id", "is_inbox_tag")
4002 correspondent_count = (
4003 Correspondent.objects.count()
4004 if can_view_global_stats
4005 else get_objects_for_user_owner_aware(
4006 user,
4007 "documents.view_correspondent",
4008 Correspondent,
4009 ).count()
4010 )
4011 document_type_count = (
4012 DocumentType.objects.count()
4013 if can_view_global_stats
4014 else get_objects_for_user_owner_aware(
4015 user,
4016 "documents.view_documenttype",
4017 DocumentType,
4018 ).count()
4019 )
4020 storage_path_count = (
4021 StoragePath.objects.count()
4022 if can_view_global_stats
4023 else get_objects_for_user_owner_aware(
4024 user,
4025 "documents.view_storagepath",
4026 StoragePath,
4027 ).count()
4028 )
4030 inbox_tag_pks = list(
4031 tags.filter(is_inbox_tag=True).values_list("pk", flat=True),
4032 )
4034 documents_inbox = (
4035 documents.filter(tags__id__in=inbox_tag_pks).values("id").distinct().count()
4036 if inbox_tag_pks
4037 else None
4038 )
4040 # Single SQL request for document stats and mime type counts
4041 mime_type_stats = list(
4042 documents.values("mime_type")
4043 .annotate(
4044 mime_type_count=Count("id"),
4045 mime_type_chars=Sum("content_length"),
4046 )
4047 .order_by("-mime_type_count"),
4048 )
4050 # Calculate totals from grouped results
4051 documents_total = sum(row["mime_type_count"] for row in mime_type_stats)
4052 character_count = sum(row["mime_type_chars"] or 0 for row in mime_type_stats)
4053 document_file_type_counts = [
4054 {"mime_type": row["mime_type"], "mime_type_count": row["mime_type_count"]}
4055 for row in mime_type_stats
4056 ]
4058 current_asn = Document.objects.aggregate(
4059 Max("archive_serial_number", default=0),
4060 ).get(
4061 "archive_serial_number__max",
4062 )
4064 return Response(
4065 {
4066 "documents_total": documents_total,
4067 "documents_inbox": documents_inbox,
4068 "inbox_tag": (
4069 inbox_tag_pks[0] if inbox_tag_pks else None
4070 ), # backwards compatibility
4071 "inbox_tags": (inbox_tag_pks or None),
4072 "document_file_type_counts": document_file_type_counts,
4073 "character_count": character_count,
4074 "tag_count": len(tags),
4075 "correspondent_count": correspondent_count,
4076 "document_type_count": document_type_count,
4077 "storage_path_count": storage_path_count,
4078 "current_asn": current_asn,
4079 },
4080 )
4083@extend_schema_view(
4084 post=extend_schema(
4085 operation_id="bulk_download",
4086 description="Download multiple documents as a ZIP archive.",
4087 responses={
4088 (HTTPStatus.OK, "application/zip"): OpenApiTypes.BINARY,
4089 HTTPStatus.FORBIDDEN: None,
4090 },
4091 ),
4092)
4093class BulkDownloadView(DocumentSelectionMixin, GenericAPIView[Any]):
4094 permission_classes = (IsAuthenticated, ViewDocumentsPermissions)
4095 serializer_class = BulkDownloadSerializer
4096 parser_classes = (parsers.JSONParser,)
4098 def post(self, request, format=None):
4099 serializer = self.get_serializer(data=request.data)
4100 serializer.is_valid(raise_exception=True)
4102 ids = self._resolve_document_ids(
4103 user=request.user,
4104 validated_data=serializer.validated_data,
4105 )
4106 documents = Document.objects.filter(pk__in=ids)
4107 versioned_documents = []
4108 compression = serializer.validated_data.get("compression")
4109 content = serializer.validated_data.get("content")
4110 follow_filename_format = serializer.validated_data.get("follow_formatting")
4112 permitted_ids = set(permitted_document_ids(request.user))
4113 for document in documents:
4114 root_doc = get_root_document(document)
4115 if root_doc.pk not in permitted_ids: 4115 ↛ 4116line 4115 didn't jump to line 4116 because the condition on line 4115 was never true
4116 return HttpResponseForbidden("Insufficient permissions")
4117 versioned_documents.append(
4118 get_latest_version_for_root(
4119 root_doc,
4120 ),
4121 )
4123 if content == "both":
4124 strategy_class = OriginalAndArchiveStrategy
4125 elif content == "originals":
4126 strategy_class = OriginalsOnlyStrategy
4127 else:
4128 strategy_class = ArchiveOnlyStrategy
4130 settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
4131 fd, temp_name = tempfile.mkstemp(
4132 dir=settings.SCRATCH_DIR,
4133 suffix="-compressed-archive",
4134 )
4135 os.close(fd)
4136 temp_path = Path(temp_name)
4138 try:
4139 with zipfile.ZipFile(temp_path, "w", compression) as zipf:
4140 strategy = strategy_class(
4141 zipf,
4142 follow_formatting=follow_filename_format,
4143 )
4144 for document in versioned_documents:
4145 strategy.add_document(document)
4147 f = temp_path.open("rb")
4148 temp_path.unlink()
4149 except Exception:
4150 temp_path.unlink(missing_ok=True)
4151 raise
4153 return FileResponse(
4154 f,
4155 as_attachment=True,
4156 filename="documents.zip",
4157 content_type="application/zip",
4158 )
4161@extend_schema_view(
4162 **generate_object_with_permissions_schema(StoragePathSerializer),
4163 test=extend_schema(
4164 operation_id="storage_paths_test",
4165 description="Test a storage path template against a document.",
4166 request=StoragePathTestSerializer,
4167 responses={
4168 (HTTPStatus.OK, "application/json"): OpenApiTypes.STR,
4169 },
4170 ),
4171)
4172class StoragePathViewSet(PermissionsAwareDocumentCountMixin, ModelViewSet[StoragePath]):
4173 model = StoragePath
4175 queryset = StoragePath.objects.select_related("owner").order_by(
4176 Lower("name"),
4177 )
4179 serializer_class = StoragePathSerializer
4180 pagination_class = StandardPagination
4181 permission_classes = (IsAuthenticated, PaperlessObjectPermissions)
4182 filter_backends = (
4183 DjangoFilterBackend,
4184 OrderingFilter,
4185 PermittedObjectsFilter,
4186 )
4187 filterset_class = StoragePathFilterSet
4188 ordering_fields = ("name", "path", "matching_algorithm", "match", "document_count")
4190 def get_permissions(self):
4191 if self.action == "test":
4192 # Test action does not require object level permissions
4193 self.permission_classes = (IsAuthenticated, ViewDocumentsPermissions)
4194 return super().get_permissions()
4196 def destroy(self, request, *args, **kwargs):
4197 """
4198 When a storage path is deleted, see if documents
4199 using it require a rename/move
4200 """
4201 instance = self.get_object()
4202 doc_ids = [doc.id for doc in instance.documents.all()]
4204 # perform the deletion so renaming/moving can happen
4205 response = super().destroy(request, *args, **kwargs)
4207 if doc_ids: 4207 ↛ 4208line 4207 didn't jump to line 4208 because the condition on line 4207 was never true
4208 bulk_edit.bulk_update_documents.apply_async(
4209 kwargs={"document_ids": doc_ids},
4210 headers={"trigger_source": PaperlessTask.TriggerSource.SYSTEM},
4211 )
4213 return response
4215 @action(methods=["post"], detail=False)
4216 def test(self, request):
4217 """
4218 Test storage path against a document
4219 """
4220 serializer = StoragePathTestSerializer(
4221 data=request.data,
4222 context={"request": request},
4223 )
4224 serializer.is_valid(raise_exception=True)
4226 document = serializer.validated_data.get("document")
4227 path = serializer.validated_data.get("path")
4229 result = format_filename(document, path)
4230 if result:
4231 extension = (
4232 Path(str(document.filename)).suffix if document.filename else ""
4233 ) or document.file_type
4234 result_path = Path(result)
4235 result = str(result_path.with_name(f"{result_path.name}{extension}"))
4236 return Response(result)
4239class UiSettingsView(GenericAPIView[Any]):
4240 queryset = UiSettings.objects.all()
4241 permission_classes = (IsAuthenticated, PaperlessObjectPermissions)
4242 serializer_class = UiSettingsViewSerializer
4244 def get(self, request, format=None):
4245 serializer = self.get_serializer(data=request.data)
4246 serializer.is_valid(raise_exception=True)
4248 user = User.objects.select_related("ui_settings").get(pk=request.user.id)
4249 ui_settings = {}
4250 if hasattr(user, "ui_settings"): 4250 ↛ 4252line 4250 didn't jump to line 4252 because the condition on line 4250 was always true
4251 ui_settings = user.ui_settings.settings
4252 if "update_checking" in ui_settings: 4252 ↛ 4253line 4252 didn't jump to line 4253 because the condition on line 4252 was never true
4253 ui_settings["update_checking"]["backend_setting"] = (
4254 settings.ENABLE_UPDATE_CHECK
4255 )
4256 else:
4257 ui_settings["update_checking"] = {
4258 "backend_setting": settings.ENABLE_UPDATE_CHECK,
4259 }
4261 ui_settings["trash_delay"] = settings.EMPTY_TRASH_DELAY
4263 general_config = GeneralConfig()
4265 ui_settings["version"] = version.__full_version_str__
4267 ui_settings["app_title"] = settings.APP_TITLE
4268 if general_config.app_title is not None and len(general_config.app_title) > 0: 4268 ↛ 4269line 4268 didn't jump to line 4269 because the condition on line 4268 was never true
4269 ui_settings["app_title"] = general_config.app_title
4270 ui_settings["app_logo"] = settings.APP_LOGO
4271 if general_config.app_logo is not None and len(general_config.app_logo) > 0: 4271 ↛ 4272line 4271 didn't jump to line 4272 because the condition on line 4271 was never true
4272 ui_settings["app_logo"] = general_config.app_logo
4274 ui_settings["auditlog_enabled"] = settings.AUDIT_LOG_ENABLED
4276 ui_settings["remote_ocr"] = {
4277 "configured": RemoteEngineConfig.from_app_config().engine_is_valid(),
4278 "mode": RemoteOCRConfig().remote_ocr_mode,
4279 }
4281 if settings.GMAIL_OAUTH_ENABLED or settings.OUTLOOK_OAUTH_ENABLED: 4281 ↛ 4282line 4281 didn't jump to line 4282 because the condition on line 4281 was never true
4282 manager = PaperlessMailOAuth2Manager()
4283 if settings.GMAIL_OAUTH_ENABLED:
4284 ui_settings["gmail_oauth_url"] = manager.get_gmail_authorization_url()
4285 request.session["oauth_state"] = manager.state
4286 if settings.OUTLOOK_OAUTH_ENABLED:
4287 ui_settings["outlook_oauth_url"] = (
4288 manager.get_outlook_authorization_url()
4289 )
4290 request.session["oauth_state"] = manager.state
4292 ui_settings["email_enabled"] = settings.EMAIL_ENABLED
4294 ai_config = AIConfig()
4296 ui_settings["ai_enabled"] = ai_config.ai_enabled
4298 user_resp = {
4299 "id": user.id,
4300 "username": user.username,
4301 "is_staff": user.is_staff,
4302 "is_superuser": user.is_superuser,
4303 "groups": list(user.groups.values_list("id", flat=True)),
4304 }
4306 if len(user.first_name) > 0:
4307 user_resp["first_name"] = user.first_name
4308 if len(user.last_name) > 0:
4309 user_resp["last_name"] = user.last_name
4311 # strip <app_label>.
4312 roles = map(lambda perm: re.sub(r"^\w+.", "", perm), user.get_all_permissions())
4313 return Response(
4314 {
4315 "user": user_resp,
4316 "settings": ui_settings,
4317 "permissions": roles,
4318 },
4319 )
4321 def post(self, request, format=None):
4322 serializer = self.get_serializer(data=request.data)
4323 serializer.is_valid(raise_exception=True)
4325 serializer.save(user=self.request.user)
4327 return Response(
4328 {
4329 "success": True,
4330 },
4331 )
4334@extend_schema_view(
4335 get=extend_schema(
4336 description="Get the current version of the Paperless-NGX server",
4337 responses={
4338 (200, "application/json"): OpenApiTypes.OBJECT,
4339 },
4340 ),
4341)
4342class RemoteVersionView(GenericAPIView[Any]):
4343 cache_key = "remote_version_view_latest_release"
4345 def get(self, request, format=None):
4346 current_version = packaging_version.parse(version.__full_version_str__)
4347 remote_version = cache.get(self.cache_key)
4348 if remote_version is None:
4349 try:
4350 resp = httpx.get(
4351 "https://api.github.com/repos/paperless-ngx/paperless-ngx/releases/latest",
4352 headers={"Accept": "application/json"},
4353 )
4354 resp.raise_for_status()
4355 data = resp.json()
4356 remote_version = data["tag_name"]
4357 # Some early tags used ngx-x.y.z
4358 remote_version = remote_version.removeprefix("ngx-")
4359 except ValueError as e:
4360 logger.debug(f"An error occurred parsing remote version json: {e}")
4361 except httpx.HTTPError as e:
4362 logger.debug(f"An error occurred checking for available updates: {e}")
4364 if remote_version: 4364 ↛ 4367line 4364 didn't jump to line 4367 because the condition on line 4364 was always true
4365 cache.set(self.cache_key, remote_version, 60 * 15)
4366 else:
4367 remote_version = "0.0.0"
4369 is_greater_than_current = (
4370 packaging_version.parse(remote_version) > current_version
4371 )
4373 return Response(
4374 {
4375 "version": remote_version,
4376 "update_available": is_greater_than_current,
4377 },
4378 )
4381class _TasksViewSetSchema(AutoSchema):
4382 _UNPAGINATED_ACTIONS = frozenset({"summary", "active", "status_counts"})
4384 def _get_paginator(self):
4385 if getattr(self.view, "action", None) in self._UNPAGINATED_ACTIONS:
4386 return None
4387 return super()._get_paginator()
4390@extend_schema_view(
4391 list=extend_schema(
4392 parameters=[
4393 OpenApiParameter(
4394 name="task_id",
4395 type=str,
4396 location=OpenApiParameter.QUERY,
4397 required=False,
4398 description="Filter tasks by Celery UUID",
4399 ),
4400 ],
4401 ),
4402 acknowledge=extend_schema(
4403 operation_id="acknowledge_tasks",
4404 description="Acknowledge a list of tasks, or all visible unacknowledged tasks",
4405 request=AcknowledgeTasksViewSerializer,
4406 responses={
4407 (200, "application/json"): inline_serializer(
4408 name="AcknowledgeTasks",
4409 fields={
4410 "result": serializers.IntegerField(),
4411 },
4412 ),
4413 },
4414 ),
4415 run=extend_schema(
4416 operation_id="run_task",
4417 description="Manually dispatch a background task. Superuser only.",
4418 request=RunTaskSerializer,
4419 responses={
4420 (200, "application/json"): inline_serializer(
4421 name="RunTask",
4422 fields={"task_id": serializers.CharField()},
4423 ),
4424 (400, "application/json"): inline_serializer(
4425 name="RunTaskError",
4426 fields={"error": serializers.CharField()},
4427 ),
4428 },
4429 ),
4430 summary=extend_schema(
4431 responses={200: TaskSummarySerializer(many=True)},
4432 parameters=[
4433 OpenApiParameter(
4434 name="days",
4435 type={"type": "integer", "minimum": 1, "maximum": 365, "default": 30},
4436 location=OpenApiParameter.QUERY,
4437 required=False,
4438 description="Number of days to include in aggregation (default 30, min 1, max 365)",
4439 ),
4440 ],
4441 ),
4442 status_counts=extend_schema(
4443 responses={
4444 200: inline_serializer(
4445 name="TaskStatusCounts",
4446 fields={
4447 "all": serializers.IntegerField(),
4448 "needs_attention": serializers.IntegerField(),
4449 "in_progress": serializers.IntegerField(),
4450 "completed": serializers.IntegerField(),
4451 },
4452 ),
4453 },
4454 ),
4455 active=extend_schema(
4456 description="Currently pending and running tasks (capped at 50).",
4457 responses={200: TaskSerializerV10(many=True)},
4458 ),
4459)
4460class TasksViewSet(ReadOnlyModelViewSet[PaperlessTask]):
4461 schema = _TasksViewSetSchema()
4462 permission_classes = (IsAuthenticated, PaperlessObjectPermissions)
4463 pagination_class = StandardPagination
4464 filter_backends = (
4465 DjangoFilterBackend,
4466 OrderingFilter,
4467 )
4468 filterset_class = PaperlessTaskFilterSet
4469 ordering_fields = [
4470 "date_created",
4471 "date_done",
4472 "status",
4473 "task_type",
4474 "duration_seconds",
4475 "wait_time_seconds",
4476 ]
4477 ordering = ["-date_created"]
4478 # Needed for drf-spectacular schema generation (get_queryset touches request.user)
4479 queryset = PaperlessTask.objects.none()
4481 # v9 backwards compat: maps old task_name values to new task_type values
4482 _V9_TASK_NAME_TO_TYPE = {
4483 "check_sanity": PaperlessTask.TaskType.SANITY_CHECK,
4484 "llmindex_update": PaperlessTask.TaskType.LLM_INDEX,
4485 }
4487 # v9 backwards compat: maps old "type" query param values to new TriggerSource.
4488 # Must match the reverse of TaskSerializerV9._TRIGGER_SOURCE_TO_V9_TYPE.
4489 _V9_TYPE_TO_TRIGGER_SOURCES = {
4490 "auto_task": [
4491 PaperlessTask.TriggerSource.SYSTEM,
4492 PaperlessTask.TriggerSource.EMAIL_CONSUME,
4493 PaperlessTask.TriggerSource.FOLDER_CONSUME,
4494 ],
4495 "scheduled_task": [PaperlessTask.TriggerSource.SCHEDULED],
4496 "manual_task": [
4497 PaperlessTask.TriggerSource.MANUAL,
4498 PaperlessTask.TriggerSource.WEB_UI,
4499 PaperlessTask.TriggerSource.API_UPLOAD,
4500 ],
4501 }
4503 _RUNNABLE_TASKS = {
4504 PaperlessTask.TaskType.TRAIN_CLASSIFIER: (train_classifier, {}),
4505 PaperlessTask.TaskType.SANITY_CHECK: (sanity_check, {"raise_on_error": False}),
4506 PaperlessTask.TaskType.LLM_INDEX: (llmindex_index, {"rebuild": False}),
4507 }
4508 _STATUS_COUNT_EXCLUDED_FILTERS = frozenset({"status", "is_complete"})
4510 def get_serializer_class(self):
4511 # v9: use backwards-compatible serializer with old field names
4512 if self.request.version and int(self.request.version) < 10: 4512 ↛ 4513line 4512 didn't jump to line 4513 because the condition on line 4512 was never true
4513 return TaskSerializerV9
4514 return TaskSerializerV10
4516 def paginate_queryset(self, queryset):
4517 # v9: tasks endpoint was not paginated; preserve plain-list response
4518 if self.request.version and int(self.request.version) < 10: 4518 ↛ 4519line 4518 didn't jump to line 4519 because the condition on line 4518 was never true
4519 return None
4520 return super().paginate_queryset(queryset)
4522 def get_queryset(self):
4523 is_v9 = self.request.version and int(self.request.version) < 10
4524 if self.request.user.is_staff: 4524 ↛ 4529line 4524 didn't jump to line 4529 because the condition on line 4524 was always true
4525 queryset = PaperlessTask.objects.all()
4526 else:
4527 # Own tasks + unowned (system/scheduled) tasks. Tasks owned by other
4528 # users are never visible to non-staff regardless of API version.
4529 queryset = PaperlessTask.objects.filter(
4530 Q(owner=self.request.user) | Q(owner__isnull=True),
4531 )
4532 # v9 backwards compat: map old query params to new field names
4533 if is_v9: 4533 ↛ 4534line 4533 didn't jump to line 4534 because the condition on line 4533 was never true
4534 task_name = self.request.query_params.get("task_name")
4535 if task_name is not None:
4536 mapped = self._V9_TASK_NAME_TO_TYPE.get(task_name, task_name)
4537 queryset = queryset.filter(task_type=mapped)
4538 task_type_old = self.request.query_params.get("type")
4539 if task_type_old is not None:
4540 sources = self._V9_TYPE_TO_TRIGGER_SOURCES.get(task_type_old)
4541 if sources:
4542 queryset = queryset.filter(trigger_source__in=sources)
4543 # v10+: direct task_id param for backwards compat
4544 task_id = self.request.query_params.get("task_id")
4545 if task_id is not None:
4546 queryset = queryset.filter(task_id=task_id)
4547 return queryset
4549 def get_status_count_queryset(self):
4550 """Apply task filters except the status dimensions represented by the counts."""
4551 query_params = self.request.query_params.copy()
4552 for param in self._STATUS_COUNT_EXCLUDED_FILTERS:
4553 query_params.pop(param, None)
4555 filterset = self.filterset_class(
4556 data=query_params,
4557 queryset=self.get_queryset(),
4558 request=self.request,
4559 )
4560 if not filterset.is_valid(): 4560 ↛ 4561line 4560 didn't jump to line 4561 because the condition on line 4560 was never true
4561 raise ValidationError(filterset.errors)
4562 return filterset.qs
4564 @action(
4565 methods=["post"],
4566 detail=False,
4567 permission_classes=[IsAuthenticated, AcknowledgeTasksPermissions],
4568 )
4569 def acknowledge(self, request):
4570 queryset = self.get_queryset()
4571 serializer = AcknowledgeTasksViewSerializer(
4572 data=request.data,
4573 context={"queryset": queryset},
4574 )
4575 serializer.is_valid(raise_exception=True)
4576 if serializer.validated_data.get("all", False):
4577 tasks = queryset.filter(acknowledged=False)
4578 else:
4579 task_ids = serializer.validated_data.get("tasks")
4580 tasks = queryset.filter(id__in=task_ids)
4581 count = tasks.update(acknowledged=True)
4582 return Response({"result": count})
4584 def get_permissions(self):
4585 if self.action == "summary" and has_system_status_permission(
4586 getattr(self.request, "user", None),
4587 ):
4588 return [IsAuthenticated()]
4589 return super().get_permissions()
4591 @action(methods=["get"], detail=False)
4592 def summary(self, request):
4593 """Aggregated task statistics per task_type over the last N days (default 30)."""
4594 try:
4595 days = min(365, max(1, int(request.query_params.get("days", 30))))
4596 except (TypeError, ValueError):
4597 return Response(
4598 {"days": "Must be a positive integer."},
4599 status=status.HTTP_400_BAD_REQUEST,
4600 )
4601 cutoff = timezone.now() - timedelta(days=days)
4602 if has_system_status_permission(request.user): 4602 ↛ 4605line 4602 didn't jump to line 4605 because the condition on line 4602 was always true
4603 queryset = PaperlessTask.objects.filter(date_created__gte=cutoff)
4604 else:
4605 queryset = self.get_queryset().filter(date_created__gte=cutoff)
4607 data = queryset.values("task_type").annotate(
4608 total_count=Count("id"),
4609 pending_count=Count("id", filter=Q(status=PaperlessTask.Status.PENDING)),
4610 success_count=Count("id", filter=Q(status=PaperlessTask.Status.SUCCESS)),
4611 failure_count=Count("id", filter=Q(status=PaperlessTask.Status.FAILURE)),
4612 avg_duration_seconds=Avg(
4613 "duration_seconds",
4614 filter=Q(duration_seconds__isnull=False),
4615 ),
4616 avg_wait_time_seconds=Avg(
4617 "wait_time_seconds",
4618 filter=Q(wait_time_seconds__isnull=False),
4619 ),
4620 last_run=Max("date_created"),
4621 last_success=Max(
4622 "date_done",
4623 filter=Q(status=PaperlessTask.Status.SUCCESS),
4624 ),
4625 last_failure=Max(
4626 "date_done",
4627 filter=Q(status=PaperlessTask.Status.FAILURE),
4628 ),
4629 )
4630 serializer = TaskSummarySerializer(data, many=True)
4631 return Response(serializer.data)
4633 @action(methods=["get"], detail=False)
4634 def status_counts(self, request):
4635 """Aggregated task counts for task UI sections."""
4636 queryset = self.get_status_count_queryset()
4637 counts = queryset.aggregate(
4638 all=Count("id"),
4639 needs_attention=Count(
4640 "id",
4641 filter=Q(
4642 status__in=[
4643 PaperlessTask.Status.FAILURE,
4644 PaperlessTask.Status.REVOKED,
4645 ],
4646 ),
4647 ),
4648 in_progress=Count(
4649 "id",
4650 filter=Q(
4651 status__in=[
4652 PaperlessTask.Status.PENDING,
4653 PaperlessTask.Status.STARTED,
4654 ],
4655 ),
4656 ),
4657 completed=Count("id", filter=Q(status=PaperlessTask.Status.SUCCESS)),
4658 )
4659 return Response(counts)
4661 @action(methods=["get"], detail=False)
4662 def active(self, request):
4663 """Currently pending and running tasks (capped at 50)."""
4664 queryset = (
4665 self.get_queryset()
4666 .filter(
4667 status__in=[PaperlessTask.Status.PENDING, PaperlessTask.Status.STARTED],
4668 )
4669 .order_by("-date_created")[:50]
4670 )
4671 serializer = self.get_serializer(queryset, many=True)
4672 return Response(serializer.data)
4674 @action(methods=["post"], detail=False)
4675 def run(self, request):
4676 """Manually dispatch a background task. Superuser only."""
4677 if not request.user.is_superuser: 4677 ↛ 4678line 4677 didn't jump to line 4678 because the condition on line 4677 was never true
4678 return HttpResponseForbidden("Insufficient permissions")
4679 serializer = RunTaskSerializer(data=request.data)
4680 serializer.is_valid(raise_exception=True)
4681 task_type = serializer.validated_data.get("task_type")
4683 if task_type not in self._RUNNABLE_TASKS:
4684 return Response(
4685 {"error": f"Task type '{task_type}' cannot be manually triggered"},
4686 status=status.HTTP_400_BAD_REQUEST,
4687 )
4689 try:
4690 task_func, task_kwargs = self._RUNNABLE_TASKS[task_type]
4691 async_result = task_func.apply_async(
4692 kwargs=task_kwargs,
4693 headers={"trigger_source": PaperlessTask.TriggerSource.MANUAL},
4694 )
4695 return Response({"task_id": async_result.id})
4696 except Exception as e:
4697 logger.warning(f"Error running task: {e!s}")
4698 return HttpResponseServerError(
4699 "Error running task, check logs for more detail.",
4700 )
4703class ShareLinkViewSet(
4704 PassUserMixin,
4705 CreateModelMixin,
4706 RetrieveModelMixin,
4707 DestroyModelMixin,
4708 ListModelMixin,
4709 GenericViewSet,
4710):
4711 model = ShareLink
4713 queryset = ShareLink.objects.select_related("document")
4715 serializer_class = ShareLinkSerializer
4716 pagination_class = StandardPagination
4717 permission_classes = (IsAuthenticated, PaperlessObjectPermissions)
4718 filter_backends = (
4719 DjangoFilterBackend,
4720 OrderingFilter,
4721 PermittedObjectsFilter,
4722 )
4723 filterset_class = ShareLinkFilterSet
4724 ordering_fields = ("created", "expiration", "document__title")
4727@extend_schema_view(
4728 rebuild=extend_schema(
4729 operation_id="share_link_bundles_rebuild",
4730 description="Reset and re-queue a share link bundle for processing.",
4731 responses={
4732 HTTPStatus.OK: ShareLinkBundleSerializer,
4733 (HTTPStatus.BAD_REQUEST, "application/json"): inline_serializer(
4734 name="RebuildBundleError",
4735 fields={"detail": serializers.CharField()},
4736 ),
4737 },
4738 ),
4739)
4740class ShareLinkBundleViewSet(PassUserMixin, ModelViewSet[ShareLinkBundle]):
4741 model = ShareLinkBundle
4743 # Bundles are immutable once created; rebuild via the dedicated action
4744 # rather than PUT/PATCH.
4745 http_method_names = ["get", "post", "delete", "head", "options"]
4747 queryset = ShareLinkBundle.objects.all()
4749 serializer_class = ShareLinkBundleSerializer
4750 pagination_class = StandardPagination
4751 permission_classes = (IsAuthenticated, PaperlessObjectPermissions)
4752 filter_backends = (
4753 DjangoFilterBackend,
4754 OrderingFilter,
4755 PermittedObjectsFilter,
4756 )
4757 filterset_class = ShareLinkBundleFilterSet
4758 ordering_fields = ("created", "expiration", "status")
4760 def get_permissions(self):
4761 permissions = super().get_permissions()
4762 if self.action == "create":
4763 permissions.append(ViewDocumentsPermissions())
4764 return permissions
4766 def get_queryset(self):
4767 return (
4768 super()
4769 .get_queryset()
4770 .prefetch_related("documents")
4771 .annotate(document_total=Count("documents", distinct=True))
4772 )
4774 def create(self, request, *args, **kwargs):
4775 serializer = self.get_serializer(data=request.data)
4776 serializer.is_valid(raise_exception=True)
4777 document_ids = serializer.validated_data["document_ids"]
4778 documents_qs = Document.objects.filter(pk__in=document_ids).select_related(
4779 "owner",
4780 )
4781 found_ids = set(documents_qs.values_list("pk", flat=True))
4782 missing = sorted(set(document_ids) - found_ids)
4783 if missing: 4783 ↛ 4793line 4783 didn't jump to line 4793 because the condition on line 4783 was always true
4784 raise ValidationError(
4785 {
4786 "document_ids": _(
4787 "Documents not found: %(ids)s",
4788 )
4789 % {"ids": ", ".join(str(item) for item in missing)},
4790 },
4791 )
4793 documents = list(documents_qs)
4794 permitted_ids = set(permitted_document_ids(request.user))
4795 for document in documents:
4796 if document.pk not in permitted_ids:
4797 raise ValidationError(
4798 {
4799 "document_ids": _(
4800 "Insufficient permissions to share document %(id)s.",
4801 )
4802 % {"id": document.pk},
4803 },
4804 )
4806 document_map = {document.pk: document for document in documents}
4807 ordered_documents = [document_map[doc_id] for doc_id in document_ids]
4809 bundle = serializer.save(
4810 owner=request.user,
4811 documents=ordered_documents,
4812 )
4813 bundle.remove_file()
4814 bundle.status = ShareLinkBundle.Status.PENDING
4815 bundle.last_error = None
4816 bundle.size_bytes = None
4817 bundle.built_at = None
4818 bundle.file_path = ""
4819 bundle.save(
4820 update_fields=[
4821 "status",
4822 "last_error",
4823 "size_bytes",
4824 "built_at",
4825 "file_path",
4826 ],
4827 )
4828 build_share_link_bundle.apply_async(
4829 kwargs={"bundle_id": bundle.pk},
4830 headers={"trigger_source": PaperlessTask.TriggerSource.MANUAL},
4831 )
4832 bundle.document_total = len(ordered_documents)
4833 response_serializer = self.get_serializer(bundle)
4834 headers = self.get_success_headers(response_serializer.data)
4835 return Response(
4836 response_serializer.data,
4837 status=status.HTTP_201_CREATED,
4838 headers=headers,
4839 )
4841 @action(detail=True, methods=["post"])
4842 def rebuild(self, request, pk=None):
4843 bundle = self.get_object()
4844 if bundle.status == ShareLinkBundle.Status.PROCESSING:
4845 return Response(
4846 {"detail": _("Bundle is already being processed.")},
4847 status=status.HTTP_400_BAD_REQUEST,
4848 )
4849 bundle.remove_file()
4850 bundle.status = ShareLinkBundle.Status.PENDING
4851 bundle.last_error = None
4852 bundle.size_bytes = None
4853 bundle.built_at = None
4854 bundle.file_path = ""
4855 bundle.save(
4856 update_fields=[
4857 "status",
4858 "last_error",
4859 "size_bytes",
4860 "built_at",
4861 "file_path",
4862 ],
4863 )
4864 build_share_link_bundle.apply_async(
4865 kwargs={"bundle_id": bundle.pk},
4866 headers={"trigger_source": PaperlessTask.TriggerSource.MANUAL},
4867 )
4868 bundle.document_total = (
4869 getattr(bundle, "document_total", None) or bundle.documents.count()
4870 )
4871 serializer = self.get_serializer(bundle)
4872 return Response(serializer.data)
4875class SharedLinkView(View):
4876 authentication_classes = []
4877 permission_classes = []
4879 def get(self, request, slug):
4880 share_link = ShareLink.objects.filter(slug=slug).first()
4881 if share_link is not None:
4882 if (
4883 share_link.expiration is not None
4884 and share_link.expiration < timezone.now()
4885 ):
4886 return HttpResponseRedirect("/accounts/login/?sharelink_expired=1")
4887 try:
4888 return serve_file(
4889 doc=share_link.document,
4890 use_archive=share_link.file_version == ShareLink.FileVersion.ARCHIVE
4891 and share_link.document.has_archive_version,
4892 disposition="inline",
4893 )
4894 except FileNotFoundError:
4895 return HttpResponseRedirect("/accounts/login/?sharelink_notfound=1")
4897 bundle = ShareLinkBundle.objects.filter(slug=slug).first()
4898 if bundle is None:
4899 return HttpResponseRedirect("/accounts/login/?sharelink_notfound=1")
4901 if bundle.expiration is not None and bundle.expiration < timezone.now():
4902 return HttpResponseRedirect("/accounts/login/?sharelink_expired=1")
4904 if bundle.status in {
4905 ShareLinkBundle.Status.PENDING,
4906 ShareLinkBundle.Status.PROCESSING,
4907 }:
4908 return HttpResponse(
4909 _(
4910 "The share link bundle is still being prepared. Please try again later.",
4911 ),
4912 status=status.HTTP_202_ACCEPTED,
4913 )
4915 file_path = bundle.absolute_file_path
4917 if (
4918 bundle.status == ShareLinkBundle.Status.FAILED
4919 or file_path is None
4920 or not file_path.exists()
4921 ):
4922 return HttpResponse(
4923 _(
4924 "The share link bundle is unavailable.",
4925 ),
4926 status=status.HTTP_503_SERVICE_UNAVAILABLE,
4927 )
4929 response = FileResponse(file_path.open("rb"), content_type="application/zip")
4930 short_slug = bundle.slug[:12]
4931 download_name = f"paperless-share-{short_slug}.zip"
4932 filename_normalized = (
4933 normalize("NFKD", download_name)
4934 .encode(
4935 "ascii",
4936 "ignore",
4937 )
4938 .decode("ascii")
4939 )
4940 filename_encoded = quote(download_name)
4941 response["Content-Disposition"] = (
4942 f"attachment; filename='{filename_normalized}'; "
4943 f"filename*=utf-8''{filename_encoded}"
4944 )
4945 return response
4948def serve_file(
4949 *,
4950 doc: Document,
4951 use_archive: bool,
4952 disposition: str,
4953 follow_formatting: bool = False,
4954) -> FileResponse:
4955 if use_archive:
4956 if TYPE_CHECKING:
4957 assert doc.archive_filename
4959 file_handle = doc.archive_file
4960 filename = (
4961 doc.archive_filename
4962 if follow_formatting
4963 else doc.get_public_filename(archive=True)
4964 )
4965 mime_type = "application/pdf"
4966 else:
4967 if TYPE_CHECKING:
4968 assert doc.filename
4970 file_handle = doc.source_file
4971 filename = doc.filename if follow_formatting else doc.get_public_filename()
4972 mime_type = doc.mime_type
4973 # Support browser previewing csv files by using text mime type
4974 if mime_type in {"application/csv", "text/csv"} and disposition == "inline":
4975 mime_type = "text/plain"
4976 # Tell browsers to use UTF-8 for the text files we parse as UTF-8
4977 if mime_type in {"text/plain", "text/csv", "application/csv"}:
4978 mime_type = f"{mime_type}; charset=utf-8"
4980 response = FileResponse(file_handle, content_type=mime_type)
4981 # Firefox is not able to handle unicode characters in filename field
4982 # RFC 5987 addresses this issue
4983 # see https://datatracker.ietf.org/doc/html/rfc5987#section-4.2
4984 # Chromium cannot handle commas in the filename
4985 filename_normalized = (
4986 normalize("NFKD", filename.replace(",", "_"))
4987 .encode(
4988 "ascii",
4989 "ignore",
4990 )
4991 .decode("ascii")
4992 .replace("\\", "_")
4993 .replace('"', "_")
4994 )
4995 filename_encoded = quote(filename)
4996 content_disposition = (
4997 f"{disposition}; "
4998 f'filename="{filename_normalized}"; '
4999 f"filename*=utf-8''{filename_encoded}"
5000 )
5001 response["Content-Disposition"] = content_disposition
5002 return response
5005@extend_schema_view(
5006 post=extend_schema(
5007 operation_id="bulk_edit_objects",
5008 description="Perform a bulk edit operation on a list of objects",
5009 external_docs={
5010 "description": "Further documentation",
5011 "url": "https://docs.paperless-ngx.com/api/#objects",
5012 },
5013 responses={
5014 200: inline_serializer(
5015 name="BulkEditResult",
5016 fields={
5017 "result": serializers.CharField(),
5018 },
5019 ),
5020 },
5021 ),
5022)
5023class BulkEditObjectsView(PassUserMixin):
5024 permission_classes = (IsAuthenticated,)
5025 serializer_class = BulkEditObjectsSerializer
5026 parser_classes = (parsers.JSONParser,)
5028 def post(self, request, *args, **kwargs):
5029 serializer = self.get_serializer(data=request.data)
5030 serializer.is_valid(raise_exception=True)
5032 user = self.request.user
5033 object_type = serializer.validated_data.get("object_type")
5034 object_ids = serializer.validated_data.get("objects")
5035 apply_to_all = serializer.validated_data.get("all")
5036 object_class = serializer.get_object_class(object_type)
5037 operation = serializer.validated_data.get("operation")
5038 model_name = object_class._meta.model_name
5039 perm_codename = (
5040 f"change_{model_name}"
5041 if operation == "set_permissions"
5042 else f"delete_{model_name}"
5043 )
5045 if apply_to_all: 5045 ↛ 5074line 5045 didn't jump to line 5074 because the condition on line 5045 was always true
5046 # Support all to avoid sending large lists of ids for bulk operations, with optional filters
5047 filters = serializer.validated_data.get("filters") or {}
5048 filterset_class = {
5049 "tags": TagFilterSet,
5050 "correspondents": CorrespondentFilterSet,
5051 "document_types": DocumentTypeFilterSet,
5052 "storage_paths": StoragePathFilterSet,
5053 }[object_type]
5054 user_permitted_objects = object_class.objects.filter(
5055 id__in=permitted_object_ids(user, object_class, perm_codename),
5056 )
5057 objs = filterset_class(
5058 data=filters,
5059 queryset=user_permitted_objects,
5060 ).qs
5061 if object_type == "tags":
5062 editable_ids = set(user_permitted_objects.values_list("pk", flat=True))
5063 all_ids = set(objs.values_list("pk", flat=True))
5064 for tag in objs:
5065 all_ids.update(
5066 descendant.pk
5067 for descendant in tag.get_descendants()
5068 if descendant.pk in editable_ids
5069 )
5070 objs = object_class.objects.filter(pk__in=all_ids)
5071 objs = objs.select_related("owner")
5072 object_ids = list(objs.values_list("pk", flat=True))
5073 else:
5074 objs = object_class.objects.select_related("owner").filter(
5075 pk__in=object_ids,
5076 )
5078 if not user.is_superuser: 5078 ↛ 5079line 5078 didn't jump to line 5079 because the condition on line 5078 was never true
5079 perm = f"documents.{perm_codename}"
5080 # Limited to the owner (or unowned), same as documents, see BulkEditView
5081 has_perms = (
5082 user.has_perm(perm)
5083 and not objs.exclude(
5084 Q(owner=user) | Q(owner__isnull=True),
5085 ).exists()
5086 )
5088 if not has_perms:
5089 return HttpResponseForbidden("Insufficient permissions")
5091 if operation == "set_permissions":
5092 permissions = serializer.validated_data.get("permissions")
5093 owner = serializer.validated_data.get("owner")
5094 merge = serializer.validated_data.get("merge")
5096 try:
5097 qs = object_class.objects.filter(id__in=object_ids)
5099 # if merge is true, we dont want to remove the owner
5100 if "owner" in serializer.validated_data and (
5101 not merge or (merge and owner is not None)
5102 ):
5103 # if merge is true, we dont want to overwrite the owner
5104 qs_owner_update = qs.filter(owner__isnull=True) if merge else qs
5105 qs_owner_update.update(owner=owner)
5107 if "permissions" in serializer.validated_data: 5107 ↛ 5126line 5107 didn't jump to line 5126 because the condition on line 5107 was always true
5108 set_permissions_for_objects(
5109 permissions=permissions,
5110 model=object_class,
5111 pks=qs.values_list("pk", flat=True),
5112 merge=merge,
5113 )
5115 except Exception as e:
5116 logger.warning(
5117 f"An error occurred performing bulk permissions edit: {e!s}",
5118 )
5119 return HttpResponseBadRequest(
5120 "Error performing bulk permissions edit, check logs for more detail.",
5121 )
5123 elif operation == "delete": 5123 ↛ 5126line 5123 didn't jump to line 5126 because the condition on line 5123 was always true
5124 objs.delete()
5126 return Response({"result": "OK"})
5129class WorkflowTriggerViewSet(ModelViewSet[WorkflowTrigger]):
5130 permission_classes = (IsAuthenticated, PaperlessObjectPermissions)
5132 serializer_class = WorkflowTriggerSerializer
5133 pagination_class = StandardPagination
5135 model = WorkflowTrigger
5137 queryset = WorkflowTrigger.objects.all()
5139 def partial_update(self, request, *args, **kwargs):
5140 if "id" in request.data and str(request.data["id"]) != str(kwargs["pk"]):
5141 return HttpResponseBadRequest(
5142 "ID in body does not match URL",
5143 )
5144 return super().partial_update(request, *args, **kwargs)
5147class WorkflowActionViewSet(ModelViewSet[WorkflowAction]):
5148 permission_classes = (IsAuthenticated, PaperlessObjectPermissions)
5150 serializer_class = WorkflowActionSerializer
5151 pagination_class = StandardPagination
5153 model = WorkflowAction
5155 queryset = WorkflowAction.objects.all().prefetch_related(
5156 "assign_tags",
5157 "assign_view_users",
5158 "assign_view_groups",
5159 "assign_change_users",
5160 "assign_change_groups",
5161 "assign_custom_fields",
5162 )
5164 def partial_update(self, request, *args, **kwargs):
5165 if "id" in request.data and str(request.data["id"]) != str(kwargs["pk"]):
5166 return HttpResponseBadRequest(
5167 "ID in body does not match URL",
5168 )
5169 return super().partial_update(request, *args, **kwargs)
5172class WorkflowViewSet(ModelViewSet[Workflow]):
5173 permission_classes = (IsAuthenticated, PaperlessObjectPermissions)
5175 serializer_class = WorkflowSerializer
5176 pagination_class = StandardPagination
5178 model = Workflow
5180 queryset = (
5181 Workflow.objects.all()
5182 .order_by("order")
5183 .prefetch_related(
5184 Prefetch(
5185 "triggers",
5186 queryset=WorkflowTrigger.objects.prefetch_related(
5187 "filter_has_tags",
5188 "filter_has_all_tags",
5189 "filter_has_not_tags",
5190 "filter_has_any_correspondents",
5191 "filter_has_not_correspondents",
5192 "filter_has_any_document_types",
5193 "filter_has_not_document_types",
5194 "filter_has_any_storage_paths",
5195 "filter_has_not_storage_paths",
5196 ),
5197 ),
5198 Prefetch(
5199 "actions",
5200 queryset=WorkflowAction.objects.order_by(
5201 "order",
5202 "pk",
5203 ).prefetch_related(
5204 "assign_tags",
5205 "assign_view_users",
5206 "assign_view_groups",
5207 "assign_change_users",
5208 "assign_change_groups",
5209 "assign_custom_fields",
5210 "remove_tags",
5211 "remove_correspondents",
5212 "remove_document_types",
5213 "remove_storage_paths",
5214 "remove_custom_fields",
5215 "remove_owners",
5216 "remove_view_users",
5217 "remove_view_groups",
5218 "remove_change_users",
5219 "remove_change_groups",
5220 ),
5221 ),
5222 )
5223 )
5226class CustomFieldViewSet(PermissionsAwareDocumentCountMixin, ModelViewSet[CustomField]):
5227 permission_classes = (IsAuthenticated, PaperlessObjectPermissions)
5229 serializer_class = CustomFieldSerializer
5230 pagination_class = StandardPagination
5231 filter_backends = (
5232 DjangoFilterBackend,
5233 OrderingFilter,
5234 )
5235 filterset_class = CustomFieldFilterSet
5237 model = CustomField
5238 document_count_through = CustomFieldInstance
5239 document_count_source_field = "field_id"
5241 queryset = CustomField.objects.all().order_by("name")
5244@extend_schema_view(
5245 get=extend_schema(
5246 description="Get the current system status of the Paperless-NGX server",
5247 responses={
5248 (200, "application/json"): inline_serializer(
5249 name="SystemStatus",
5250 fields={
5251 "pngx_version": serializers.CharField(),
5252 "server_os": serializers.CharField(),
5253 "install_type": serializers.CharField(),
5254 "storage": inline_serializer(
5255 name="Storage",
5256 fields={
5257 "total": serializers.IntegerField(),
5258 "available": serializers.IntegerField(),
5259 },
5260 ),
5261 "database": inline_serializer(
5262 name="Database",
5263 fields={
5264 "type": serializers.CharField(),
5265 "url": serializers.CharField(),
5266 "status": serializers.CharField(),
5267 "error": serializers.CharField(),
5268 "migration_status": inline_serializer(
5269 name="MigrationStatus",
5270 fields={
5271 "latest_migration": serializers.CharField(),
5272 "unapplied_migrations": serializers.ListSerializer(
5273 child=serializers.CharField(),
5274 ),
5275 },
5276 ),
5277 },
5278 ),
5279 "tasks": inline_serializer(
5280 name="Tasks",
5281 fields={
5282 "redis_url": serializers.CharField(),
5283 "redis_status": serializers.CharField(),
5284 "redis_error": serializers.CharField(),
5285 "celery_status": serializers.CharField(),
5286 "summary": inline_serializer(
5287 name="TasksSummaryOverview",
5288 fields={
5289 "days": serializers.IntegerField(),
5290 "total_count": serializers.IntegerField(),
5291 "pending_count": serializers.IntegerField(),
5292 "success_count": serializers.IntegerField(),
5293 "failure_count": serializers.IntegerField(),
5294 },
5295 ),
5296 },
5297 ),
5298 "index": inline_serializer(
5299 name="Index",
5300 fields={
5301 "status": serializers.CharField(),
5302 "error": serializers.CharField(),
5303 "last_modified": serializers.DateTimeField(),
5304 },
5305 ),
5306 "classifier": inline_serializer(
5307 name="Classifier",
5308 fields={
5309 "status": serializers.CharField(),
5310 "error": serializers.CharField(),
5311 "last_trained": serializers.DateTimeField(),
5312 },
5313 ),
5314 "sanity_check": inline_serializer(
5315 name="SanityCheck",
5316 fields={
5317 "status": serializers.CharField(),
5318 "error": serializers.CharField(),
5319 "last_run": serializers.DateTimeField(),
5320 },
5321 ),
5322 },
5323 ),
5324 },
5325 ),
5326)
5327class SystemStatusView(PassUserMixin):
5328 permission_classes = (IsAuthenticated,)
5329 TASK_SUMMARY_DAYS = 30
5331 def get(self, request, format=None):
5332 if not has_system_status_permission(request.user): 5332 ↛ 5333line 5332 didn't jump to line 5333 because the condition on line 5332 was never true
5333 return HttpResponseForbidden("Insufficient permissions")
5335 current_version = version.__full_version_str__
5337 install_type = "bare-metal"
5338 if os.environ.get("KUBERNETES_SERVICE_HOST") is not None: 5338 ↛ 5339line 5338 didn't jump to line 5339 because the condition on line 5338 was never true
5339 install_type = "kubernetes"
5340 elif os.environ.get("PNGX_CONTAINERIZED") == "1": 5340 ↛ 5343line 5340 didn't jump to line 5343 because the condition on line 5340 was always true
5341 install_type = "docker"
5343 db_conn = connections["default"]
5344 db_url = str(db_conn.settings_dict["NAME"])
5345 db_error = None
5347 try:
5348 db_conn.ensure_connection()
5349 db_status = "OK"
5350 loader = MigrationLoader(connection=db_conn)
5351 all_migrations = [f"{app}.{name}" for app, name in loader.graph.nodes]
5352 applied_migrations = [
5353 f"{m.app}.{m.name}"
5354 for m in MigrationRecorder.Migration.objects.all().order_by("id")
5355 ]
5356 except Exception as e: # pragma: no cover
5357 applied_migrations = []
5358 db_status = "ERROR"
5359 logger.exception(
5360 f"System status detected a possible problem while connecting to the database: {e}",
5361 )
5362 db_error = "Error connecting to database, check logs for more detail."
5364 media_stats = os.statvfs(settings.MEDIA_ROOT)
5366 redis_url = settings._CHANNELS_REDIS_URL
5367 redis_url_parsed = urlparse(redis_url)
5368 redis_constructed_url = f"{redis_url_parsed.scheme}://{redis_url_parsed.path or redis_url_parsed.hostname}"
5369 if redis_url_parsed.hostname is not None: 5369 ↛ 5371line 5369 didn't jump to line 5371 because the condition on line 5369 was always true
5370 redis_constructed_url += f":{redis_url_parsed.port}"
5371 redis_error = None
5372 with Redis.from_url(url=redis_url) as client:
5373 try:
5374 client.ping()
5375 redis_status = "OK"
5376 except Exception as e:
5377 redis_status = "ERROR"
5378 logger.exception(
5379 f"System status detected a possible problem while connecting to redis: {e}",
5380 )
5381 redis_error = "Error connecting to redis, check logs for more detail."
5383 celery_error = None
5384 celery_url = None
5385 try:
5386 celery_ping = None
5387 for ping_attempt in range(3):
5388 celery_ping = celery_app.control.inspect().ping()
5389 if celery_ping: 5389 ↛ 5390line 5389 didn't jump to line 5390 because the condition on line 5389 was never true
5390 break
5391 if ping_attempt < 2:
5392 sleep(0.25)
5394 if not celery_ping: 5394 ↛ 5400line 5394 didn't jump to line 5400 because the condition on line 5394 was always true
5395 celery_active = "WARNING"
5396 celery_error = (
5397 "No celery workers responded to ping. This may be temporary."
5398 )
5399 else:
5400 celery_url, first_worker_ping = next(iter(celery_ping.items()))
5401 if (
5402 isinstance(first_worker_ping, dict)
5403 and first_worker_ping.get("ok") == "pong"
5404 ):
5405 celery_active = "OK"
5406 else:
5407 celery_active = "WARNING"
5408 celery_error = "Celery worker responded unexpectedly."
5409 except Exception as e:
5410 celery_active = "ERROR"
5411 logger.exception(
5412 f"System status detected a possible problem while connecting to celery: {e}",
5413 )
5414 celery_error = "Error connecting to celery, check logs for more detail."
5416 index_error = None
5417 try:
5418 from documents.search import get_backend
5420 get_backend() # triggers open/rebuild; raises on error
5421 index_status = "OK"
5422 # Use the most-recently modified file in the index directory as a proxy
5423 # for last index write time (Tantivy has no single last_modified() call).
5424 index_dir = settings.INDEX_DIR
5425 mtimes = [p.stat().st_mtime for p in index_dir.iterdir() if p.is_file()]
5426 index_last_modified = (
5427 make_aware(datetime.fromtimestamp(max(mtimes))) if mtimes else None
5428 )
5429 except Exception as e:
5430 index_status = "ERROR"
5431 index_error = "Error opening index, check logs for more detail."
5432 logger.exception(
5433 f"System status detected a possible problem while opening the index: {e}",
5434 )
5435 index_last_modified = None
5437 last_trained_task = (
5438 PaperlessTask.objects.filter(
5439 task_type=PaperlessTask.TaskType.TRAIN_CLASSIFIER,
5440 status__in=PaperlessTask.COMPLETE_STATUSES, # ignore running tasks
5441 )
5442 .order_by("-date_done")
5443 .first()
5444 )
5445 classifier_status = "OK"
5446 classifier_error = None
5447 if last_trained_task is None: 5447 ↛ 5450line 5447 didn't jump to line 5450 because the condition on line 5447 was always true
5448 classifier_status = "WARNING"
5449 classifier_error = "No classifier training tasks found"
5450 elif last_trained_task.status != PaperlessTask.Status.SUCCESS:
5451 classifier_status = "ERROR"
5452 classifier_error = (
5453 last_trained_task.result_data.get("error_message")
5454 if last_trained_task.result_data
5455 else None
5456 )
5457 classifier_last_trained = (
5458 last_trained_task.date_done if last_trained_task else None
5459 )
5461 last_sanity_check = (
5462 PaperlessTask.objects.filter(
5463 task_type=PaperlessTask.TaskType.SANITY_CHECK,
5464 status__in=PaperlessTask.COMPLETE_STATUSES, # ignore running tasks
5465 )
5466 .order_by("-date_done")
5467 .first()
5468 )
5469 sanity_check_status = "OK"
5470 sanity_check_error = None
5471 if last_sanity_check is None: 5471 ↛ 5474line 5471 didn't jump to line 5474 because the condition on line 5471 was always true
5472 sanity_check_status = "WARNING"
5473 sanity_check_error = "No sanity check tasks found"
5474 elif last_sanity_check.status != PaperlessTask.Status.SUCCESS:
5475 sanity_check_status = "ERROR"
5476 sanity_check_error = (
5477 last_sanity_check.result_data.get("error_message")
5478 if last_sanity_check.result_data
5479 else None
5480 )
5481 sanity_check_last_run = (
5482 last_sanity_check.date_done if last_sanity_check else None
5483 )
5485 ai_config = AIConfig()
5486 if not ai_config.llm_index_enabled: 5486 ↛ 5491line 5486 didn't jump to line 5491 because the condition on line 5486 was always true
5487 llmindex_status = "DISABLED"
5488 llmindex_error = None
5489 llmindex_last_modified = None
5490 else:
5491 last_llmindex_update = (
5492 PaperlessTask.objects.filter(
5493 task_type=PaperlessTask.TaskType.LLM_INDEX,
5494 )
5495 .order_by("-date_done")
5496 .first()
5497 )
5498 llmindex_status = "OK"
5499 llmindex_error = None
5500 if last_llmindex_update is None:
5501 llmindex_status = "WARNING"
5502 llmindex_error = "No LLM index update tasks found"
5503 elif last_llmindex_update.status == PaperlessTask.Status.FAILURE:
5504 llmindex_status = "ERROR"
5505 llmindex_error = (
5506 last_llmindex_update.result_data.get("error_message")
5507 if last_llmindex_update.result_data
5508 else None
5509 )
5510 llmindex_last_modified = (
5511 last_llmindex_update.date_done if last_llmindex_update else None
5512 )
5514 summary_cutoff = timezone.now() - timedelta(days=self.TASK_SUMMARY_DAYS)
5515 task_summary_agg = PaperlessTask.objects.filter(
5516 date_created__gte=summary_cutoff,
5517 ).aggregate(
5518 total_count=Count("id"),
5519 pending_count=Count(
5520 "id",
5521 filter=Q(status=PaperlessTask.Status.PENDING),
5522 ),
5523 success_count=Count(
5524 "id",
5525 filter=Q(status=PaperlessTask.Status.SUCCESS),
5526 ),
5527 failure_count=Count(
5528 "id",
5529 filter=Q(status=PaperlessTask.Status.FAILURE),
5530 ),
5531 )
5532 task_summary = {
5533 "days": self.TASK_SUMMARY_DAYS,
5534 **task_summary_agg,
5535 }
5537 return Response(
5538 {
5539 "pngx_version": current_version,
5540 "server_os": platform.platform(),
5541 "install_type": install_type,
5542 "storage": {
5543 "total": media_stats.f_frsize * media_stats.f_blocks,
5544 "available": media_stats.f_frsize * media_stats.f_bavail,
5545 },
5546 "database": {
5547 "type": db_conn.vendor,
5548 "url": db_url,
5549 "status": db_status,
5550 "error": db_error,
5551 "migration_status": {
5552 "latest_migration": applied_migrations[-1],
5553 "unapplied_migrations": [
5554 m for m in all_migrations if m not in applied_migrations
5555 ],
5556 },
5557 },
5558 "tasks": {
5559 "redis_url": redis_constructed_url,
5560 "redis_status": redis_status,
5561 "redis_error": redis_error,
5562 "celery_status": celery_active,
5563 "celery_url": celery_url,
5564 "celery_error": celery_error,
5565 "index_status": index_status,
5566 "index_last_modified": index_last_modified,
5567 "index_error": index_error,
5568 "classifier_status": classifier_status,
5569 "classifier_last_trained": classifier_last_trained,
5570 "classifier_error": classifier_error,
5571 "sanity_check_status": sanity_check_status,
5572 "sanity_check_last_run": sanity_check_last_run,
5573 "sanity_check_error": sanity_check_error,
5574 "llmindex_status": llmindex_status,
5575 "llmindex_last_modified": llmindex_last_modified,
5576 "llmindex_error": llmindex_error,
5577 "summary": task_summary,
5578 },
5579 },
5580 )
5583class TrashView(ListModelMixin, PassUserMixin):
5584 permission_classes = (IsAuthenticated, TrashPermissions)
5585 serializer_class = TrashSerializer
5587 class _TrashPermittedObjectsFilter(PermittedObjectsFilter):
5588 include_granted = False
5590 filter_backends = (_TrashPermittedObjectsFilter,)
5591 pagination_class = StandardPagination
5593 model = Document
5595 # A version is listed separately only when its root is not in the trash.
5596 queryset = Document.deleted_objects.exclude(
5597 root_document_id__in=Document.deleted_objects.values("id"),
5598 )
5600 def get(self, request: Request, format: str | None = None) -> Response:
5601 self.serializer_class = DocumentSerializer
5602 return self.list(request, format)
5604 def post(
5605 self,
5606 request: Request,
5607 *args: Any,
5608 **kwargs: Any,
5609 ) -> Response | HttpResponse:
5610 serializer = self.get_serializer(data=request.data)
5611 serializer.is_valid(raise_exception=True)
5613 doc_ids = serializer.validated_data.get("documents")
5614 docs = (
5615 Document.global_objects.filter(id__in=doc_ids)
5616 if doc_ids is not None
5617 else self.filter_queryset(self.get_queryset()).all()
5618 )
5619 if docs.exclude( 5619 ↛ 5626line 5619 didn't jump to line 5626 because the condition on line 5619 was never true
5620 pk__in=permitted_document_ids(
5621 request.user,
5622 perm="delete_document",
5623 include_deleted=True,
5624 ),
5625 ).exists():
5626 return HttpResponseForbidden("Insufficient permissions")
5627 action = serializer.validated_data.get("action")
5628 if action == "restore":
5629 restored = list(self.get_queryset().filter(id__in=doc_ids))
5630 if len(restored) != len(doc_ids): 5630 ↛ 5631line 5630 didn't jump to line 5631 because the condition on line 5630 was never true
5631 raise ValidationError(
5632 {
5633 "documents": [
5634 "Restore the root document instead of one of its versions.",
5635 ],
5636 },
5637 )
5638 for doc in restored: 5638 ↛ 5639line 5638 didn't jump to line 5639 because the loop on line 5638 never started
5639 doc.restore(strict=False)
5640 if restored: 5640 ↛ 5641line 5640 didn't jump to line 5641 because the condition on line 5640 was never true
5641 from documents.search import get_backend
5643 with get_backend().batch_update() as batch:
5644 batch.add_or_update_ids([doc.pk for doc in restored])
5645 elif action == "empty": 5645 ↛ 5649line 5645 didn't jump to line 5649 because the condition on line 5645 was always true
5646 if doc_ids is None:
5647 doc_ids = [doc.id for doc in docs]
5648 empty_trash(doc_ids=doc_ids)
5649 return Response({"result": "OK", "doc_ids": doc_ids})
5652def serve_logo(request: HttpRequest, filename: str | None = None) -> FileResponse:
5653 """
5654 Serves the configured logo file with Content-Disposition: attachment.
5655 Prevents inline execution of SVGs. See GHSA-6p53-hqqw-8j62
5656 """
5657 config = ApplicationConfiguration.objects.first()
5658 app_logo = config.app_logo
5660 if app_logo:
5661 path = Path(app_logo.path)
5662 logo_name = app_logo.name
5663 else:
5664 if not settings.APP_LOGO:
5665 raise Http404("No logo configured")
5667 logo_root = (Path(settings.MEDIA_ROOT) / "logo").resolve()
5668 path = (Path(settings.MEDIA_ROOT) / settings.APP_LOGO.lstrip("/")).resolve()
5669 if not path.is_relative_to(logo_root) or not path.is_file():
5670 raise Http404("Configured logo not found")
5672 logo_name = path.name
5674 content_type = magic.from_file(path, mime=True) or "application/octet-stream"
5675 logo_file = app_logo.open("rb") if app_logo else path.open("rb")
5677 return FileResponse(
5678 logo_file,
5679 content_type=content_type,
5680 filename=logo_name,
5681 as_attachment=True,
5682 )